deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

faster GC with retries

pull/4736/head
Thibault Duplessis 2018-12-01 12:41:55 +07:00
parent ebd7b716f0
commit c9774bd525
5 changed files with 50 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/common/src/main/Future.scala
View File

@ -58,10 +58,10 @@ object Future {
if (duration == 0.millis) run
else run zip akka.pattern.after(duration, system.scheduler)(funit) dmap (_._1)
def retry[T](op: => Fu[T], delay: FiniteDuration, retries: Int, logger: lila.log.Logger)(implicit system: akka.actor.ActorSystem): Fu[T] =
op recoverWith {
def retry[T](op: () => Fu[T], delay: FiniteDuration, retries: Int, logger: Option[lila.log.Logger])(implicit system: akka.actor.ActorSystem): Fu[T] =
op() recoverWith {
case e if retries > 0 =>
logger.info(s"$retries retries - ${e.getMessage}")
logger foreach { _.info(s"$retries retries - ${e.getMessage}") }
akka.pattern.after(delay, system.scheduler)(retry(op, delay, retries - 1, logger))
}
}

8
modules/gameSearch/src/main/GameSearchApi.scala
View File

@ -30,10 +30,10 @@ final class GameSearchApi(
def store(game: Game) = storable(game) ?? {
GameRepo isAnalysed game.id flatMap { analysed =>
lila.common.Future.retry(
client.store(Id(game.id), toDoc(game, analysed)),
10.seconds,
5,
logger
() => client.store(Id(game.id), toDoc(game, analysed)),
delay = 10.seconds,
retries = 5,
logger.some
)(system)
}
}

60
modules/security/src/main/GarbageCollector.scala
View File

@ -1,8 +1,8 @@
package lila.security
import org.joda.time.DateTime
import scala.concurrent.duration._
import play.api.mvc.RequestHeader
import scala.concurrent.duration._
import lila.common.{ EmailAddress, IpAddress, HTTPRequest }
import lila.user.{ User, UserRepo }
@ -18,41 +18,51 @@ final class GarbageCollector(
private val done = new lila.memo.ExpireSetMemo(10 minutes)
private case class ApplyData(user: User, ip: IpAddress, email: EmailAddress, req: RequestHeader)
// User just signed up and doesn't have security data yet, so wait a bit
def delay(user: User, email: EmailAddress, req: RequestHeader): Unit =
if (user.createdAt.isAfter(DateTime.now minusDays 3)) {
val ip = HTTPRequest lastRemoteAddress req
debug(email, s"${user.username} $email $ip", "pre")
system.scheduler.scheduleOnce(1 minute) {
apply(user, ip, email, req)
system.scheduler.scheduleOnce(6 seconds) {
val applyData = ApplyData(user, ip, email, req)
lila.common.Future.retry(
() => ensurePrintAvailable(applyData),
delay = 10 seconds,
retries = 5,
logger = none
)(system).nevermind >> apply(applyData)
}
}
private def apply(user: User, ip: IpAddress, email: EmailAddress, req: RequestHeader): Funit =
userSpy(user) flatMap { spy =>
system.lilaBus.publish(
lila.security.Signup(user, email, req, spy.prints.headOption.map(_.value)),
'userSignup
)
debug(email, spy, s"spy ${user.username}")
badOtherAccounts(spy.otherUsers.map(_.user)) ?? { others =>
debug(email, others.map(_.id), s"others ${user.username}")
lila.common.Future.exists(spy.ips)(ipTrust.isSuspicious).map {
_ ?? {
val ipBan = spy.usersSharingIp.forall { u =>
isBadAccount(u) || !u.seenAt.exists(DateTime.now.minusMonths(2).isBefore)
}
if (!done.get(user.id)) {
collect(user, email, others, ipBan)
done put user.id
private def ensurePrintAvailable(data: ApplyData): Funit =
userSpy userHasPrint data.user flatMap {
case false => fufail("No print available yet")
case _ => funit
}
private def apply(data: ApplyData): Funit = data match {
case ApplyData(user, ip, email, req) =>
userSpy(user) flatMap { spy =>
system.lilaBus.publish(
lila.security.Signup(user, email, req, spy.prints.headOption.map(_.value)),
'userSignup
)
badOtherAccounts(spy.otherUsers.map(_.user)) ?? { others =>
lila.common.Future.exists(spy.ips)(ipTrust.isSuspicious).map {
_ ?? {
val ipBan = spy.usersSharingIp.forall { u =>
isBadAccount(u) || !u.seenAt.exists(DateTime.now.minusMonths(2).isBefore)
}
if (!done.get(user.id)) {
collect(user, email, others, ipBan)
done put user.id
}
}
}
}
}
}
private def debug(email: EmailAddress, stuff: Any, as: String = "-") =
if (email.value contains "iralas".reverse) logger.info(s"GC debug $as: $stuff")
}
private def badOtherAccounts(accounts: Set[User]): Option[List[User]] = {
val others = accounts.toList

9
modules/security/src/main/UserSpy.scala
View File

@ -1,8 +1,8 @@
package lila.security
import scala.collection.breakOut
import reactivemongo.api.ReadPreference
import org.joda.time.DateTime
import reactivemongo.api.ReadPreference
import scala.collection.breakOut
import lila.common.IpAddress
import lila.db.dsl._
@ -57,6 +57,11 @@ private[security] final class UserSpyApi(firewall: Firewall, geoIP: GeoIP, coll:
usersSharingFingerprint = sharingFingerprint
)
private[security] def userHasPrint(u: User): Fu[Boolean] = coll.exists(
$doc("user" -> u.id, "fp" $exists true),
readPreference = ReadPreference.secondaryPreferred
)
private def exploreSimilar(field: String)(user: User)(implicit coll: Coll): Fu[Set[User]] =
nextValues(field)(user) flatMap { nValues =>
nextUsers(field)(nValues, user)

2
modules/studySearch/src/main/StudySearchApi.scala
View File

@ -111,7 +111,7 @@ final class StudySearchApi(
Enumeratee.grouped(Iteratee takeUpTo 12) |>>>
Iteratee.foldM[Seq[Study], Int](0) {
case (nb, studies) => studies.map { study =>
lila.common.Future.retry(doStore(study), 5 seconds, 10, retryLogger)(system)
lila.common.Future.retry(() => doStore(study), 5 seconds, 10, retryLogger.some)(system)
}.sequenceFu inject {
studies.headOption.ifTrue(nb % 100 == 0) foreach { study =>
logger.info(s"Indexed $nb studies - ${study.createdAt}")