use a bloom filter to determine if a user is a class student

uses sun.misc.Unsafe! If production blows up, that's why.
pull/8158/head
Thibault Duplessis 2021-02-10 22:46:41 +01:00
parent c98b68d7bc
commit 4e8c4cfbd4
9 changed files with 120 additions and 69 deletions

View File

@ -24,15 +24,13 @@ final class Clas(
Ok(views.html.clas.clas.teacherIndex(classes)) Ok(views.html.clas.clas.teacherIndex(classes))
} }
case Some(me) => case Some(me) =>
env.clas.api.student.isStudent(me.id) flatMap { if (env.clas.studentCache.isStudent(me.id))
case false => renderHome env.clas.api.student.clasIdsOfUser(me.id) flatMap
case _ => env.clas.api.clas.byIds map {
env.clas.api.student.clasIdsOfUser(me.id) flatMap case List(single) => Redirect(routes.Clas.show(single.id.value))
env.clas.api.clas.byIds map { case many => Ok(views.html.clas.clas.studentIndex(many))
case List(single) => Redirect(routes.Clas.show(single.id.value)) }
case many => Ok(views.html.clas.clas.studentIndex(many)) else renderHome
}
}
} }
} }

View File

@ -515,35 +515,30 @@ abstract private[controllers] class LilaController(val env: Env)
val isPage = HTTPRequest isSynchronousHttp ctx.req val isPage = HTTPRequest isSynchronousHttp ctx.req
val nonce = isPage option Nonce.random val nonce = isPage option Nonce.random
ctx.me.fold(fuccess(PageData.anon(ctx.req, nonce, blindMode(ctx)))) { me => ctx.me.fold(fuccess(PageData.anon(ctx.req, nonce, blindMode(ctx)))) { me =>
env.pref.api.getPref(me, ctx.req) zip env.pref.api.getPref(me, ctx.req) zip {
(if (isGranted(_.Teacher, me)) fuccess(true) else env.clas.api.student.isStudent(me.id)) zip { if (isPage) {
if (isPage) { env.user.lightUserApi preloadUser me
env.user.lightUserApi preloadUser me env.team.api.nbRequests(me.id) zip
env.team.api.nbRequests(me.id) zip env.challenge.api.countInFor.get(me.id) zip
env.challenge.api.countInFor.get(me.id) zip env.notifyM.api.unreadCount(Notifies(me.id)).dmap(_.value) zip
env.notifyM.api.unreadCount(Notifies(me.id)).dmap(_.value) zip env.mod.inquiryApi.forMod(me)
env.mod.inquiryApi.forMod(me) } else
} else fuccess {
fuccess { (((0, 0), 0), none)
(((0, 0), 0), none) }
} } map { case (pref, teamNbRequests ~ nbChallenges ~ nbNotifications ~ inquiry) =>
} map { PageData(
case ( teamNbRequests,
(pref, hasClas), nbChallenges,
teamNbRequests ~ nbChallenges ~ nbNotifications ~ inquiry nbNotifications,
) => pref,
PageData( blindMode = blindMode(ctx),
teamNbRequests, hasFingerprint = hasFingerPrint,
nbChallenges, hasClas = isGranted(_.Teacher, me) || env.clas.studentCache.isStudent(me.id),
nbNotifications, inquiry = inquiry,
pref, nonce = nonce
blindMode = blindMode(ctx), )
hasFingerprint = hasFingerPrint, }
hasClas = hasClas,
inquiry = inquiry,
nonce = nonce
)
}
} }
} }

View File

@ -371,7 +371,7 @@ lazy val teamSearch = module("teamSearch",
lazy val clas = module("clas", lazy val clas = module("clas",
Seq(common, memo, db, user, security, msg, history, puzzle), Seq(common, memo, db, user, security, msg, history, puzzle),
reactivemongo.bundle reactivemongo.bundle ++ Seq(bloomFilter)
) )
lazy val bookmark = module("bookmark", lazy val bookmark = module("bookmark",

View File

@ -182,7 +182,7 @@ final class ClasApi(
): Fu[Student.WithPassword] = { ): Fu[Student.WithPassword] = {
val email = EmailAddress(s"noreply.class.${clas.id}.${data.username}@lichess.org") val email = EmailAddress(s"noreply.class.${clas.id}.${data.username}@lichess.org")
val password = Student.password.generate val password = Student.password.generate
lila.mon.clas.studentCreate(teacher.id) lila.mon.clas.student.create(teacher.id).increment()
userRepo userRepo
.create( .create(
username = data.username, username = data.username,
@ -260,7 +260,7 @@ ${clas.desc}""",
student student
.archive(Student.id(user.id, clas.id), user, v = false) .archive(Student.id(user.id, clas.id), user, v = false)
.map2[ClasInvite.Feedback](_ => Already) getOrElse { .map2[ClasInvite.Feedback](_ => Already) getOrElse {
lila.mon.clas.studentInvite(teacher.id) lila.mon.clas.student.invite(teacher.id).increment()
val invite = ClasInvite.make(clas, user, realName, teacher) val invite = ClasInvite.make(clas, user, realName, teacher)
colls.invite.insert colls.invite.insert
.one(invite) .one(invite)

View File

@ -44,7 +44,7 @@ final class ClasProgressApi(
gameRepo: GameRepo, gameRepo: GameRepo,
historyApi: lila.history.HistoryApi, historyApi: lila.history.HistoryApi,
puzzleColls: lila.puzzle.PuzzleColls, puzzleColls: lila.puzzle.PuzzleColls,
getStudentIds: () => Fu[Set[User.ID]] studentCache: ClasStudentCache
)(implicit ec: scala.concurrent.ExecutionContext) { )(implicit ec: scala.concurrent.ExecutionContext) {
case class PlayStats(nb: Int, wins: Int, millis: Long) case class PlayStats(nb: Int, wins: Int, millis: Long)
@ -166,8 +166,5 @@ final class ClasProgressApi(
} }
private[clas] def onFinishGame(game: lila.game.Game): Unit = private[clas] def onFinishGame(game: lila.game.Game): Unit =
if (game.userIds.nonEmpty) if (game.userIds.exists(studentCache.isStudent)) gameRepo.denormalizePerfType(game)
getStudentIds() foreach { studentIds =>
if (game.userIds.exists(studentIds.contains)) gameRepo.denormalizePerfType(game)
}
} }

View File

@ -0,0 +1,52 @@
package lila.clas
import akka.actor.Scheduler
import akka.stream.Materializer
import akka.stream.scaladsl._
import bloomfilter.mutable.BloomFilter
import reactivemongo.akkastream.{ cursorProducer, AkkaStreamCursor }
import reactivemongo.api.ReadPreference
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext
import lila.db.dsl._
import lila.memo.CacheApi
import lila.user.User
final class ClasStudentCache(colls: ClasColls, cacheApi: CacheApi)(implicit
ec: ExecutionContext,
scheduler: Scheduler,
mat: Materializer
) {
private val expectedElements = 200_000
private val falsePositiveRate = 0.001
private var bloomFilter = BloomFilter[User.ID](expectedElements, falsePositiveRate)
def isStudent(userId: User.ID) = bloomFilter mightContain userId pp userId
private def rebuildBloomFilter(): Unit = {
val nextBloom = BloomFilter[User.ID](expectedElements, falsePositiveRate)
colls.student
.find($empty, $doc("userId" -> true).some)
.cursor[Bdoc](ReadPreference.secondaryPreferred)
.documentSource()
.toMat(Sink.fold[Int, Bdoc](0) { case (total, doc) =>
doc string "userId" foreach nextBloom.add
total + 1
})(Keep.right)
.run()
.addEffect { nb =>
lila.mon.clas.student.bloomFilter.count.update(nb)
bloomFilter.dispose()
bloomFilter = nextBloom
}
.monSuccess(_.clas.student.bloomFilter.fu)
.addEffectAnyway {
scheduler.scheduleOnce(5 minutes) { rebuildBloomFilter() }.unit
}
.unit
}
scheduler.scheduleOnce(20 seconds) { rebuildBloomFilter() }.unit
}

View File

@ -17,17 +17,21 @@ final class Env(
authenticator: lila.user.Authenticator, authenticator: lila.user.Authenticator,
cacheApi: lila.memo.CacheApi, cacheApi: lila.memo.CacheApi,
baseUrl: BaseUrl baseUrl: BaseUrl
)(implicit ec: scala.concurrent.ExecutionContext) { )(implicit
ec: scala.concurrent.ExecutionContext,
scheduler: akka.actor.Scheduler,
mat: akka.stream.Materializer
) {
lazy val nameGenerator = wire[NameGenerator] lazy val nameGenerator: NameGenerator = wire[NameGenerator]
lazy val forms = wire[ClasForm] lazy val forms = wire[ClasForm]
private val colls = wire[ClasColls] private val colls = wire[ClasColls]
lazy val api: ClasApi = wire[ClasApi] lazy val studentCache = wire[ClasStudentCache]
private def getStudentIds = () => api.student.allIds lazy val api: ClasApi = wire[ClasApi]
lazy val progressApi = wire[ClasProgressApi] lazy val progressApi = wire[ClasProgressApi]

View File

@ -355,6 +355,10 @@ object mon {
object student { object student {
def create(teacher: String) = counter("clas.student.create").withTag("teacher", teacher) def create(teacher: String) = counter("clas.student.create").withTag("teacher", teacher)
def invite(teacher: String) = counter("clas.student.invite").withTag("teacher", teacher) def invite(teacher: String) = counter("clas.student.invite").withTag("teacher", teacher)
object bloomFilter {
def count = gauge("clas.student.bloomFilter.count").withoutTags()
def fu = future("clas.student.bloomFilter.future")
}
} }
} }
object tournament { object tournament {

View File

@ -5,25 +5,26 @@ object Dependencies {
val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/ornicar/lila-maven/master" val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/ornicar/lila-maven/master"
val scalalib = "com.github.ornicar" %% "scalalib" % "7.0.2" val scalalib = "com.github.ornicar" %% "scalalib" % "7.0.2"
val hasher = "com.roundeights" %% "hasher" % "1.2.1" val hasher = "com.roundeights" %% "hasher" % "1.2.1"
val jodaTime = "joda-time" % "joda-time" % "2.10.10" val jodaTime = "joda-time" % "joda-time" % "2.10.10"
val chess = "org.lichess" %% "scalachess" % "10.2.0" val chess = "org.lichess" %% "scalachess" % "10.2.0"
val compression = "org.lichess" %% "compression" % "1.6" val compression = "org.lichess" %% "compression" % "1.6"
val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.3.1-THIB" val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.3.1-THIB"
val prismic = "io.prismic" %% "scala-kit" % "1.2.19-THIB213" val prismic = "io.prismic" %% "scala-kit" % "1.2.19-THIB213"
val scrimage = "com.sksamuel.scrimage" % "scrimage-core" % "4.0.17" val scrimage = "com.sksamuel.scrimage" % "scrimage-core" % "4.0.17"
val scaffeine = "com.github.blemale" %% "scaffeine" % "4.0.2" % "compile" val scaffeine = "com.github.blemale" %% "scaffeine" % "4.0.2" % "compile"
val googleOAuth = "com.google.auth" % "google-auth-library-oauth2-http" % "0.23.0" val googleOAuth = "com.google.auth" % "google-auth-library-oauth2-http" % "0.23.0"
val scalaUri = "io.lemonlabs" %% "scala-uri" % "2.3.1" val scalaUri = "io.lemonlabs" %% "scala-uri" % "2.3.1"
val scalatags = "com.lihaoyi" %% "scalatags" % "0.9.3" val scalatags = "com.lihaoyi" %% "scalatags" % "0.9.3"
val lettuce = "io.lettuce" % "lettuce-core" % "5.3.6.RELEASE" val lettuce = "io.lettuce" % "lettuce-core" % "5.3.6.RELEASE"
val epoll = "io.netty" % "netty-transport-native-epoll" % "4.1.58.Final" classifier "linux-x86_64" val epoll = "io.netty" % "netty-transport-native-epoll" % "4.1.58.Final" classifier "linux-x86_64"
val autoconfig = "io.methvin.play" %% "autoconfig-macros" % "0.3.2" % "provided" val autoconfig = "io.methvin.play" %% "autoconfig-macros" % "0.3.2" % "provided"
val scalatest = "org.scalatest" %% "scalatest" % "3.1.0" % Test val scalatest = "org.scalatest" %% "scalatest" % "3.1.0" % Test
val uaparser = "org.uaparser" %% "uap-scala" % "0.11.0" val uaparser = "org.uaparser" %% "uap-scala" % "0.11.0"
val specs2 = "org.specs2" %% "specs2-core" % "4.10.6" % Test val specs2 = "org.specs2" %% "specs2-core" % "4.10.6" % Test
val apacheText = "org.apache.commons" % "commons-text" % "1.9" val apacheText = "org.apache.commons" % "commons-text" % "1.9"
val bloomFilter = "com.github.alexandrnikitin" %% "bloom-filter" % "0.13.1"
object flexmark { object flexmark {
val version = "0.50.50" val version = "0.50.50"