deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

use a bloom filter to determine if a user is a class student 

uses sun.misc.Unsafe! If production blows up, that's why.
pull/8158/head
Thibault Duplessis 2021-02-10 22:46:41 +01:00
parent c98b68d7bc
commit 4e8c4cfbd4
9 changed files with 120 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/controllers/Clas.scala
View File

@ -24,15 +24,13 @@ final class Clas(
Ok(views.html.clas.clas.teacherIndex(classes))
}
case Some(me) =>
env.clas.api.student.isStudent(me.id) flatMap {
case false => renderHome
case _ =>
env.clas.api.student.clasIdsOfUser(me.id) flatMap
env.clas.api.clas.byIds map {
case List(single) => Redirect(routes.Clas.show(single.id.value))
case many => Ok(views.html.clas.clas.studentIndex(many))
}
}
if (env.clas.studentCache.isStudent(me.id))
env.clas.api.student.clasIdsOfUser(me.id) flatMap
env.clas.api.clas.byIds map {
case List(single) => Redirect(routes.Clas.show(single.id.value))
case many => Ok(views.html.clas.clas.studentIndex(many))
}
else renderHome
}
}

53
app/controllers/LilaController.scala
View File

@ -515,35 +515,30 @@ abstract private[controllers] class LilaController(val env: Env)
val isPage = HTTPRequest isSynchronousHttp ctx.req
val nonce = isPage option Nonce.random
ctx.me.fold(fuccess(PageData.anon(ctx.req, nonce, blindMode(ctx)))) { me =>
env.pref.api.getPref(me, ctx.req) zip
(if (isGranted(_.Teacher, me)) fuccess(true) else env.clas.api.student.isStudent(me.id)) zip {
if (isPage) {
env.user.lightUserApi preloadUser me
env.team.api.nbRequests(me.id) zip
env.challenge.api.countInFor.get(me.id) zip
env.notifyM.api.unreadCount(Notifies(me.id)).dmap(_.value) zip
env.mod.inquiryApi.forMod(me)
} else
fuccess {
(((0, 0), 0), none)
}
} map {
case (
(pref, hasClas),
teamNbRequests ~ nbChallenges ~ nbNotifications ~ inquiry
) =>
PageData(
teamNbRequests,
nbChallenges,
nbNotifications,
pref,
blindMode = blindMode(ctx),
hasFingerprint = hasFingerPrint,
hasClas = hasClas,
inquiry = inquiry,
nonce = nonce
)
}
env.pref.api.getPref(me, ctx.req) zip {
if (isPage) {
env.user.lightUserApi preloadUser me
env.team.api.nbRequests(me.id) zip
env.challenge.api.countInFor.get(me.id) zip
env.notifyM.api.unreadCount(Notifies(me.id)).dmap(_.value) zip
env.mod.inquiryApi.forMod(me)
} else
fuccess {
(((0, 0), 0), none)
}
} map { case (pref, teamNbRequests ~ nbChallenges ~ nbNotifications ~ inquiry) =>
PageData(
teamNbRequests,
nbChallenges,
nbNotifications,
pref,
blindMode = blindMode(ctx),
hasFingerprint = hasFingerPrint,
hasClas = isGranted(_.Teacher, me) || env.clas.studentCache.isStudent(me.id),
inquiry = inquiry,
nonce = nonce
)
}
}
}

2
build.sbt
View File

@ -371,7 +371,7 @@ lazy val teamSearch = module("teamSearch",
lazy val clas = module("clas",
Seq(common, memo, db, user, security, msg, history, puzzle),
reactivemongo.bundle
reactivemongo.bundle ++ Seq(bloomFilter)
)
lazy val bookmark = module("bookmark",

4
modules/clas/src/main/ClasApi.scala
View File

@ -182,7 +182,7 @@ final class ClasApi(
): Fu[Student.WithPassword] = {
val email = EmailAddress(s"noreply.class.${clas.id}.${data.username}@lichess.org")
val password = Student.password.generate
lila.mon.clas.studentCreate(teacher.id)
lila.mon.clas.student.create(teacher.id).increment()
userRepo
.create(
username = data.username,
@ -260,7 +260,7 @@ ${clas.desc}""",
student
.archive(Student.id(user.id, clas.id), user, v = false)
.map2[ClasInvite.Feedback](_ => Already) getOrElse {
lila.mon.clas.studentInvite(teacher.id)
lila.mon.clas.student.invite(teacher.id).increment()
val invite = ClasInvite.make(clas, user, realName, teacher)
colls.invite.insert
.one(invite)

7
modules/clas/src/main/ClasProgress.scala
View File

@ -44,7 +44,7 @@ final class ClasProgressApi(
gameRepo: GameRepo,
historyApi: lila.history.HistoryApi,
puzzleColls: lila.puzzle.PuzzleColls,
getStudentIds: () => Fu[Set[User.ID]]
studentCache: ClasStudentCache
)(implicit ec: scala.concurrent.ExecutionContext) {
case class PlayStats(nb: Int, wins: Int, millis: Long)
@ -166,8 +166,5 @@ final class ClasProgressApi(
}
private[clas] def onFinishGame(game: lila.game.Game): Unit =
if (game.userIds.nonEmpty)
getStudentIds() foreach { studentIds =>
if (game.userIds.exists(studentIds.contains)) gameRepo.denormalizePerfType(game)
}
if (game.userIds.exists(studentCache.isStudent)) gameRepo.denormalizePerfType(game)
}

52
modules/clas/src/main/ClasStudentCache.scala 100644
View File

@ -0,0 +1,52 @@
package lila.clas
import akka.actor.Scheduler
import akka.stream.Materializer
import akka.stream.scaladsl._
import bloomfilter.mutable.BloomFilter
import reactivemongo.akkastream.{ cursorProducer, AkkaStreamCursor }
import reactivemongo.api.ReadPreference
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext
import lila.db.dsl._
import lila.memo.CacheApi
import lila.user.User
final class ClasStudentCache(colls: ClasColls, cacheApi: CacheApi)(implicit
ec: ExecutionContext,
scheduler: Scheduler,
mat: Materializer
) {
private val expectedElements = 200_000
private val falsePositiveRate = 0.001
private var bloomFilter = BloomFilter[User.ID](expectedElements, falsePositiveRate)
def isStudent(userId: User.ID) = bloomFilter mightContain userId pp userId
private def rebuildBloomFilter(): Unit = {
val nextBloom = BloomFilter[User.ID](expectedElements, falsePositiveRate)
colls.student
.find($empty, $doc("userId" -> true).some)
.cursor[Bdoc](ReadPreference.secondaryPreferred)
.documentSource()
.toMat(Sink.fold[Int, Bdoc](0) { case (total, doc) =>
doc string "userId" foreach nextBloom.add
total + 1
})(Keep.right)
.run()
.addEffect { nb =>
lila.mon.clas.student.bloomFilter.count.update(nb)
bloomFilter.dispose()
bloomFilter = nextBloom
}
.monSuccess(_.clas.student.bloomFilter.fu)
.addEffectAnyway {
scheduler.scheduleOnce(5 minutes) { rebuildBloomFilter() }.unit
}
.unit
}
scheduler.scheduleOnce(20 seconds) { rebuildBloomFilter() }.unit
}

12
modules/clas/src/main/Env.scala
View File

@ -17,17 +17,21 @@ final class Env(
authenticator: lila.user.Authenticator,
cacheApi: lila.memo.CacheApi,
baseUrl: BaseUrl
)(implicit ec: scala.concurrent.ExecutionContext) {
)(implicit
ec: scala.concurrent.ExecutionContext,
scheduler: akka.actor.Scheduler,
mat: akka.stream.Materializer
) {
lazy val nameGenerator = wire[NameGenerator]
lazy val nameGenerator: NameGenerator = wire[NameGenerator]
lazy val forms = wire[ClasForm]
private val colls = wire[ClasColls]
lazy val api: ClasApi = wire[ClasApi]
lazy val studentCache = wire[ClasStudentCache]
private def getStudentIds = () => api.student.allIds
lazy val api: ClasApi = wire[ClasApi]
lazy val progressApi = wire[ClasProgressApi]

4
modules/common/src/main/mon.scala
View File

@ -355,6 +355,10 @@ object mon {
object student {
def create(teacher: String) = counter("clas.student.create").withTag("teacher", teacher)
def invite(teacher: String) = counter("clas.student.invite").withTag("teacher", teacher)
object bloomFilter {
def count = gauge("clas.student.bloomFilter.count").withoutTags()
def fu = future("clas.student.bloomFilter.future")
}
}
}
object tournament {

39
project/Dependencies.scala
View File

@ -5,25 +5,26 @@ object Dependencies {
val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/ornicar/lila-maven/master"
val scalalib = "com.github.ornicar" %% "scalalib" % "7.0.2"
val hasher = "com.roundeights" %% "hasher" % "1.2.1"
val jodaTime = "joda-time" % "joda-time" % "2.10.10"
val chess = "org.lichess" %% "scalachess" % "10.2.0"
val compression = "org.lichess" %% "compression" % "1.6"
val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.3.1-THIB"
val prismic = "io.prismic" %% "scala-kit" % "1.2.19-THIB213"
val scrimage = "com.sksamuel.scrimage" % "scrimage-core" % "4.0.17"
val scaffeine = "com.github.blemale" %% "scaffeine" % "4.0.2" % "compile"
val googleOAuth = "com.google.auth" % "google-auth-library-oauth2-http" % "0.23.0"
val scalaUri = "io.lemonlabs" %% "scala-uri" % "2.3.1"
val scalatags = "com.lihaoyi" %% "scalatags" % "0.9.3"
val lettuce = "io.lettuce" % "lettuce-core" % "5.3.6.RELEASE"
val epoll = "io.netty" % "netty-transport-native-epoll" % "4.1.58.Final" classifier "linux-x86_64"
val autoconfig = "io.methvin.play" %% "autoconfig-macros" % "0.3.2" % "provided"
val scalatest = "org.scalatest" %% "scalatest" % "3.1.0" % Test
val uaparser = "org.uaparser" %% "uap-scala" % "0.11.0"
val specs2 = "org.specs2" %% "specs2-core" % "4.10.6" % Test
val apacheText = "org.apache.commons" % "commons-text" % "1.9"
val scalalib = "com.github.ornicar" %% "scalalib" % "7.0.2"
val hasher = "com.roundeights" %% "hasher" % "1.2.1"
val jodaTime = "joda-time" % "joda-time" % "2.10.10"
val chess = "org.lichess" %% "scalachess" % "10.2.0"
val compression = "org.lichess" %% "compression" % "1.6"
val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.3.1-THIB"
val prismic = "io.prismic" %% "scala-kit" % "1.2.19-THIB213"
val scrimage = "com.sksamuel.scrimage" % "scrimage-core" % "4.0.17"
val scaffeine = "com.github.blemale" %% "scaffeine" % "4.0.2" % "compile"
val googleOAuth = "com.google.auth" % "google-auth-library-oauth2-http" % "0.23.0"
val scalaUri = "io.lemonlabs" %% "scala-uri" % "2.3.1"
val scalatags = "com.lihaoyi" %% "scalatags" % "0.9.3"
val lettuce = "io.lettuce" % "lettuce-core" % "5.3.6.RELEASE"
val epoll = "io.netty" % "netty-transport-native-epoll" % "4.1.58.Final" classifier "linux-x86_64"
val autoconfig = "io.methvin.play" %% "autoconfig-macros" % "0.3.2" % "provided"
val scalatest = "org.scalatest" %% "scalatest" % "3.1.0" % Test
val uaparser = "org.uaparser" %% "uap-scala" % "0.11.0"
val specs2 = "org.specs2" %% "specs2-core" % "4.10.6" % Test
val apacheText = "org.apache.commons" % "commons-text" % "1.9"
val bloomFilter = "com.github.alexandrnikitin" %% "bloom-filter" % "0.13.1"
object flexmark {
val version = "0.50.50"