diff --git a/app/controllers/Analyse.scala b/app/controllers/Analyse.scala index 09c46ab7a1..f56b224a53 100644 --- a/app/controllers/Analyse.scala +++ b/app/controllers/Analyse.scala @@ -58,9 +58,7 @@ final class Analyse( system = false ) ) - }.sequenceFu >> - env.fishnet.awaiter(games.map(_.id), 2 minutes) >> - env.mod.assessApi.ensurePlayerAssess(user.id, games) + }.sequenceFu >> env.fishnet.awaiter(games.map(_.id), 2 minutes) } inject NoContent ) } diff --git a/app/controllers/GameMod.scala b/app/controllers/GameMod.scala index ed37a3f42f..934c5bc814 100644 --- a/app/controllers/GameMod.scala +++ b/app/controllers/GameMod.scala @@ -23,7 +23,7 @@ final class GameMod(env: Env) extends LilaController(env) { env.activity.read.recentSwissRanks(user.id) zip env.game.gameRepo.recentPovsByUserFromSecondary(user, 100, toDbSelect(filter)) flatMap { case ((arenas, swisses), povs) => - env.mod.assessApi.ofPovs(povs) map { games => + env.mod.assessApi.makeAndGetFullOrBasicsFor(povs) map { games => Ok(views.html.mod.games(user, form, games, arenas.currentPageResults, swisses)) } } diff --git a/app/views/mod/games.scala b/app/views/mod/games.scala index 926e272ad1..643a8ca217 100644 --- a/app/views/mod/games.scala +++ b/app/views/mod/games.scala @@ -1,6 +1,7 @@ package views.html.mod import controllers.GameMod +import scala.util.chaining._ import controllers.routes import play.api.data.Form @@ -23,7 +24,7 @@ object games { def apply( user: User, filterForm: Form[GameMod.Filter], - games: List[(Pov, Option[PlayerAssessment])], + games: List[(Pov, Either[PlayerAssessment, PlayerAssessment.Basics])], arenas: Seq[TourEntry], swisses: Seq[(Swiss.IdName, Int)] )(implicit @@ -129,21 +130,22 @@ object games { } ), assessment match { - case Some(ass) => - frag( - td(dataSort := ass.analysis.avg)(ass.analysis.toString), - td(dataSort := ass.basics.moveTimes.avg)( - s"${ass.basics.moveTimes / 10}", - ~ass.basics.mtStreak ?? frag(br, "STREAK") - ), - td(dataSort := ass.basics.blurs)( - s"${ass.basics.blurs}%", - ass.basics.blurStreak.filter(8 <=) map { s => - frag(br, s"STREAK $s/12") - } - ) + case Left(full) => td(dataSort := full.analysis.avg)(full.analysis.toString) + case _ => td + }, + assessment.fold(_.basics, identity) pipe { basics => + frag( + td(dataSort := basics.moveTimes.avg)( + s"${basics.moveTimes / 10}", + basics.mtStreak ?? frag(br, "streak") + ), + td(dataSort := basics.blurs)( + s"${basics.blurs}%", + basics.blurStreak.filter(8 <=) map { s => + frag(br, s"streak $s/12") + } ) - case _ => frag(td, td, td) // don't use colspan, it breaks the tablesorter + ) }, td(dataSort := pov.game.movedAt.getSeconds.toString)( a(href := routes.Round.watcher(pov.gameId, pov.color.name))( diff --git a/app/views/user/mod.scala b/app/views/user/mod.scala index 6d1834b341..25479cff7b 100644 --- a/app/views/user/mod.scala +++ b/app/views/user/mod.scala @@ -464,13 +464,13 @@ object mod { td( span(cls := s"sig sig_${Display.moveTimeSig(result)}", dataIcon := "J"), s" ${result.basics.moveTimes / 10}", - (~result.basics.mtStreak) ?? frag(br, "STREAK") + result.basics.mtStreak ?? frag(br, "streak") ), td( span(cls := s"sig sig_${Display.blurSig(result)}", dataIcon := "J"), s" ${result.basics.blurs}%", result.basics.blurStreak.filter(8.<=) map { s => - frag(br, s"STREAK $s/12") + frag(br, s"streak $s/12") } ), td( diff --git a/modules/analyse/src/main/Analysis.scala b/modules/analyse/src/main/Analysis.scala index 20f3a0c115..a9c792ec29 100644 --- a/modules/analyse/src/main/Analysis.scala +++ b/modules/analyse/src/main/Analysis.scala @@ -55,7 +55,7 @@ object Analysis { type ID = String - implicit private[analyse] val analysisBSONHandler = new BSON[Analysis] { + implicit val analysisBSONHandler = new BSON[Analysis] { def reads(r: BSON.Reader) = { val startPly = r intD "ply" val raw = r str "data" diff --git a/modules/analyse/src/main/AnalysisRepo.scala b/modules/analyse/src/main/AnalysisRepo.scala index f60905251a..7ee1f4b077 100644 --- a/modules/analyse/src/main/AnalysisRepo.scala +++ b/modules/analyse/src/main/AnalysisRepo.scala @@ -3,7 +3,7 @@ package lila.analyse import lila.db.dsl._ import lila.game.Game -final class AnalysisRepo(coll: Coll)(implicit ec: scala.concurrent.ExecutionContext) { +final class AnalysisRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionContext) { import Analysis.analysisBSONHandler diff --git a/modules/db/src/main/CollExt.scala b/modules/db/src/main/CollExt.scala index 131d57b66b..084546c605 100644 --- a/modules/db/src/main/CollExt.scala +++ b/modules/db/src/main/CollExt.scala @@ -73,31 +73,13 @@ trait CollExt { self: dsl with QueryBuilderExt => def exists(selector: Bdoc): Fu[Boolean] = countSel(selector).dmap(0 !=) - def byOrderedIds[D: BSONDocumentReader, I: BSONWriter]( + def idsMap[D: BSONDocumentReader, I: BSONWriter]( ids: Iterable[I], projection: Option[Bdoc] = None, readPreference: ReadPreference = ReadPreference.primary - )(docId: D => I): Fu[List[D]] = - mapByOrderedIds[D, I](ids, projection, readPreference)(docId) map { m => - ids.view.flatMap(m.get).toList - } - - def optionsByOrderedIds[D: BSONDocumentReader, I: BSONWriter]( - ids: Iterable[I], - projection: Option[Bdoc] = None, - readPreference: ReadPreference = ReadPreference.primary - )(docId: D => I): Fu[List[Option[D]]] = - mapByOrderedIds[D, I](ids, projection, readPreference)(docId) map { m => - ids.view.map(m.get).toList - } - - private def mapByOrderedIds[D: BSONDocumentReader, I: BSONWriter]( - ids: Iterable[I], - projection: Option[Bdoc], - readPreference: ReadPreference )(docId: D => I): Fu[Map[I, D]] = projection - .fold(coll.find($inIds(ids))) { proj => + .fold(coll find $inIds(ids)) { proj => coll.find($inIds(ids), proj.some) } .cursor[D](readPreference) @@ -106,12 +88,22 @@ trait CollExt { self: dsl with QueryBuilderExt => _.view.map(u => docId(u) -> u).toMap } - def idsMap[D: BSONDocumentReader, I: BSONWriter]( + def byOrderedIds[D: BSONDocumentReader, I: BSONWriter]( ids: Iterable[I], + projection: Option[Bdoc] = None, readPreference: ReadPreference = ReadPreference.primary - )(docId: D => I): Fu[Map[I, D]] = - byIds[D, I](ids, readPreference) map { docs => - docs.view.map(u => docId(u) -> u).toMap + )(docId: D => I): Fu[List[D]] = + idsMap[D, I](ids, projection, readPreference)(docId) map { m => + ids.view.flatMap(m.get).toList + } + + def optionsByOrderedIds[D: BSONDocumentReader, I: BSONWriter]( + ids: Iterable[I], + projection: Option[Bdoc] = None, + readPreference: ReadPreference = ReadPreference.primary + )(docId: D => I): Fu[List[Option[D]]] = + idsMap[D, I](ids, projection, readPreference)(docId) map { m => + ids.view.map(m.get).toList } def primitive[V: BSONReader](selector: Bdoc, field: String): Fu[List[V]] = diff --git a/modules/evaluation/src/main/EvaluationBsonHandlers.scala b/modules/evaluation/src/main/EvaluationBsonHandlers.scala index 64bb2737fb..a1b565b65f 100644 --- a/modules/evaluation/src/main/EvaluationBsonHandlers.scala +++ b/modules/evaluation/src/main/EvaluationBsonHandlers.scala @@ -54,7 +54,7 @@ object EvaluationBsonHandlers { hold = r bool "hold", blurs = r int "blurs", blurStreak = r intO "blurStreak", - mtStreak = r boolO "mtStreak" + mtStreak = r boolD "mtStreak" ), analysis = Statistics.IntAvgSd( avg = r int "sfAvg", @@ -80,7 +80,7 @@ object EvaluationBsonHandlers { "blurs" -> o.basics.blurs, "hold" -> o.basics.hold, "blurStreak" -> o.basics.blurStreak, - "mtStreak" -> o.basics.mtStreak, + "mtStreak" -> w.boolO(o.basics.mtStreak), "tcFactor" -> o.tcFactor ) } diff --git a/modules/evaluation/src/main/PlayerAssessment.scala b/modules/evaluation/src/main/PlayerAssessment.scala index 4b816f9548..d3189e4c5b 100644 --- a/modules/evaluation/src/main/PlayerAssessment.scala +++ b/modules/evaluation/src/main/PlayerAssessment.scala @@ -1,9 +1,10 @@ package lila.evaluation import chess.{ Color, Speed } +import org.joda.time.DateTime + import lila.analyse.{ Accuracy, Analysis } import lila.game.{ Game, Player, Pov } -import org.joda.time.DateTime import lila.user.User case class PlayerAssessment( @@ -27,13 +28,55 @@ object PlayerAssessment { hold: Boolean, blurs: Int, blurStreak: Option[Int], - mtStreak: Option[Boolean] + mtStreak: Boolean ) - def make(pov: Pov, analysis: Analysis, holdAlerts: Player.HoldAlert.Map): PlayerAssessment = { + private def highestChunkBlursOf(pov: Pov) = + pov.player.blurs.booleans.sliding(12).map(_.count(identity)).max + + private def highlyConsistentMoveTimeStreaksOf(pov: Pov): Boolean = + pov.game.clock.exists(_.estimateTotalSeconds > 60) && { + Statistics.slidingMoveTimesCvs(pov) ?? { + _ exists Statistics.cvIndicatesHighlyFlatTimesForStreaks + } + } + + def makeBasics(pov: Pov, holdAlerts: Option[Player.HoldAlert]): PlayerAssessment.Basics = { import Statistics._ import pov.{ color, game } + Basics( + moveTimes = intAvgSd(~game.moveTimes(color) map (_.roundTenths)), + blurs = game playerBlurPercent color, + hold = holdAlerts.exists(_.suspicious), + blurStreak = highestChunkBlursOf(pov).some.filter(0 <), + mtStreak = highlyConsistentMoveTimeStreaksOf(pov) + ) + } + + def make(pov: Pov, analysis: Analysis, holdAlerts: Option[Player.HoldAlert]): PlayerAssessment = { + import Statistics._ + import pov.{ color, game } + + val basics = makeBasics(pov, holdAlerts) + + lazy val highBlurRate: Boolean = + !game.isSimul && game.playerBlurPercent(color) > 90 + + lazy val moderateBlurRate: Boolean = + !game.isSimul && game.playerBlurPercent(color) > 70 + + val highestChunkBlurs = highestChunkBlursOf(pov) + + val highChunkBlurRate: Boolean = highestChunkBlurs >= 11 + + val moderateChunkBlurRate: Boolean = highestChunkBlurs >= 8 + + lazy val highlyConsistentMoveTimes: Boolean = + game.clock.exists(_.estimateTotalSeconds > 60) && { + moveTimeCoefVariation(pov) ?? cvIndicatesHighlyFlatTimes + } + lazy val suspiciousErrorRate: Boolean = listAverage(Accuracy.diffsList(pov, analysis)) < (game.speed match { case Speed.Bullet => 25 @@ -50,47 +93,15 @@ object PlayerAssessment { } } - lazy val highBlurRate: Boolean = - !game.isSimul && game.playerBlurPercent(color) > 90 - - lazy val moderateBlurRate: Boolean = - !game.isSimul && game.playerBlurPercent(color) > 70 - - lazy val suspiciousHoldAlert: Boolean = - holdAlerts(color).exists(_.suspicious) - - lazy val highestChunkBlurs: Int = - game.player(color).blurs.booleans.sliding(12).map(_.count(identity)).max - - lazy val highChunkBlurRate: Boolean = - highestChunkBlurs >= 11 - - lazy val moderateChunkBlurRate: Boolean = - highestChunkBlurs >= 8 - - lazy val highlyConsistentMoveTimes: Boolean = - game.clock.exists(_.estimateTotalSeconds > 60) && { - moveTimeCoefVariation(pov) ?? cvIndicatesHighlyFlatTimes - } - - // moderatelyConsistentMoveTimes must stay in Statistics because it's used in classes that do not use Assessible - - lazy val highlyConsistentMoveTimeStreaks: Boolean = - game.clock.exists(_.estimateTotalSeconds > 60) && { - slidingMoveTimesCvs(pov) ?? { - _ exists cvIndicatesHighlyFlatTimesForStreaks - } - } - lazy val flags: PlayerFlags = PlayerFlags( suspiciousErrorRate, alwaysHasAdvantage, highBlurRate || highChunkBlurRate, moderateBlurRate || moderateChunkBlurRate, - highlyConsistentMoveTimes || highlyConsistentMoveTimeStreaks, + highlyConsistentMoveTimes || highlyConsistentMoveTimeStreaksOf(pov), moderatelyConsistentMoveTimes(pov), noFastMoves(pov), - suspiciousHoldAlert + basics.hold ) val T = true @@ -136,7 +147,6 @@ object PlayerAssessment { case Speed.Classical => 0.6 case _ => 1.0 } - PlayerAssessment( _id = s"${game.id}/${color.name}", gameId = game.id, @@ -144,14 +154,8 @@ object PlayerAssessment { color = color, assessment = assessment, date = DateTime.now, - Basics( - moveTimes = Statistics.intAvgSd(~game.moveTimes(color) map (_.roundTenths)), - blurs = game playerBlurPercent color, - hold = suspiciousHoldAlert, - blurStreak = highestChunkBlurs.some.filter(0 <), - mtStreak = highlyConsistentMoveTimeStreaks.some.filter(identity) - ), - analysis = Statistics.intAvgSd(Accuracy.diffsList(pov, analysis)), + basics = basics, + analysis = intAvgSd(Accuracy.diffsList(pov, analysis)), flags = flags, tcFactor = tcFactor.some ) diff --git a/modules/game/src/main/GameRepo.scala b/modules/game/src/main/GameRepo.scala index c4282b7b36..d189f82a28 100644 --- a/modules/game/src/main/GameRepo.scala +++ b/modules/game/src/main/GameRepo.scala @@ -270,30 +270,48 @@ final class GameRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont def setBorderAlert(pov: Pov) = setHoldAlert(pov, Player.HoldAlert(0, 0, 20)) - def holdAlerts(game: Game): Fu[Player.HoldAlert.Map] = - coll.one[Bdoc]( - $doc( - F.id -> game.id, - $or( - holdAlertField(chess.White) $exists true, - holdAlertField(chess.Black) $exists true - ) - ), - $doc( - F.id -> false, - holdAlertField(chess.White) -> true, - holdAlertField(chess.Black) -> true - ) - ) map { - _.fold(Player.HoldAlert.emptyMap) { doc => - def holdAlertOf(playerField: String) = - doc.child(playerField).flatMap(_.getAsOpt[Player.HoldAlert](Player.BSONFields.holdAlert)) - Color.Map( - white = holdAlertOf("p0"), - black = holdAlertOf("p1") - ) + object holdAlert { + private val holdAlertSelector = $or( + holdAlertField(chess.White) $exists true, + holdAlertField(chess.Black) $exists true + ) + private val holdAlertProjection = $doc( + holdAlertField(chess.White) -> true, + holdAlertField(chess.Black) -> true + ) + private def holdAlertOf(doc: Bdoc, color: Color): Option[Player.HoldAlert] = + doc.child(color.fold("p0", "p1")).flatMap(_.getAsOpt[Player.HoldAlert](Player.BSONFields.holdAlert)) + + def game(game: Game): Fu[Player.HoldAlert.Map] = + coll.one[Bdoc]( + $doc(F.id -> game.id, holdAlertSelector), + holdAlertProjection + ) map { + _.fold(Player.HoldAlert.emptyMap) { doc => + Color.Map(white = holdAlertOf(doc, chess.White), black = holdAlertOf(doc, chess.Black)) + } } - } + + def povs(povs: Seq[Pov]): Fu[Map[Game.ID, Player.HoldAlert]] = + coll + .find( + $doc($inIds(povs.map(_.gameId)), holdAlertSelector), + holdAlertProjection.some + ) + .cursor[Bdoc](ReadPreference.secondaryPreferred) + .list() map { docs => + val idColors = povs.view.map { p => + p.gameId -> p.color + }.toMap + val holds = for { + doc <- docs + id <- doc string "_id" + color <- idColors get id + holds <- holdAlertOf(doc, color) + } yield id -> holds + holds.toMap + } + } def hasHoldAlert(pov: Pov): Fu[Boolean] = coll.exists( diff --git a/modules/mod/src/main/AssessApi.scala b/modules/mod/src/main/AssessApi.scala index f0ffbd687c..07a3d1161b 100644 --- a/modules/mod/src/main/AssessApi.scala +++ b/modules/mod/src/main/AssessApi.scala @@ -1,6 +1,12 @@ package lila.mod +import chess.{ Black, Color, White } +import org.joda.time.DateTime +import reactivemongo.api.bson._ +import reactivemongo.api.ReadPreference + import lila.analyse.{ Analysis, AnalysisRepo } +import lila.common.ThreadLocalRandom import lila.db.BSON.BSONJodaDateTimeHandler import lila.db.dsl._ import lila.evaluation.Statistics @@ -9,13 +15,6 @@ import lila.game.{ Game, Player, Pov, Source } import lila.report.{ ModId, SuspectId } import lila.user.User -import org.joda.time.DateTime -import reactivemongo.api.ReadPreference -import reactivemongo.api.bson._ -import lila.common.ThreadLocalRandom - -import chess.Color - final class AssessApi( assessRepo: AssessmentRepo, modApi: ModApi, @@ -29,6 +28,7 @@ final class AssessApi( private def bottomDate = DateTime.now.minusSeconds(3600 * 24 * 30 * 6) // matches a mongo expire index import lila.evaluation.EvaluationBsonHandlers._ + import Analysis.analysisBSONHandler private def createPlayerAssessment(assessed: PlayerAssessment) = assessRepo.coll.update.one($id(assessed._id), assessed, upsert = true).void @@ -60,17 +60,53 @@ final class AssessApi( PlayerAggregateAssessment.WithGames(pag, _) } - def ofPovs(povs: List[Pov]): Fu[List[(Pov, Option[PlayerAssessment])]] = + private def buildMissing(povs: List[Pov]): Funit = assessRepo.coll - .idsMap[PlayerAssessment, String]( - povs.map(p => s"${p.gameId}/${p.color.name}"), - ReadPreference.secondaryPreferred - )(_.gameId) - .map { ass => - povs.map { pov => - (pov, ass get pov.gameId) + .distinctEasy[Game.ID, Set]("gameId", $inIds(povs.map(p => s"${p.gameId}/${p.color.name}"))) flatMap { + existingIds => + val missing = povs collect { + case pov if pov.game.metadata.analysed && !existingIds.contains(pov.gameId) => pov.gameId + } + missing.nonEmpty ?? + analysisRepo.coll + .idsMap[Analysis, Game.ID](missing)(_.id) + .flatMap { ans => + povs + .flatMap { pov => + ans get pov.gameId map { pov -> _ } + } + .map { case (pov, analysis) => + gameRepo.holdAlert game pov.game flatMap { holdAlerts => + createPlayerAssessment(PlayerAssessment.make(pov, analysis, holdAlerts(pov.color))) + } + } + .sequenceFu + .void + } + } + + def makeAndGetFullOrBasicsFor( + povs: List[Pov] + ): Fu[List[(Pov, Either[PlayerAssessment, PlayerAssessment.Basics])]] = + buildMissing(povs) >> + assessRepo.coll + .idsMap[PlayerAssessment, Game.ID]( + ids = povs.map(p => s"${p.gameId}/${p.color.name}"), + readPreference = ReadPreference.secondaryPreferred + )(_.gameId) + .flatMap { fulls => + val basicsPovs = povs.filterNot(p => fulls.exists(_._1 == p.gameId)) + gameRepo.holdAlert.povs(basicsPovs) map { holds => + povs map { pov => + pov -> { + fulls.get(pov.gameId) match { + case Some(full) => Left(full) + case None => Right(PlayerAssessment.makeBasics(pov, holds get pov.gameId)) + } + } + } + } } - } def getPlayerAggregateAssessmentWithGames( userId: User.ID, @@ -93,7 +129,7 @@ final class AssessApi( }) >> assessUser(user.id) def onAnalysisReady(game: Game, analysis: Analysis, thenAssessUser: Boolean = true): Funit = - gameRepo holdAlerts game flatMap { holdAlerts => + gameRepo.holdAlert game game flatMap { holdAlerts => def consistentMoveTimes(game: Game)(player: Player) = Statistics.moderatelyConsistentMoveTimes(Pov(game, player)) val shouldAssess = @@ -106,24 +142,13 @@ final class AssessApi( else if (game.createdAt isBefore bottomDate) false else true shouldAssess.?? { - createPlayerAssessment(PlayerAssessment.make(game pov chess.White, analysis, holdAlerts)) >> - createPlayerAssessment(PlayerAssessment.make(game pov chess.Black, analysis, holdAlerts)) - } >> ((shouldAssess && thenAssessUser) ?? { - game.whitePlayer.userId.??(assessUser) >> game.blackPlayer.userId.??(assessUser) - }) - } - - def ensurePlayerAssess(userId: User.ID, games: List[Game]): Funit = - analysisRepo.associateToGames(games take 100) flatMap { - _.map { case (game, analysis) => - game.playerByUserId(userId) ?? { player => - gameRepo holdAlerts game flatMap { holdAlerts => - createPlayerAssessment( - PlayerAssessment.make(game pov player.color, analysis, holdAlerts) - ) - } + createPlayerAssessment(PlayerAssessment.make(game pov White, analysis, holdAlerts.white)) >> + createPlayerAssessment(PlayerAssessment.make(game pov Black, analysis, holdAlerts.black)) + } >> { + (shouldAssess && thenAssessUser) ?? { + game.whitePlayer.userId.??(assessUser) >> game.blackPlayer.userId.??(assessUser) } - }.sequenceFu.void + } } def assessUser(userId: User.ID): Funit = @@ -200,7 +225,7 @@ final class AssessApi( else if (game.createdAt isBefore bottomDate) fuccess(none) // someone is using a bot else - gameRepo holdAlerts game map { holdAlerts => + gameRepo.holdAlert game game map { holdAlerts => if (Player.HoldAlert suspicious holdAlerts) HoldAlert.some // white has consistent move times else if (whiteSuspCoefVariation.isDefined && randomPercent(70))