deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

refactor a lot of evaluation code 

dangerous, but necessary.
pull/8265/head
Thibault Duplessis 2021-02-25 21:14:53 +01:00
parent e541bb9c04
commit 60d4acadbb
10 changed files with 158 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/views/mod/games.scala
View File

@ -131,14 +131,14 @@ object games {
assessment match {
case Some(ass) =>
frag(
td(dataSort := ass.sfAvg)(s"${ass.sfAvg} ± ${ass.sfSd}"),
td(dataSort := ass.mtAvg)(
s"${ass.mtAvg / 10} ± ${ass.mtSd / 10}",
~ass.mtStreak ?? frag(br, "STREAK")
td(dataSort := ass.analysis.avg)(ass.analysis.toString),
td(dataSort := ass.basics.moveTimes.avg)(
s"${ass.basics.moveTimes / 10}",
~ass.basics.mtStreak ?? frag(br, "STREAK")
),
td(dataSort := ass.blurs)(
s"${ass.blurs}%",
ass.blurStreak.filter(8 <=) map { s =>
td(dataSort := ass.basics.blurs)(
s"${ass.basics.blurs}%",
ass.basics.blurStreak.filter(8 <=) map { s =>
frag(br, s"STREAK $s/12")
}
)

12
app/views/user/mod.scala
View File

@ -459,23 +459,23 @@ object mod {
),
td(
span(cls := s"sig sig_${Display.stockfishSig(result)}", dataIcon := "J"),
s" ${result.sfAvg} ± ${result.sfSd}"
s" ${result.analysis}"
),
td(
span(cls := s"sig sig_${Display.moveTimeSig(result)}", dataIcon := "J"),
s" ${result.mtAvg / 10} ± ${result.mtSd / 10}",
(~result.mtStreak) ?? frag(br, "STREAK")
s" ${result.basics.moveTimes / 10}",
(~result.basics.mtStreak) ?? frag(br, "STREAK")
),
td(
span(cls := s"sig sig_${Display.blurSig(result)}", dataIcon := "J"),
s" ${result.blurs}%",
result.blurStreak.filter(8.<=) map { s =>
s" ${result.basics.blurs}%",
result.basics.blurStreak.filter(8.<=) map { s =>
frag(br, s"STREAK $s/12")
}
),
td(
span(cls := s"sig sig_${Display.holdSig(result)}", dataIcon := "J"),
if (result.hold) "Yes" else "No"
if (result.basics.hold) "Yes" else "No"
),
td(
div(cls := "aggregate")(

144
modules/evaluation/src/main/Assessible.scala
View File

@ -1,144 +0,0 @@
package lila.evaluation
import chess.{ Color, Speed }
import lila.analyse.{ Accuracy, Analysis }
import lila.game.{ Game, Player, Pov }
import org.joda.time.DateTime
case class Assessible(pov: Pov, analysis: Analysis, holdAlerts: Player.HoldAlert.Map) {
import Statistics._
import pov.{ color, game }
lazy val suspiciousErrorRate: Boolean =
listAverage(Accuracy.diffsList(Pov(game, color), analysis)) < (game.speed match {
case Speed.Bullet => 25
case Speed.Blitz => 20
case _ => 15
})
lazy val alwaysHasAdvantage: Boolean =
!analysis.infos.exists { info =>
info.cp.fold(info.mate.fold(false) { a =>
a.signum == color.fold(-1, 1)
}) { cp =>
color.fold(cp.centipawns < -100, cp.centipawns > 100)
}
}
lazy val highBlurRate: Boolean =
!game.isSimul && game.playerBlurPercent(color) > 90
lazy val moderateBlurRate: Boolean =
!game.isSimul && game.playerBlurPercent(color) > 70
lazy val suspiciousHoldAlert: Boolean =
holdAlerts(color).exists(_.suspicious)
lazy val highestChunkBlurs: Int =
game.player(color).blurs.booleans.sliding(12).map(_.count(identity)).max
lazy val highChunkBlurRate: Boolean =
highestChunkBlurs >= 11
lazy val moderateChunkBlurRate: Boolean =
highestChunkBlurs >= 8
lazy val highlyConsistentMoveTimes: Boolean =
if (game.clock.forall(_.estimateTotalSeconds > 60))
moveTimeCoefVariation(Pov(game, color)) ?? { cvIndicatesHighlyFlatTimes(_) }
else
false
// moderatelyConsistentMoveTimes must stay in Statistics because it's used in classes that do not use Assessible
lazy val highlyConsistentMoveTimeStreaks: Boolean =
if (game.clock.forall(_.estimateTotalSeconds > 60))
slidingMoveTimesCvs(Pov(game, color)) ?? {
_ exists cvIndicatesHighlyFlatTimesForStreaks
}
else
false
lazy val mkFlags: PlayerFlags = PlayerFlags(
suspiciousErrorRate,
alwaysHasAdvantage,
highBlurRate || highChunkBlurRate,
moderateBlurRate || moderateChunkBlurRate,
highlyConsistentMoveTimes || highlyConsistentMoveTimeStreaks,
moderatelyConsistentMoveTimes(Pov(game, color)),
noFastMoves(Pov(game, color)),
suspiciousHoldAlert
)
private val T = true
private val F = false
private def rankCheating: GameAssessment = {
import GameAssessment._
val flags = mkFlags
val assessment = flags match {
// SF1 SF2 BLR1 BLR2 HCMT MCMT NFM Holds
case PlayerFlags(T, _, T, _, _, _, T, _) => Cheating // high accuracy, high blurs, no fast moves
case PlayerFlags(T, _, _, T, _, _, _, _) => Cheating // high accuracy, moderate blurs
case PlayerFlags(T, _, _, _, T, _, _, _) => Cheating // high accuracy, highly consistent move times
case PlayerFlags(_, _, T, _, T, _, _, _) => Cheating // high blurs, highly consistent move times
case PlayerFlags(_, _, _, T, _, T, _, _) => LikelyCheating // moderate blurs, consistent move times
case PlayerFlags(T, _, _, _, _, _, _, T) => LikelyCheating // Holds are bad, hmk?
case PlayerFlags(_, T, _, _, _, _, _, T) => LikelyCheating // Holds are bad, hmk?
case PlayerFlags(_, _, _, _, T, _, _, _) => LikelyCheating // very consistent move times
case PlayerFlags(_, T, T, _, _, _, _, _) => LikelyCheating // always has advantage, high blurs
case PlayerFlags(_, T, _, _, _, T, T, _) => Unclear // always has advantage, consistent move times
case PlayerFlags(T, _, _, _, _, T, T, _) =>
Unclear // high accuracy, consistent move times, no fast moves
case PlayerFlags(T, _, _, F, _, F, T, _) =>
Unclear // high accuracy, no fast moves, but doesn't blur or flat line
case PlayerFlags(T, _, _, _, _, _, F, _) => UnlikelyCheating // high accuracy, but has fast moves
case PlayerFlags(F, F, _, _, _, _, _, _) => NotCheating // low accuracy, doesn't hold advantage
case _ => NotCheating
}
if (flags.suspiciousHoldAlert) assessment
else if (~game.wonBy(color)) assessment
else if (assessment == Cheating) LikelyCheating
else if (assessment == LikelyCheating) Unclear
else assessment
}
lazy val sfAvg: Int = listAverage(Accuracy.diffsList(Pov(game, color), analysis)).toInt
lazy val sfSd: Int = listDeviation(Accuracy.diffsList(Pov(game, color), analysis)).toInt
lazy val mtAvg: Int = listAverage(~game.moveTimes(color) map (_.roundTenths)).toInt
lazy val mtSd: Int = listDeviation(~game.moveTimes(color) map (_.roundTenths)).toInt
lazy val blurs: Int = game.playerBlurPercent(color)
lazy val tcFactor: Double = game.speed match {
case Speed.Bullet | Speed.Blitz => 1.25
case Speed.Rapid => 1.0
case Speed.Classical => 0.6
case _ => 1.0
}
def playerAssessment: PlayerAssessment =
PlayerAssessment(
_id = game.id + "/" + color.name,
gameId = game.id,
userId = ~game.player(color).userId,
color = color,
assessment = rankCheating,
date = DateTime.now,
// meta
flags = mkFlags,
sfAvg = sfAvg,
sfSd = sfSd,
mtAvg = mtAvg,
mtSd = mtSd,
blurs = blurs,
hold = suspiciousHoldAlert,
blurStreak = highestChunkBlurs.some.filter(0 <),
mtStreak = highlyConsistentMoveTimeStreaks.some.filter(identity),
tcFactor = tcFactor.some
)
}

87
modules/evaluation/src/main/EvaluationBsonHandlers.scala 100644
View File

@ -0,0 +1,87 @@
package lila.evaluation
import reactivemongo.api.bson._
import lila.db.BSON
import lila.db.dsl._
object EvaluationBsonHandlers {
implicit val playerFlagsHandler = new BSON[PlayerFlags] {
def reads(r: BSON.Reader): PlayerFlags =
PlayerFlags(
suspiciousErrorRate = r boolD "ser",
alwaysHasAdvantage = r boolD "aha",
highBlurRate = r boolD "hbr",
moderateBlurRate = r boolD "mbr",
highlyConsistentMoveTimes = r boolD "hcmt",
moderatelyConsistentMoveTimes = r boolD "cmt",
noFastMoves = r boolD "nfm",
suspiciousHoldAlert = r boolD "sha"
)
def writes(w: BSON.Writer, o: PlayerFlags) =
$doc(
"ser" -> w.boolO(o.suspiciousErrorRate),
"aha" -> w.boolO(o.alwaysHasAdvantage),
"hbr" -> w.boolO(o.highBlurRate),
"mbr" -> w.boolO(o.moderateBlurRate),
"hcmt" -> w.boolO(o.highlyConsistentMoveTimes),
"cmt" -> w.boolO(o.moderatelyConsistentMoveTimes),
"nfm" -> w.boolO(o.noFastMoves),
"sha" -> w.boolO(o.suspiciousHoldAlert)
)
}
implicit val GameAssessmentBSONHandler =
BSONIntegerHandler.as[GameAssessment](GameAssessment.orDefault, _.id)
implicit val playerAssessmentHandler = new BSON[PlayerAssessment] {
def reads(r: BSON.Reader): PlayerAssessment = PlayerAssessment(
_id = r str "_id",
gameId = r str "gameId",
userId = r str "userId",
color = chess.Color.fromWhite(r bool "white"),
assessment = r.get[GameAssessment]("assessment"),
date = r date "date",
basics = PlayerAssessment.Basics(
moveTimes = Statistics.IntAvgSd(
avg = r int "mtAvg",
sd = r int "mtSd"
),
hold = r bool "hold",
blurs = r int "blurs",
blurStreak = r intO "blurStreak",
mtStreak = r boolO "mtStreak"
),
analysis = Statistics.IntAvgSd(
avg = r int "sfAvg",
sd = r int "sfSd"
),
flags = r.get[PlayerFlags]("flags"),
tcFactor = r doubleO "tcFactor"
)
def writes(w: BSON.Writer, o: PlayerAssessment) =
$doc(
"_id" -> o._id,
"gameId" -> o.gameId,
"userId" -> o.userId,
"white" -> o.color.white,
"assessment" -> o.assessment,
"date" -> o.date,
"flags" -> o.flags,
"sfAvg" -> o.analysis.avg,
"sfSd" -> o.analysis.sd,
"mtAvg" -> o.basics.moveTimes.avg,
"mtSd" -> o.basics.moveTimes.sd,
"blurs" -> o.basics.blurs,
"hold" -> o.basics.hold,
"blurStreak" -> o.basics.blurStreak,
"mtStreak" -> o.basics.mtStreak,
"tcFactor" -> o.tcFactor
)
}
}

3
modules/evaluation/src/main/GameAssessments.scala
View File

@ -41,7 +41,4 @@ object GameAssessment {
a.id -> a
}.toMap
def orDefault(id: Int) = byId.getOrElse(id, NotCheating)
implicit val GameAssessmentBSONHandler =
reactivemongo.api.bson.BSONIntegerHandler.as[GameAssessment](orDefault, _.id)
}

58
modules/evaluation/src/main/PlayerAggregateAssessment.scala
View File

@ -88,7 +88,7 @@ case class PlayerAggregateAssessment(
val n = filteredAssessments.size
if (n < 2) none
else {
val filteredSfAvg = filteredAssessments.map(_.sfAvg)
val filteredSfAvg = filteredAssessments.map(_.analysis.avg)
val avg = listAverage(filteredSfAvg)
// listDeviation does not apply Bessel's correction, so we do it here by using sqrt(n - 1) instead of sqrt(n)
val width = listDeviation(filteredSfAvg) / sqrt(n - 1) * 1.96
@ -97,16 +97,16 @@ case class PlayerAggregateAssessment(
}
// Average SF Avg and CI given blur rate
val sfAvgBlurs = sfAvgGiven(_.blurs > 70)
val sfAvgNoBlurs = sfAvgGiven(_.blurs <= 70)
val sfAvgBlurs = sfAvgGiven(_.basics.blurs > 70)
val sfAvgNoBlurs = sfAvgGiven(_.basics.blurs <= 70)
// Average SF Avg and CI given move time coef of variance
val sfAvgLowVar = sfAvgGiven(a => a.mtSd.toDouble / a.mtAvg < 0.5)
val sfAvgHighVar = sfAvgGiven(a => a.mtSd.toDouble / a.mtAvg >= 0.5)
val sfAvgLowVar = sfAvgGiven(a => a.basics.moveTimes.sd.toDouble / a.basics.moveTimes.avg < 0.5)
val sfAvgHighVar = sfAvgGiven(a => a.basics.moveTimes.sd.toDouble / a.basics.moveTimes.avg >= 0.5)
// Average SF Avg and CI given bot
val sfAvgHold = sfAvgGiven(_.hold)
val sfAvgNoHold = sfAvgGiven(!_.hold)
val sfAvgHold = sfAvgGiven(_.basics.hold)
val sfAvgNoHold = sfAvgGiven(!_.basics.hold)
def isGreatUser = user.perfs.bestRating > 2500 && user.count.rated >= 100
@ -135,47 +135,3 @@ object PlayerAggregateAssessment {
def pov(pa: PlayerAssessment) = games find (_.id == pa.gameId) map { lila.game.Pov(_, pa.color) }
}
}
case class PlayerFlags(
suspiciousErrorRate: Boolean,
alwaysHasAdvantage: Boolean,
highBlurRate: Boolean,
moderateBlurRate: Boolean,
highlyConsistentMoveTimes: Boolean,
moderatelyConsistentMoveTimes: Boolean,
noFastMoves: Boolean,
suspiciousHoldAlert: Boolean
)
object PlayerFlags {
import reactivemongo.api.bson._
import lila.db.BSON
implicit val playerFlagsBSONHandler = new BSON[PlayerFlags] {
def reads(r: BSON.Reader): PlayerFlags =
PlayerFlags(
suspiciousErrorRate = r boolD "ser",
alwaysHasAdvantage = r boolD "aha",
highBlurRate = r boolD "hbr",
moderateBlurRate = r boolD "mbr",
highlyConsistentMoveTimes = r boolD "hcmt",
moderatelyConsistentMoveTimes = r boolD "cmt",
noFastMoves = r boolD "nfm",
suspiciousHoldAlert = r boolD "sha"
)
def writes(w: BSON.Writer, o: PlayerFlags) =
BSONDocument(
"ser" -> w.boolO(o.suspiciousErrorRate),
"aha" -> w.boolO(o.alwaysHasAdvantage),
"hbr" -> w.boolO(o.highBlurRate),
"mbr" -> w.boolO(o.moderateBlurRate),
"hcmt" -> w.boolO(o.highlyConsistentMoveTimes),
"cmt" -> w.boolO(o.moderatelyConsistentMoveTimes),
"nfm" -> w.boolO(o.noFastMoves),
"sha" -> w.boolO(o.suspiciousHoldAlert)
)
}
}

64
modules/evaluation/src/main/PlayerAssessment.scala
View File

@ -13,35 +13,21 @@ case class PlayerAssessment(
color: Color,
assessment: GameAssessment,
date: DateTime,
// meta
basics: PlayerAssessment.Basics,
analysis: Statistics.IntAvgSd,
flags: PlayerFlags,
sfAvg: Int,
sfSd: Int,
mtAvg: Int,
mtSd: Int,
blurs: Int,
hold: Boolean,
blurStreak: Option[Int],
mtStreak: Option[Boolean],
tcFactor: Option[Double]
)
object PlayerAssessment {
case class Basic(
gameId: Game.ID,
userId: User.ID,
color: Color,
date: DateTime,
// meta
flags: PlayerFlags,
mtAvg: Int,
mtSd: Int,
blurs: Int,
// when you don't have computer analysis
case class Basics(
moveTimes: Statistics.IntAvgSd,
hold: Boolean,
blurs: Int,
blurStreak: Option[Int],
mtStreak: Option[Boolean],
tcFactor: Option[Double]
mtStreak: Option[Boolean]
)
def make(pov: Pov, analysis: Analysis, holdAlerts: Player.HoldAlert.Map): PlayerAssessment = {
@ -49,7 +35,7 @@ object PlayerAssessment {
import pov.{ color, game }
lazy val suspiciousErrorRate: Boolean =
listAverage(Accuracy.diffsList(Pov(game, color), analysis)) < (game.speed match {
listAverage(Accuracy.diffsList(pov, analysis)) < (game.speed match {
case Speed.Bullet => 25
case Speed.Blitz => 20
case _ => 15
@ -96,7 +82,7 @@ object PlayerAssessment {
}
}
lazy val mkFlags: PlayerFlags = PlayerFlags(
lazy val flags: PlayerFlags = PlayerFlags(
suspiciousErrorRate,
alwaysHasAdvantage,
highBlurRate || highChunkBlurRate,
@ -110,9 +96,8 @@ object PlayerAssessment {
val T = true
val F = false
def rankCheating: GameAssessment = {
def assessment: GameAssessment = {
import GameAssessment._
val flags = mkFlags
val assessment = flags match {
// SF1 SF2 BLR1 BLR2 HCMT MCMT NFM Holds
case PlayerFlags(T, _, T, _, _, _, T, _) => Cheating // high accuracy, high blurs, no fast moves
@ -145,13 +130,7 @@ object PlayerAssessment {
else assessment
}
lazy val sfAvg: Int = listAverage(Accuracy.diffsList(Pov(game, color), analysis)).toInt
lazy val sfSd: Int = listDeviation(Accuracy.diffsList(Pov(game, color), analysis)).toInt
lazy val mtAvg: Int = listAverage(~game.moveTimes(color) map (_.roundTenths)).toInt
lazy val mtSd: Int = listDeviation(~game.moveTimes(color) map (_.roundTenths)).toInt
lazy val blurs: Int = game.playerBlurPercent(color)
lazy val tcFactor: Double = game.speed match {
val tcFactor: Double = game.speed match {
case Speed.Bullet | Speed.Blitz => 1.25
case Speed.Rapid => 1.0
case Speed.Classical => 0.6
@ -163,18 +142,17 @@ object PlayerAssessment {
gameId = game.id,
userId = ~game.player(color).userId,
color = color,
assessment = rankCheating,
assessment = assessment,
date = DateTime.now,
// meta
flags = mkFlags,
sfAvg = sfAvg,
sfSd = sfSd,
mtAvg = mtAvg,
mtSd = mtSd,
blurs = blurs,
hold = suspiciousHoldAlert,
blurStreak = highestChunkBlurs.some.filter(0 <),
mtStreak = highlyConsistentMoveTimeStreaks.some.filter(identity),
Basics(
moveTimes = Statistics.intAvgSd(~game.moveTimes(color) map (_.roundTenths)),
blurs = game playerBlurPercent color,
hold = suspiciousHoldAlert,
blurStreak = highestChunkBlurs.some.filter(0 <),
mtStreak = highlyConsistentMoveTimeStreaks.some.filter(identity)
),
analysis = Statistics.intAvgSd(Accuracy.diffsList(pov, analysis)),
flags = flags,
tcFactor = tcFactor.some
)
}

15
modules/evaluation/src/main/PlayerFlags.scala 100644
View File

@ -0,0 +1,15 @@
package lila.evaluation
import lila.db.dsl._
import lila.db.BSONReadOnly
case class PlayerFlags(
suspiciousErrorRate: Boolean,
alwaysHasAdvantage: Boolean,
highBlurRate: Boolean,
moderateBlurRate: Boolean,
highlyConsistentMoveTimes: Boolean,
moderatelyConsistentMoveTimes: Boolean,
noFastMoves: Boolean,
suspiciousHoldAlert: Boolean
)

10
modules/evaluation/src/main/Statistics.scala
View File

@ -5,6 +5,16 @@ import lila.common.Maths
object Statistics {
case class IntAvgSd(avg: Int, sd: Int) {
override def toString = s"$avg ± $sd"
def /(div: Int) = IntAvgSd(avg / div, sd / div)
}
def intAvgSd(values: List[Int]) = IntAvgSd(
avg = listAverage(values).toInt,
sd = listDeviation(values).toInt
)
// Coefficient of Variance
def coefVariation(a: List[Int]): Option[Float] = {
val s = Stats(a)

12
modules/mod/src/main/AssessApi.scala
View File

@ -4,7 +4,7 @@ import lila.analyse.{ Analysis, AnalysisRepo }
import lila.db.BSON.BSONJodaDateTimeHandler
import lila.db.dsl._
import lila.evaluation.Statistics
import lila.evaluation.{ AccountAction, Assessible, PlayerAggregateAssessment, PlayerAssessment, PlayerFlags }
import lila.evaluation.{ AccountAction, PlayerAggregateAssessment, PlayerAssessment }
import lila.game.{ Game, Player, Pov, Source }
import lila.report.{ ModId, SuspectId }
import lila.user.User
@ -28,9 +28,7 @@ final class AssessApi(
private def bottomDate = DateTime.now.minusSeconds(3600 * 24 * 30 * 6) // matches a mongo expire index
import PlayerFlags.playerFlagsBSONHandler
implicit private val playerAssessmentBSONhandler = Macros.handler[PlayerAssessment]
import lila.evaluation.EvaluationBsonHandlers._
private def createPlayerAssessment(assessed: PlayerAssessment) =
assessRepo.coll.update.one($id(assessed._id), assessed, upsert = true).void
@ -108,8 +106,8 @@ final class AssessApi(
else if (game.createdAt isBefore bottomDate) false
else true
shouldAssess.?? {
createPlayerAssessment(Assessible(game pov chess.White, analysis, holdAlerts).playerAssessment) >>
createPlayerAssessment(Assessible(game pov chess.Black, analysis, holdAlerts).playerAssessment)
createPlayerAssessment(PlayerAssessment.make(game pov chess.White, analysis, holdAlerts)) >>
createPlayerAssessment(PlayerAssessment.make(game pov chess.Black, analysis, holdAlerts))
} >> ((shouldAssess && thenAssessUser) ?? {
game.whitePlayer.userId.??(assessUser) >> game.blackPlayer.userId.??(assessUser)
})
@ -121,7 +119,7 @@ final class AssessApi(
game.playerByUserId(userId) ?? { player =>
gameRepo holdAlerts game flatMap { holdAlerts =>
createPlayerAssessment(
Assessible(game pov player.color, analysis, holdAlerts).playerAssessment
PlayerAssessment.make(game pov player.color, analysis, holdAlerts)
)
}
}