From 94332d246f055d007b9d21459407440144d3e1fb Mon Sep 17 00:00:00 2001 From: Isaac Levy Date: Wed, 14 Feb 2018 17:54:44 -0500 Subject: [PATCH] Collect more lag statistics Fancy mathematical stats monitoring Stats.scala has been moved to scalachess --- modules/common/src/main/Stats.scala | 77 -------------------- modules/common/src/main/mon.scala | 8 +- modules/common/src/test/StatsTest.scala | 68 ----------------- modules/evaluation/src/main/Statistics.scala | 4 +- modules/round/src/main/Finisher.scala | 31 +++++++- modules/round/src/main/Round.scala | 30 ++------ project/Dependencies.scala | 2 +- 7 files changed, 44 insertions(+), 176 deletions(-) delete mode 100644 modules/common/src/main/Stats.scala delete mode 100644 modules/common/src/test/StatsTest.scala diff --git a/modules/common/src/main/Stats.scala b/modules/common/src/main/Stats.scala deleted file mode 100644 index 8388870701..0000000000 --- a/modules/common/src/main/Stats.scala +++ /dev/null @@ -1,77 +0,0 @@ -package lila.common - -// Welford's numerically stable online variance. -// -sealed trait Stats { - def samples: Int - def mean: Float - def variance: Option[Float] - def record(value: Float): Stats - def +(o: Stats): Stats - - def record[T](values: Traversable[T])(implicit n: Numeric[T]): Stats = - values.foldLeft(this) { (s, v) => s record n.toFloat(v) } - - def stdDev = variance.map { Math.sqrt(_).toFloat } -} - -protected final case class StatHolder( - samples: Int, - mean: Float, - sn: Float -) extends Stats { - def variance = (samples > 1) option (sn / (samples - 1)) - - def record(value: Float) = { - val newSamples = samples + 1 - val delta = value - mean - val newMean = mean + delta / newSamples - val newSN = sn + delta * (value - newMean) - - StatHolder( - samples = newSamples, - mean = newMean, - sn = newSN - ) - } - - def +(o: Stats) = o match { - case EmptyStats => this - case StatHolder(oSamples, oMean, oSN) => { - val invTotal = 1f / (samples + oSamples) - val combMean = { - if (samples == oSamples) (mean + oMean) * 0.5f - else (mean * samples + oMean * oSamples) * invTotal - } - - val meanDiff = mean - oMean - - StatHolder( - samples = samples + oSamples, - mean = combMean, - sn = sn + oSN + meanDiff * meanDiff * samples * oSamples * invTotal - ) - } - } -} - -protected object EmptyStats extends Stats { - val samples = 0 - val mean = 0f - val variance = None - - def record(value: Float) = StatHolder( - samples = 1, - mean = value, - sn = 0f - ) - - def +(o: Stats) = o -} - -object Stats { - val empty = EmptyStats - - def apply(value: Float) = empty.record(value) - def apply[T: Numeric](values: Traversable[T]) = empty.record(values) -} \ No newline at end of file diff --git a/modules/common/src/main/mon.scala b/modules/common/src/main/mon.scala index 0a16d772f9..cc62fec032 100644 --- a/modules/common/src/main/mon.scala +++ b/modules/common/src/main/mon.scala @@ -163,12 +163,14 @@ object mon { def create = makeTrace("round.move.trace") } object lag { - val avgReported = rec("round.move.lag.avg_reported") - private val estErrorRec = rec("round.move.lag.estimate_error_1000") - def estimateError(e: Int) = estErrorRec(e + 1000) val compDeviation = rec("round.move.lag.comp_deviation") def uncomped(key: String) = rec(s"round.move.lag.uncomped.$key") val uncompedAll = rec(s"round.move.lag.uncomped.all") + val stdDev = rec(s"round.move.lag.stddev_ms") + val mean = rec(s"round.move.lag.mean_ms") + val coefVar = rec(s"round.move.lag.coef_var_1000") + val compEstStdErr = rec(s"round.move.lag.comp_est_stderr_1000") + val compEstOverErr = rec("round.move.lag.avg_over_error_ms") } } object error { diff --git a/modules/common/src/test/StatsTest.scala b/modules/common/src/test/StatsTest.scala deleted file mode 100644 index 87c40614c5..0000000000 --- a/modules/common/src/test/StatsTest.scala +++ /dev/null @@ -1,68 +0,0 @@ -package lila.common - -import org.specs2.mutable.Specification -import org.specs2.matcher.ValidationMatchers - -class StatsTest extends Specification with ValidationMatchers { - - def realMean(elts: Seq[Float]): Float = elts.sum / elts.size - - def realVar(elts: Seq[Float]): Float = { - val mean = realMean(elts).toDouble - (elts map { x => Math.pow(x - mean, 2) } sum).toFloat / (elts.size - 1) - } - - def beApprox(comp: Float) = (f: Float) => { - if (comp.isNaN) f.isNaN must beTrue - else comp must beCloseTo(f +/- 0.001f * comp) - } - - def beLike(comp: Stats) = (s: Stats) => { - s.samples must_== comp.samples - s.mean must beApprox(comp.mean) - (s.variance, comp.variance) match { - case (Some(sv), Some(cv)) => sv must beApprox(cv) - case (sv, cv) => sv must_== cv - } - } - - "empty stats" should { - "have good defaults" in { - Stats.empty.variance must_== None - Stats.empty.mean must_== 0f - Stats.empty.samples must_== 0 - } - - "convert to StatHolder" in { - Stats(5) must beLike(StatHolder(0, 0f, 0f).record(5)) - - "with good stats" in { - Stats(5).samples must_== 1 - Stats(5).variance must_== None - Stats(5).mean must_== 5f - } - } - - } - - "large values" should { - // Tight data w/ large mean. Shuffled for Stats. - val base = (1 to 100) ++ (1 to 100) ++ (1 to 200) - val data = base map { _ + 1e5f } - val shuffledData = base.sortWith(_ % 8 > _ % 8) map { _ + 1e5f } - - val statsN = Stats.empty record shuffledData - "match actuals" in { - statsN.mean must beApprox(realMean(data)) - statsN.variance.get must beApprox(realVar(data)) - statsN.samples must_== 400 - } - "match concat" in { - statsN must_== (Stats.empty + statsN) - statsN must_== (statsN + Stats.empty) - statsN must beLike(Stats(data take 1) + Stats(data drop 1)) - statsN must beLike(Stats(data take 100) + Stats(data drop 100)) - statsN must beLike(Stats(data take 200) + Stats(data drop 200)) - } - } -} diff --git a/modules/evaluation/src/main/Statistics.scala b/modules/evaluation/src/main/Statistics.scala index 5bd52147df..0f3480a0d6 100644 --- a/modules/evaluation/src/main/Statistics.scala +++ b/modules/evaluation/src/main/Statistics.scala @@ -1,7 +1,7 @@ package lila.evaluation -import chess.Centis -import lila.common.{ Maths, Stats } +import chess.{ Centis, Stats } +import lila.common.Maths object Statistics { diff --git a/modules/round/src/main/Finisher.scala b/modules/round/src/main/Finisher.scala index 8421120e0d..cef2603eec 100644 --- a/modules/round/src/main/Finisher.scala +++ b/modules/round/src/main/Finisher.scala @@ -1,6 +1,6 @@ package lila.round -import chess.{ Status, Color } +import chess.{ Status, DecayingStats, Color, Clock } import lila.game.actorApi.{ FinishGame, AbortedBy } import lila.game.{ GameRepo, Game, Pov, RatingDiffs } @@ -59,6 +59,33 @@ private[round] final class Finisher( )(implicit proxy: GameProxy): Fu[Events] = apply(game, status, winner, message) >>- playban.other(game, status, winner) + private def recordLagStats(game: Game): Unit = for { + clock <- game.clock + player <- clock.players.all + lt = player.lag + stats = lt.lagStats + moves = lt.moves if moves > 4 + sd <- stats.stdDev + mean = stats.mean if mean > 0 + uncomp <- lt.totalUncomped / moves + compEstStdErr <- lt.compEstStdErr + quotaStr = f"${lt.quotaGain.centis / 10}%02d" + compEstOvers = lt.compEstOvers.centis + } { + import lila.mon.round.move.{ lag => lRec } + lRec.mean(Math.round(10 * mean)) + lRec.stdDev(Math.round(10 * sd)) + // wikipedia.org/wiki/Coefficient_of_variation#Estimation + lRec.coefVar(Math.round((1000f + 250f / moves) * sd / mean)) + lRec.uncomped(quotaStr)(uncomp.centis) + lRec.uncompedAll(uncomp.centis) + lt.lagEstimator match { + case h: DecayingStats => lRec.compDeviation(h.deviation.toInt) + } + lRec.compEstStdErr(Math.round(1000 * compEstStdErr)) + lRec.compEstOverErr(Math.round(10f * compEstOvers / moves)) + } + private def apply( game: Game, makeStatus: Status.type => Status, @@ -70,6 +97,7 @@ private[round] final class Finisher( if (game.nonAi && game.isCorrespondence) Color.all foreach notifier.gameEnd(prog.game) lila.mon.game.finish(status.name)() val g = prog.game + recordLagStats(g) proxy.save(prog) >> GameRepo.finish( id = g.id, @@ -92,6 +120,7 @@ private[round] final class Finisher( message foreach { messenger.system(g, _) } GameRepo game g.id foreach { newGame => bus.publish(finish.copy(game = newGame | g), 'finishGame) + } prog.events :+ lila.game.Event.EndData(g, ratingDiffs) } diff --git a/modules/round/src/main/Round.scala b/modules/round/src/main/Round.scala index 466523d2ce..b84b48e4db 100644 --- a/modules/round/src/main/Round.scala +++ b/modules/round/src/main/Round.scala @@ -6,7 +6,7 @@ import org.joda.time.DateTime import scala.concurrent.duration._ import actorApi._, round._ -import chess.{ DecayingStats, Color } +import chess.Color import lila.game.{ Game, Progress, Pov, Event } import lila.hub.actorApi.DeployPost import lila.hub.actorApi.map._ @@ -60,7 +60,7 @@ private[round] final class Round( handleHumanPlay(p) { pov => if (pov.game.outoftime(withGrace = true)) finisher.outOfTime(pov.game) else { - recordLagStats(pov) + recordLag(pov) player.human(p, self)(pov) } } >>- { @@ -240,33 +240,15 @@ private[round] final class Round( (game withClock newClock) ++ colors.map { Event.ClockInc(_, centis) } } - private def recordLagStats(pov: Pov) = + private def recordLag(pov: Pov) = if ((pov.game.playedTurns & 30) == 10) { // Triggers every 32 moves starting on ply 10. // i.e. 10, 11, 42, 43, 74, 75, ... for { + user <- pov.player.userId clock <- pov.game.clock - lt = clock.lag(pov.color) - lag <- lt.avgLag - } { - pov.player.userId.foreach { UserLagCache.put(_, lag) } - if (pov.game.playedTurns < 12) { - import lila.mon.round.move.{ lag => lRec } - lRec.avgReported(lag.centis) - lt.history match { - case h: DecayingStats => lRec.compDeviation(h.deviation.toInt) - } - for { - lowEst <- lt.lowEstimate - avgComp <- lt.avgLagComp - uncomp <- (lt.totalLag - lt.totalComp) / lt.lagSteps - } { - lRec.estimateError((avgComp - lowEst).centis) - lRec.uncomped(f"${lt.quotaGain.centis / 10}%02d")(uncomp.centis) - lRec.uncompedAll(uncomp.centis) - } - } - } + lag <- clock.lag(pov.color).lagMean + } UserLagCache.put(user, lag) } private def scheduleExpiration: Funit = proxy.game map { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c17b3c8fac..4c4f20815d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,7 @@ object Dependencies { val findbugs = "com.google.code.findbugs" % "jsr305" % "3.0.1" val hasher = "com.roundeights" %% "hasher" % "1.2.0" val jodaTime = "joda-time" % "joda-time" % "2.9.9" - val chess = "org.lichess" %% "scalachess" % "8.4" + val chess = "org.lichess" %% "scalachess" % "8.6" val compression = "org.lichess" %% "compression" % "1.2" val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.2.3-THIB" val prismic = "io.prismic" %% "scala-kit" % "1.2.11-THIB"