Improve monitor UI and add AI latency

app/ai/RemoteAi.scala

@@ -13,7 +13,8 @@ final class RemoteAi(
 
   // tells whether the remote AI is healthy or not
   // frequently updated by a scheduled actor
-  private var health = false
+  private var ping = none[Int]
+  val pingAlert = 3000
 
   private lazy val http = new Http with ThreadSafety with NoLogging
   private lazy val urlObj = url(remoteUrl)
@@ -28,16 +29,17 @@ final class RemoteAi(
     }
   }
 
-  def or(fallback: Ai) = if (health) this else fallback
+  def or(fallback: Ai) = if (currentHealth) this else fallback
 
-  def currentHealth = health
+  def currentPing = ping
+  def currentHealth = ping.fold(_ < pingAlert, false)
 
   def diagnose: IO[Unit] = for {
-    h ← healthCheck
-    _ ← h.fold(
-      health.fold(io(), putStrLn("remote AI is up")),
-      putStrLn("remote AI is down"))
-    _ ← io { health = h }
+    p ← tryPing
+    _ ← p.fold(_ < pingAlert, false).fold(
+      currentHealth.fold(io(), putStrLn("remote AI is up, ping = " + p)),
+      putStrLn("remote AI is down, ping = " + p))
+    _ ← io { ping = p }
   } yield ()
 
   private def fetchNewFen(oldFen: String, level: Int): IO[String] = io {
@@ -47,8 +49,12 @@ final class RemoteAi(
     ) as_str)
   }
 
-  private def healthCheck: IO[Boolean] = fetchNewFen(
-    oldFen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq",
-    level = 1
-  ).catchLeft map (_.isRight)
+  private def tryPing: IO[Option[Int]] = for {
+    start ← io(nowMillis)
+    received ← fetchNewFen(
+      oldFen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq",
+      level = 1
+    ).catchLeft map (_.isRight)
+    delay ← io(nowMillis - start)
+  } yield received option delay.toInt
 }

app/core/Cron.scala

@@ -69,7 +69,7 @@ object Cron {
       }
     }
 
-    effect(1 minute, "ai diagnose") {
+    effect(10 seconds, "ai diagnose") {
       env.ai.remoteAi.diagnose
     }
     env.ai.remoteAi.diagnose.unsafePerformIO

app/core/Global.scala

@@ -20,7 +20,7 @@ object Global extends GlobalSettings {
   }
 
   override def onRouteRequest(req: RequestHeader): Option[Handler] = {
-    println(req)
+    //println(req)
     env.monitor.rpsProvider.countRequest()
     env.i18n.requestHandler(req) orElse super.onRouteRequest(req)
   }

app/monitor/Reporting.scala

@@ -36,7 +36,7 @@ final class Reporting(
   var rps = 0
   var cpu = 0
   var mongoStatus = MongoStatus.default
-  var remoteAi = false
+  var remoteAi = none[Int]
 
   var displays = 0
 
@@ -89,7 +89,7 @@ final class Reporting(
           memory = memoryStats.getHeapMemoryUsage.getUsed / 1024 / 1024
           rps = rpsProvider.rps
           cpu = ((cpuStats.getCpuUsage() * 1000).round / 10.0).toInt
-          remoteAi = env.ai.remoteAi.currentHealth
+          remoteAi = env.ai.remoteAi.currentPing
         }
       } onComplete {
         case Left(a) ⇒ println("Reporting: " + a.getMessage)
@@ -125,7 +125,7 @@ final class Reporting(
     nbPlaying,
     game.nbHubs,
     loadAvg.toString,
-    remoteAi.fold(1, 0)
+    remoteAi.isDefined.fold(1, 0)
   ) mkString " "
 
   private def monitorData = List(
@@ -141,7 +141,8 @@ final class Reporting(
     "dbMemory" -> mongoStatus.memory,
     "dbConn" -> mongoStatus.connection,
     "dbQps" -> mongoStatus.qps,
-    "dbLock" -> math.round(mongoStatus.lock * 10) / 10d
+    "dbLock" -> math.round(mongoStatus.lock * 10) / 10d,
+    "ai" -> (remoteAi | 9999)
   ) map {
       case (name, value) ⇒ value + ":" + name
     }

app/views/monitor/monitor.scala.html

@@ -1,9 +1,17 @@
 @(totalMemory: Long)
 
 @main("RPS") {
-<h1>Lichess Server Monitoring <span class="down">DOWN</span></h1>
+<h1>Lichess reactor <span class="down">DOWN</span><span class="up">OPERATIONAL</span></h1>
 <div id="monitors" class="clearfix">
   <script type="text/javascript">window.App = { }; window.App.totalMemory = @totalMemory;</script>
   <script type="text/javascript" src="@routes.Assets.at("javascripts/monitor.js")"></script>
 </div>
+<div id="actions">
+<a href="#" id="shutdown">Emergency reactor shutdown - DON'T CLICK</a>
+</div>
+<script>
+window.document.getElementById("shutdown").onclick = function() {
+  alert("Was worth trying. I guess.");
+};
+</script>
 }

public/javascripts/monitor.js

@@ -30,7 +30,7 @@
     this.elt.appendChild(this.needle);
 
     this.light = create("div");
-    this.light.className = "green light";
+    this.light.className = "light";
     this.elt.appendChild(this.light);
 
     var wheel = create("div");
@@ -70,7 +70,7 @@
     app.memory = new SpeedOMeter({
       name : "MEMORY",
       maxVal : app.totalMemory,
-      threshold: 0.8,
+      threshold: 0.9,
       unit : "MB",
       container : container
     });
@@ -100,7 +100,6 @@
     app.lat = new SpeedOMeter({
       name : "LATENCY",
       maxVal : 5,
-      threshold: 0.5,
       container : container
     });
 
@@ -140,7 +139,7 @@
 
     app.dbQps = new SpeedOMeter({
       name : "DB QPS",
-      maxVal : 200,
+      maxVal : 300,
       threshold: 0.8,
       container : container
     });
@@ -151,6 +150,12 @@
       container : container
     });
 
+    app.ai = new SpeedOMeter({
+      name : "AI PING",
+      maxVal : 1000,
+      container : container
+    });
+
     var iframe = create("iframe");
     iframe.src = "/monitor/stream";
     iframe.style.display = "none";
@@ -172,7 +177,7 @@
       app.lastCall = (new Date()).getTime();
       window.document.body.appendChild(iframe);
     }, 100);
-    
+
     setInterval(function () {
       if ((new Date()).getTime() - app.lastCall > 3000) {
         window.document.body.className = "down";

public/stylesheets/monitor.css

@@ -14,15 +14,34 @@ body {
 
 h1 {
   text-align: center;
-  font-size: 20px;
+  font-size: 36px;
+  color: #222;
+  text-transform: uppercase;
 }
 h1 .down {
-  color: #884444;
+  color: #aa4444;
   display: none;
 }
+h1 .up {
+  color: #446644;
+}
 body.down h1 .down {
   display: inline;
 }
+body.down h1 .up {
+  display: none;
+}
+
+#actions {
+  margin: 70px 0 20px 0;
+  text-align: center;
+}
+
+a#shutdown {
+  font-size: 14px;
+  color: #888;
+  font-family: monospace;
+}
 
 .monitor {
   float: left;
@@ -38,7 +57,7 @@ body.down h1 .down {
   color: #d0d0d0;
   font-size: 10px;
   position: absolute;
-  top: 9px;
+  top: 10px;
   left: 223px;
 }
 .monitor .screen {
@@ -90,16 +109,17 @@ body.down h1 .down {
   background: url(/assets/images/monitor/needle.png);
 }
 div.light {
-  width: 7px;
-  height: 7px;
+  width: 26px;
+  height: 26px;
   position: absolute;
-  top: 11px;
-  left: 211px;
+  top: 2px;
+  left: 200px;
+  background: url(/assets/images/status.png);
+  background-position: 0 -52px;
 }
-div.light.green {
-  background: url(/assets/images/monitor/green-light.png);
-}
-div.light.red,
 .monitor.alert div.light {  
-  background: url(/assets/images/monitor/red-light.png);
+  background-position: 0 -26px;
+}
+body.up div.light  {
+  background-position: 0 0;
 }

todo

@@ -20,6 +20,8 @@ translation contributions
 check sf2 commands to port
 make the wiki static html pages managed by git?
 compensate lag http://fr.lichess.org/forum/lichess-feedback/the-clock-display-should-account-for-lag?page=1#4
+show lobby room logs
+refactor preload
 
 next deploy:
 bin/migrate