update ExplorerIndexer for explorer v3

modules/explorer/src/main/ExplorerIndexer.scala

@@ -1,16 +1,15 @@
 package lila.explorer
 
 import akka.stream.scaladsl._
-import chess.format.pgn.Tag
-import org.joda.time.DateTime
 import org.joda.time.format.DateTimeFormat
-import play.api.libs.ws.DefaultBodyWritables._
+import play.api.libs.json._
+import play.api.libs.ws.JsonBodyWritables._
 import lila.common.ThreadLocalRandom.nextFloat
 import scala.util.{ Failure, Success, Try }
 
 import lila.common.LilaStream
 import lila.db.dsl._
-import lila.game.{ Game, GameRepo, PgnDump, Player, Query }
+import lila.game.{ Game, GameRepo, Player }
 import lila.user.{ User, UserRepo }
 
 final private class ExplorerIndexer(
@@ -24,25 +23,24 @@ final private class ExplorerIndexer(
     mat: akka.stream.Materializer
 ) {
 
-  private val separator           = "\n\n\n"
   private val pgnDateFormat       = DateTimeFormat forPattern "yyyy.MM.dd"
   private val internalEndPointUrl = s"$internalEndpoint/import/lichess"
 
   def apply(game: Game): Funit =
     getBotUserIds() flatMap { botUserIds =>
-      makeFastPgn(game, botUserIds) map {
+      makeJson(game, botUserIds) map {
         _ foreach flowBuffer.apply
       }
     }
 
   private object flowBuffer {
     private val max = 30
-    private val buf = scala.collection.mutable.ArrayBuffer.empty[String]
+    private val buf = scala.collection.mutable.ArrayBuffer.empty[JsObject]
-    def apply(pgn: String): Unit = {
+    def apply(game: JsObject): Unit = {
-      buf += pgn
+      buf += game
       val startAt = nowMillis
       if (buf.sizeIs >= max) {
-        ws.url(internalEndPointUrl).put(buf mkString separator) andThen {
+        ws.url(internalEndPointUrl).put(JsArray(buf)) andThen {
           case Success(res) if res.status == 200 =>
             lila.mon.explorer.index.time.record((nowMillis - startAt) / max)
             lila.mon.explorer.index.count(true).increment(max)
@@ -61,43 +59,49 @@ final private class ExplorerIndexer(
   private def valid(game: Game) =
     game.finished &&
       game.rated &&
-      game.turns >= 10 &&
       game.variant != chess.variant.FromPosition &&
       !Game.isOldHorde(game)
 
   private def stableRating(player: Player) = player.rating ifFalse player.provisional
 
   // probability of the game being indexed, between 0 and 1
-  private def probability(game: Game, rating: Int) = {
+  private def probability(game: Game, rating: Int): Float = {
     import lila.rating.PerfType._
     game.perfType ?? {
-      case Correspondence                      => 1
+      case Correspondence | Classical => 1.00f
-      case Rapid | Classical if rating >= 2000 => 1
+
-      case Rapid | Classical if rating >= 1800 => 2 / 5f
+      case Rapid if rating >= 2200 => 1.00f
-      case Rapid | Classical                   => 1 / 8f
+      case Rapid if rating >= 2000 => 0.30f
-      case Blitz if rating >= 2000             => 1
+      case Rapid if rating >= 1800 => 0.18f
-      case Blitz if rating >= 1800             => 1 / 4f
+      case Rapid if rating >= 1600 => 0.16f
-      case Blitz                               => 1 / 15f
+      case Rapid                   => 0.02f
-      case Bullet if rating >= 2300            => 1
+
-      case Bullet if rating >= 2200            => 4 / 5f
+      case Blitz if rating >= 2500 => 1.00f
-      case Bullet if rating >= 2000            => 1 / 4f
+      case Blitz if rating >= 2200 => 0.16f
-      case Bullet if rating >= 1800            => 1 / 7f
+      case Blitz if rating >= 2000 => 0.07f
-      case Bullet                              => 1 / 20f
+      case Blitz if rating >= 1600 => 0.05f
-      case _ if rating >= 1600                 => 1      // variant games
+      case Blitz                   => 0.02f
-      case _                                   => 1 / 2f // noob variant games
+
+      case Bullet if rating >= 2500 => 1.00f
+      case Bullet if rating >= 2200 => 0.20f
+      case Bullet if rating >= 2000 => 0.11f
+      case Bullet if rating >= 1600 => 0.07f
+      case Bullet                   => 0.02f
+
+      case UltraBullet => 1.00f
+
+      case _ if rating >= 1600 => 1.00f // variant games
+      case _                   => 0.50f // noob variant games
     }
   }
 
-  private def makeFastPgn(game: Game, botUserIds: Set[User.ID]): Fu[Option[String]] =
+  private def makeJson(game: Game, botUserIds: Set[User.ID]): Fu[Option[JsObject]] =
     ~(for {
       whiteRating <- stableRating(game.whitePlayer)
       blackRating <- stableRating(game.blackPlayer)
-      minPlayerRating  = if (game.variant.exotic) 1400 else 1500
+      if whiteRating >= 1501
-      minAverageRating = if (game.variant.exotic) 1520 else 1600
+      if blackRating >= 1501
-      if whiteRating >= minPlayerRating
-      if blackRating >= minPlayerRating
       averageRating = (whiteRating + blackRating) / 2
-      if averageRating >= minAverageRating
       if probability(game, averageRating) > nextFloat()
       if !game.userIds.exists(botUserIds.contains)
       if valid(game)
@@ -107,22 +111,25 @@ final private class ExplorerIndexer(
           game.player(color).userId flatMap { id =>
             usernames.find(_.toLowerCase == id)
           } orElse game.player(color).userId getOrElse "?"
-        val fenTags = initialFen.?? { fen =>
+        Json
-          List(Tag(_.FEN, fen))
+          .obj(
-        }
+            "id"      -> game.id,
-        val otherTags = List(
+            "variant" -> game.variant.key,
-          Tag("LichessID", game.id),
+            "speed"   -> game.speed.key,
-          Tag(_.Variant, game.variant.name),
+            "white" -> Json.obj(
-          Tag.timeControl(game.clock.map(_.config)),
+              "name"   -> username(chess.White),
-          Tag(_.White, username(chess.White)),
+              "rating" -> whiteRating
-          Tag(_.Black, username(chess.Black)),
+            ),
-          Tag(_.WhiteElo, whiteRating),
+            "black" -> Json.obj(
-          Tag(_.BlackElo, blackRating),
+              "name"   -> username(chess.Black),
-          Tag(_.Result, PgnDump.result(game)),
+              "rating" -> blackRating
-          Tag(_.Date, pgnDateFormat.print(game.createdAt))
+            ),
-        )
+            "winner" -> game.winnerColor.map(_.name),
-        val allTags = fenTags ::: otherTags
+            "date"   -> pgnDateFormat.print(game.createdAt),
-        s"${allTags.mkString("\n")}\n\n${game.pgnMoves.take(maxPlies).mkString(" ")}".some
+            "fen"    -> initialFen.map(_.value),
+            "moves"  -> game.pgnMoves.mkString(" ")
+          )
+          .some
       }
     })
 

modules/explorer/src/main/package.scala

@@ -1,6 +1,3 @@
 package lila
 
-package object explorer extends PackageObject {
+package object explorer extends PackageObject {}
-
-  val maxPlies = 50
-}