From 18bcdbccca526cc201785ed9e6398e21f0c1e4e0 Mon Sep 17 00:00:00 2001 From: Niklas Fiekas Date: Wed, 27 Oct 2021 18:53:57 +0200 Subject: [PATCH] update ExplorerIndexer for explorer v3 --- .../explorer/src/main/ExplorerIndexer.scala | 103 ++++++++++-------- modules/explorer/src/main/package.scala | 5 +- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/modules/explorer/src/main/ExplorerIndexer.scala b/modules/explorer/src/main/ExplorerIndexer.scala index 2983b117f5..af847e8778 100644 --- a/modules/explorer/src/main/ExplorerIndexer.scala +++ b/modules/explorer/src/main/ExplorerIndexer.scala @@ -1,16 +1,15 @@ package lila.explorer import akka.stream.scaladsl._ -import chess.format.pgn.Tag -import org.joda.time.DateTime import org.joda.time.format.DateTimeFormat -import play.api.libs.ws.DefaultBodyWritables._ +import play.api.libs.json._ +import play.api.libs.ws.JsonBodyWritables._ import lila.common.ThreadLocalRandom.nextFloat import scala.util.{ Failure, Success, Try } import lila.common.LilaStream import lila.db.dsl._ -import lila.game.{ Game, GameRepo, PgnDump, Player, Query } +import lila.game.{ Game, GameRepo, Player } import lila.user.{ User, UserRepo } final private class ExplorerIndexer( @@ -24,25 +23,24 @@ final private class ExplorerIndexer( mat: akka.stream.Materializer ) { - private val separator = "\n\n\n" private val pgnDateFormat = DateTimeFormat forPattern "yyyy.MM.dd" private val internalEndPointUrl = s"$internalEndpoint/import/lichess" def apply(game: Game): Funit = getBotUserIds() flatMap { botUserIds => - makeFastPgn(game, botUserIds) map { + makeJson(game, botUserIds) map { _ foreach flowBuffer.apply } } private object flowBuffer { private val max = 30 - private val buf = scala.collection.mutable.ArrayBuffer.empty[String] - def apply(pgn: String): Unit = { - buf += pgn + private val buf = scala.collection.mutable.ArrayBuffer.empty[JsObject] + def apply(game: JsObject): Unit = { + buf += game val startAt = nowMillis if (buf.sizeIs >= max) { - ws.url(internalEndPointUrl).put(buf mkString separator) andThen { + ws.url(internalEndPointUrl).put(JsArray(buf)) andThen { case Success(res) if res.status == 200 => lila.mon.explorer.index.time.record((nowMillis - startAt) / max) lila.mon.explorer.index.count(true).increment(max) @@ -61,43 +59,49 @@ final private class ExplorerIndexer( private def valid(game: Game) = game.finished && game.rated && - game.turns >= 10 && game.variant != chess.variant.FromPosition && !Game.isOldHorde(game) private def stableRating(player: Player) = player.rating ifFalse player.provisional // probability of the game being indexed, between 0 and 1 - private def probability(game: Game, rating: Int) = { + private def probability(game: Game, rating: Int): Float = { import lila.rating.PerfType._ game.perfType ?? { - case Correspondence => 1 - case Rapid | Classical if rating >= 2000 => 1 - case Rapid | Classical if rating >= 1800 => 2 / 5f - case Rapid | Classical => 1 / 8f - case Blitz if rating >= 2000 => 1 - case Blitz if rating >= 1800 => 1 / 4f - case Blitz => 1 / 15f - case Bullet if rating >= 2300 => 1 - case Bullet if rating >= 2200 => 4 / 5f - case Bullet if rating >= 2000 => 1 / 4f - case Bullet if rating >= 1800 => 1 / 7f - case Bullet => 1 / 20f - case _ if rating >= 1600 => 1 // variant games - case _ => 1 / 2f // noob variant games + case Correspondence | Classical => 1.00f + + case Rapid if rating >= 2200 => 1.00f + case Rapid if rating >= 2000 => 0.30f + case Rapid if rating >= 1800 => 0.18f + case Rapid if rating >= 1600 => 0.16f + case Rapid => 0.02f + + case Blitz if rating >= 2500 => 1.00f + case Blitz if rating >= 2200 => 0.16f + case Blitz if rating >= 2000 => 0.07f + case Blitz if rating >= 1600 => 0.05f + case Blitz => 0.02f + + case Bullet if rating >= 2500 => 1.00f + case Bullet if rating >= 2200 => 0.20f + case Bullet if rating >= 2000 => 0.11f + case Bullet if rating >= 1600 => 0.07f + case Bullet => 0.02f + + case UltraBullet => 1.00f + + case _ if rating >= 1600 => 1.00f // variant games + case _ => 0.50f // noob variant games } } - private def makeFastPgn(game: Game, botUserIds: Set[User.ID]): Fu[Option[String]] = + private def makeJson(game: Game, botUserIds: Set[User.ID]): Fu[Option[JsObject]] = ~(for { whiteRating <- stableRating(game.whitePlayer) blackRating <- stableRating(game.blackPlayer) - minPlayerRating = if (game.variant.exotic) 1400 else 1500 - minAverageRating = if (game.variant.exotic) 1520 else 1600 - if whiteRating >= minPlayerRating - if blackRating >= minPlayerRating + if whiteRating >= 1501 + if blackRating >= 1501 averageRating = (whiteRating + blackRating) / 2 - if averageRating >= minAverageRating if probability(game, averageRating) > nextFloat() if !game.userIds.exists(botUserIds.contains) if valid(game) @@ -107,22 +111,25 @@ final private class ExplorerIndexer( game.player(color).userId flatMap { id => usernames.find(_.toLowerCase == id) } orElse game.player(color).userId getOrElse "?" - val fenTags = initialFen.?? { fen => - List(Tag(_.FEN, fen)) - } - val otherTags = List( - Tag("LichessID", game.id), - Tag(_.Variant, game.variant.name), - Tag.timeControl(game.clock.map(_.config)), - Tag(_.White, username(chess.White)), - Tag(_.Black, username(chess.Black)), - Tag(_.WhiteElo, whiteRating), - Tag(_.BlackElo, blackRating), - Tag(_.Result, PgnDump.result(game)), - Tag(_.Date, pgnDateFormat.print(game.createdAt)) - ) - val allTags = fenTags ::: otherTags - s"${allTags.mkString("\n")}\n\n${game.pgnMoves.take(maxPlies).mkString(" ")}".some + Json + .obj( + "id" -> game.id, + "variant" -> game.variant.key, + "speed" -> game.speed.key, + "white" -> Json.obj( + "name" -> username(chess.White), + "rating" -> whiteRating + ), + "black" -> Json.obj( + "name" -> username(chess.Black), + "rating" -> blackRating + ), + "winner" -> game.winnerColor.map(_.name), + "date" -> pgnDateFormat.print(game.createdAt), + "fen" -> initialFen.map(_.value), + "moves" -> game.pgnMoves.mkString(" ") + ) + .some } }) diff --git a/modules/explorer/src/main/package.scala b/modules/explorer/src/main/package.scala index 0d476113a8..533fce38df 100644 --- a/modules/explorer/src/main/package.scala +++ b/modules/explorer/src/main/package.scala @@ -1,6 +1,3 @@ package lila -package object explorer extends PackageObject { - - val maxPlies = 50 -} +package object explorer extends PackageObject {}