2016-02-08 08:42:33 -07:00
|
|
|
package lila.explorer
|
|
|
|
|
2016-02-09 02:34:10 -07:00
|
|
|
import scala.util.Random.nextFloat
|
2016-02-09 03:07:35 -07:00
|
|
|
import scala.util.{ Try, Success, Failure }
|
2016-02-08 08:42:33 -07:00
|
|
|
|
|
|
|
import chess.variant.Variant
|
2016-02-08 22:00:49 -07:00
|
|
|
import org.joda.time.DateTime
|
2016-02-18 06:04:15 -07:00
|
|
|
import org.joda.time.format.DateTimeFormat
|
2016-02-08 08:42:33 -07:00
|
|
|
import play.api.libs.iteratee._
|
2016-02-09 02:57:23 -07:00
|
|
|
import play.api.libs.ws.WS
|
2016-02-08 08:42:33 -07:00
|
|
|
import play.api.Play.current
|
|
|
|
|
|
|
|
import lila.db.api._
|
|
|
|
import lila.db.Implicits._
|
|
|
|
import lila.game.BSONHandlers.gameBSONHandler
|
|
|
|
import lila.game.tube.gameTube
|
|
|
|
import lila.game.{ Game, GameRepo, Query, PgnDump, Player }
|
2016-02-11 07:56:53 -07:00
|
|
|
import lila.user.UserRepo
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2016-03-04 06:58:13 -07:00
|
|
|
private final class ExplorerIndexer(
|
|
|
|
endpoint: String,
|
|
|
|
massImportEndpoint: String) {
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2016-02-08 22:00:49 -07:00
|
|
|
private val maxGames = Int.MaxValue
|
2016-03-04 22:50:07 -07:00
|
|
|
private val batchSize = 50
|
2016-02-09 18:00:19 -07:00
|
|
|
private val maxPlies = 50
|
2016-02-08 22:00:49 -07:00
|
|
|
private val separator = "\n\n\n"
|
2016-02-09 03:07:35 -07:00
|
|
|
private val datePattern = "yyyy-MM-dd"
|
2016-02-11 07:56:53 -07:00
|
|
|
private val dateFormatter = DateTimeFormat forPattern datePattern
|
|
|
|
private val dateTimeFormatter = DateTimeFormat forPattern s"$datePattern HH:mm"
|
|
|
|
private val pgnDateFormat = DateTimeFormat forPattern "yyyy.MM.dd";
|
2016-03-04 06:58:13 -07:00
|
|
|
private val endPointUrl = s"$endpoint/import/lichess"
|
|
|
|
private val massImportEndPointUrl = s"$massImportEndpoint/import/lichess"
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2016-02-09 03:27:12 -07:00
|
|
|
private def parseDate(str: String): Option[DateTime] =
|
|
|
|
Try(dateFormatter parseDateTime str).toOption
|
2016-02-09 03:07:35 -07:00
|
|
|
|
2016-03-04 22:50:07 -07:00
|
|
|
type GamePGN = (Game, String)
|
|
|
|
|
2016-02-10 09:55:01 -07:00
|
|
|
def apply(sinceStr: String): Funit =
|
|
|
|
parseDate(sinceStr).fold(fufail[Unit](s"Invalid date $sinceStr")) { since =>
|
|
|
|
logger.info(s"Start indexing since $since")
|
2016-02-08 08:42:33 -07:00
|
|
|
val query = $query(
|
2016-02-11 07:56:53 -07:00
|
|
|
Query.createdSince(since) ++
|
2016-02-09 03:07:35 -07:00
|
|
|
Query.rated ++
|
2016-02-08 08:42:33 -07:00
|
|
|
Query.finished ++
|
2016-02-20 04:42:44 -07:00
|
|
|
Query.turnsMoreThan(8) ++
|
2016-02-09 20:41:21 -07:00
|
|
|
Query.noProvisional ++
|
2016-02-23 20:36:37 -07:00
|
|
|
Query.bothRatingsGreaterThan(1501)
|
2016-02-09 08:28:44 -07:00
|
|
|
)
|
2016-03-04 22:25:10 -07:00
|
|
|
import reactivemongo.api._
|
2016-02-08 08:42:33 -07:00
|
|
|
pimpQB(query)
|
|
|
|
.sort(Query.sortChronological)
|
2016-03-14 06:51:00 -06:00
|
|
|
.cursor[Game](ReadPreference.secondaryPreferred)
|
2016-02-08 08:42:33 -07:00
|
|
|
.enumerate(maxGames, stopOnError = true) &>
|
2016-03-04 22:50:07 -07:00
|
|
|
Enumeratee.mapM[Game].apply[Option[GamePGN]] { game =>
|
2016-02-09 03:27:12 -07:00
|
|
|
makeFastPgn(game) map {
|
|
|
|
_ map { game -> _ }
|
|
|
|
}
|
|
|
|
} &>
|
2016-03-05 20:13:44 -07:00
|
|
|
Enumeratee.collect { case Some(el) => el } &>
|
2016-02-08 08:42:33 -07:00
|
|
|
Enumeratee.grouped(Iteratee takeUpTo batchSize) |>>>
|
2016-03-04 22:50:07 -07:00
|
|
|
Iteratee.foldM[Seq[GamePGN], Long](nowMillis) {
|
|
|
|
case (millis, pairs) =>
|
2016-03-04 22:24:07 -07:00
|
|
|
WS.url(massImportEndPointUrl).put(pairs.map(_._2) mkString separator).flatMap {
|
|
|
|
case res if res.status == 200 =>
|
2016-02-09 03:27:12 -07:00
|
|
|
val date = pairs.headOption.map(_._1.createdAt) ?? dateTimeFormatter.print
|
|
|
|
val nb = pairs.size
|
2016-02-09 22:24:48 -07:00
|
|
|
val gameMs = (nowMillis - millis) / nb.toDouble
|
2016-03-04 22:50:07 -07:00
|
|
|
logger.info(s"$date $nb ${gameMs.toInt} ms/game ${(1000 / gameMs).toInt} games/s")
|
2016-03-04 22:24:07 -07:00
|
|
|
funit
|
|
|
|
case res => fufail(s"Stop import because of status ${res.status}")
|
|
|
|
} >> {
|
2016-03-04 22:50:07 -07:00
|
|
|
pairs.headOption match {
|
|
|
|
case None => fufail(s"No games left, import complete!")
|
|
|
|
case Some((g, _)) if (g.createdAt.isAfter(DateTime.now.minusMinutes(10))) =>
|
|
|
|
fufail(s"Found a recent game, import complete!")
|
|
|
|
case _ => funit
|
|
|
|
}
|
2016-02-09 03:27:12 -07:00
|
|
|
} inject nowMillis
|
2016-02-08 08:42:33 -07:00
|
|
|
} void
|
2016-02-09 03:07:35 -07:00
|
|
|
}
|
2016-02-08 22:00:49 -07:00
|
|
|
|
2016-02-18 06:04:15 -07:00
|
|
|
def apply(game: Game): Funit = makeFastPgn(game) map {
|
|
|
|
_ foreach flowBuffer.apply
|
|
|
|
}
|
|
|
|
|
|
|
|
private object flowBuffer {
|
2016-02-22 22:24:32 -07:00
|
|
|
private val max = 30
|
2016-02-18 06:04:15 -07:00
|
|
|
private val buf = scala.collection.mutable.ArrayBuffer.empty[String]
|
|
|
|
def apply(pgn: String) {
|
|
|
|
buf += pgn
|
2016-02-18 22:39:25 -07:00
|
|
|
val startAt = nowMillis
|
2016-02-18 06:04:15 -07:00
|
|
|
if (buf.size >= max) {
|
2016-03-04 06:58:13 -07:00
|
|
|
WS.url(endPointUrl).put(buf mkString separator) andThen {
|
2016-02-18 06:04:15 -07:00
|
|
|
case Success(res) if res.status == 200 =>
|
2016-03-10 11:21:04 -07:00
|
|
|
lila.mon.explorer.index.time(((nowMillis - startAt) / max).toInt)
|
2016-03-11 03:18:22 -07:00
|
|
|
lila.mon.explorer.index.success(max)
|
2016-03-10 11:21:04 -07:00
|
|
|
case Success(res) =>
|
|
|
|
logger.warn(s"[${res.status}]")
|
2016-03-11 03:18:22 -07:00
|
|
|
lila.mon.explorer.index.failure(max)
|
2016-03-10 11:21:04 -07:00
|
|
|
case Failure(err) =>
|
|
|
|
logger.warn(s"$err")
|
2016-03-11 03:18:22 -07:00
|
|
|
lila.mon.explorer.index.failure(max)
|
2016-02-18 06:04:15 -07:00
|
|
|
}
|
|
|
|
buf.clear
|
|
|
|
}
|
2016-02-08 22:00:49 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private def valid(game: Game) =
|
|
|
|
game.finished &&
|
|
|
|
game.rated &&
|
|
|
|
game.turns >= 10 &&
|
2016-02-10 09:55:01 -07:00
|
|
|
game.variant != chess.variant.FromPosition &&
|
|
|
|
(game.variant != chess.variant.Horde || game.createdAt.isAfter(Query.hordeWhitePawnsSince))
|
2016-02-08 22:00:49 -07:00
|
|
|
|
|
|
|
private def stableRating(player: Player) = player.rating ifFalse player.provisional
|
|
|
|
|
2016-02-09 02:34:10 -07:00
|
|
|
// probability of the game being indexed, between 0 and 1
|
|
|
|
private def probability(game: Game, rating: Int) = {
|
|
|
|
import lila.rating.PerfType._
|
|
|
|
game.perfType ?? {
|
2016-02-20 18:54:53 -07:00
|
|
|
case Correspondence => 1
|
2016-03-01 08:27:40 -07:00
|
|
|
case Classical if rating >= 2000 => 1
|
2016-02-24 11:17:31 -07:00
|
|
|
case Classical if rating >= 1800 => 2 / 5f
|
2016-03-07 08:23:27 -07:00
|
|
|
case Classical => 1 / 8f
|
2016-02-20 18:54:53 -07:00
|
|
|
case Blitz if rating >= 2000 => 1
|
2016-03-05 20:07:11 -07:00
|
|
|
case Blitz if rating >= 1800 => 1 / 4f
|
|
|
|
case Blitz => 1 / 8f
|
2016-02-20 18:54:53 -07:00
|
|
|
case Bullet if rating >= 2200 => 1
|
2016-02-24 11:17:31 -07:00
|
|
|
case Bullet if rating >= 2000 => 1 / 3f
|
2016-03-05 20:07:11 -07:00
|
|
|
case Bullet if rating >= 1800 => 1 / 5f
|
|
|
|
case Bullet => 1 / 7f
|
2016-02-20 18:54:53 -07:00
|
|
|
case _ if rating >= 1600 => 1 // variant games
|
|
|
|
case _ => 1 / 2f // noob variant games
|
2016-02-09 02:34:10 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-08 22:00:49 -07:00
|
|
|
private def makeFastPgn(game: Game): Fu[Option[String]] = ~(for {
|
|
|
|
whiteRating <- stableRating(game.whitePlayer)
|
|
|
|
blackRating <- stableRating(game.blackPlayer)
|
2016-02-19 20:58:29 -07:00
|
|
|
minPlayerRating = if (game.variant.exotic) 1400 else 1500
|
2016-03-05 20:13:44 -07:00
|
|
|
minAverageRating = if (game.variant.exotic) 1520 else 1600
|
2016-02-19 20:58:29 -07:00
|
|
|
if whiteRating >= minPlayerRating
|
|
|
|
if blackRating >= minPlayerRating
|
2016-02-09 02:34:10 -07:00
|
|
|
averageRating = (whiteRating + blackRating) / 2
|
2016-02-19 20:58:29 -07:00
|
|
|
if averageRating >= minAverageRating
|
2016-02-09 02:34:10 -07:00
|
|
|
if probability(game, averageRating) > nextFloat
|
2016-02-08 22:00:49 -07:00
|
|
|
if valid(game)
|
2016-02-11 07:56:53 -07:00
|
|
|
} yield GameRepo initialFen game flatMap { initialFen =>
|
|
|
|
UserRepo.usernamesByIds(game.userIds) map { usernames =>
|
|
|
|
def username(color: chess.Color) = game.player(color).userId flatMap { id =>
|
|
|
|
usernames.find(_.toLowerCase == id)
|
|
|
|
} orElse game.player(color).userId getOrElse "?"
|
|
|
|
val fenTags = initialFen.?? { fen => List(s"[FEN $fen]") }
|
2016-02-19 19:12:40 -07:00
|
|
|
val timeControl = game.clock.fold("-") { c => s"${c.limit}+${c.increment}" }
|
2016-02-11 07:56:53 -07:00
|
|
|
val otherTags = List(
|
|
|
|
s"[LichessID ${game.id}]",
|
|
|
|
s"[Variant ${game.variant.name}]",
|
2016-02-19 19:12:40 -07:00
|
|
|
s"[TimeControl $timeControl]",
|
2016-02-11 07:56:53 -07:00
|
|
|
s"[White ${username(chess.White)}]",
|
|
|
|
s"[Black ${username(chess.Black)}]",
|
|
|
|
s"[WhiteElo $whiteRating]",
|
|
|
|
s"[BlackElo $blackRating]",
|
|
|
|
s"[Result ${PgnDump.result(game)}]",
|
|
|
|
s"[Date ${pgnDateFormat.print(game.createdAt)}]")
|
|
|
|
val allTags = fenTags ::: otherTags
|
2016-02-12 09:09:38 -07:00
|
|
|
s"${allTags.mkString("\n")}\n\n${game.pgnMoves.take(maxPlies).mkString(" ")}".some
|
2016-02-11 07:56:53 -07:00
|
|
|
}
|
2016-02-08 22:00:49 -07:00
|
|
|
})
|
|
|
|
|
|
|
|
private val logger = play.api.Logger("explorer")
|
2016-02-08 08:42:33 -07:00
|
|
|
}
|