2016-02-08 08:42:33 -07:00
|
|
|
package lila.explorer
|
|
|
|
|
2019-12-08 20:20:36 -07:00
|
|
|
import akka.stream.scaladsl._
|
2018-10-18 01:31:34 -06:00
|
|
|
import chess.format.pgn.Tag
|
2016-02-08 22:00:49 -07:00
|
|
|
import org.joda.time.DateTime
|
2016-02-18 06:04:15 -07:00
|
|
|
import org.joda.time.format.DateTimeFormat
|
2020-08-07 08:22:26 -06:00
|
|
|
import play.api.libs.ws.DefaultBodyWritables._
|
2018-10-18 01:31:34 -06:00
|
|
|
import scala.util.Random.nextFloat
|
2019-12-13 07:30:20 -07:00
|
|
|
import scala.util.{ Failure, Success, Try }
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2019-12-03 14:30:53 -07:00
|
|
|
import lila.common.LilaStream
|
2016-04-01 11:50:57 -06:00
|
|
|
import lila.db.dsl._
|
2019-12-13 07:30:20 -07:00
|
|
|
import lila.game.{ Game, GameRepo, PgnDump, Player, Query }
|
2018-10-18 01:31:34 -06:00
|
|
|
import lila.user.{ User, UserRepo }
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2019-12-13 07:30:20 -07:00
|
|
|
final private class ExplorerIndexer(
|
2019-12-03 14:30:53 -07:00
|
|
|
gameRepo: GameRepo,
|
|
|
|
userRepo: UserRepo,
|
|
|
|
getBotUserIds: lila.user.GetBotIds,
|
2020-08-07 08:22:26 -06:00
|
|
|
ws: play.api.libs.ws.StandaloneWSClient,
|
2019-12-03 14:30:53 -07:00
|
|
|
internalEndpoint: InternalEndpoint
|
2020-06-24 03:37:18 -06:00
|
|
|
)(implicit
|
|
|
|
ec: scala.concurrent.ExecutionContext,
|
|
|
|
mat: akka.stream.Materializer
|
|
|
|
) {
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2019-12-13 07:30:20 -07:00
|
|
|
private val separator = "\n\n\n"
|
|
|
|
private val datePattern = "yyyy-MM-dd"
|
|
|
|
private val dateFormatter = DateTimeFormat forPattern datePattern
|
2020-08-16 07:27:35 -06:00
|
|
|
private val pgnDateFormat = DateTimeFormat forPattern "yyyy.MM.dd"
|
2016-06-12 03:14:14 -06:00
|
|
|
private val internalEndPointUrl = s"$internalEndpoint/import/lichess"
|
2016-02-08 08:42:33 -07:00
|
|
|
|
2016-02-09 03:27:12 -07:00
|
|
|
private def parseDate(str: String): Option[DateTime] =
|
|
|
|
Try(dateFormatter parseDateTime str).toOption
|
2016-02-09 03:07:35 -07:00
|
|
|
|
2020-05-05 22:11:15 -06:00
|
|
|
def apply(sinceStr: String): Funit =
|
|
|
|
getBotUserIds() flatMap { botUserIds =>
|
|
|
|
parseDate(sinceStr).fold(fufail[Unit](s"Invalid date $sinceStr")) { since =>
|
|
|
|
logger.info(s"Start indexing since $since")
|
|
|
|
val query =
|
|
|
|
Query.createdSince(since) ++
|
|
|
|
Query.rated ++
|
|
|
|
Query.finished ++
|
|
|
|
Query.turnsGt(8) ++
|
|
|
|
Query.noProvisional ++
|
|
|
|
Query.bothRatingsGreaterThan(1501)
|
2016-10-29 02:57:55 -06:00
|
|
|
|
2020-05-05 22:11:15 -06:00
|
|
|
gameRepo
|
|
|
|
.sortedCursor(query, Query.sortChronological)
|
|
|
|
.documentSource()
|
|
|
|
.via(LilaStream.logRate[Game]("fetch")(logger))
|
|
|
|
.mapAsyncUnordered(8) { makeFastPgn(_, botUserIds) }
|
|
|
|
.via(LilaStream.collect)
|
|
|
|
.via(LilaStream.logRate("index")(logger))
|
|
|
|
.grouped(50)
|
|
|
|
.map(_ mkString separator)
|
|
|
|
.mapAsyncUnordered(2) { pgn =>
|
|
|
|
ws.url(internalEndPointUrl).put(pgn).flatMap {
|
|
|
|
case res if res.status == 200 => funit
|
|
|
|
case res => fufail(s"Stop import because of status ${res.status}")
|
|
|
|
}
|
2019-12-03 14:30:53 -07:00
|
|
|
}
|
2020-05-05 22:11:15 -06:00
|
|
|
.toMat(Sink.ignore)(Keep.right)
|
2020-07-07 02:34:48 -06:00
|
|
|
.run()
|
2020-05-05 22:11:15 -06:00
|
|
|
.void
|
|
|
|
}
|
2016-02-09 03:07:35 -07:00
|
|
|
}
|
2016-02-08 22:00:49 -07:00
|
|
|
|
2020-05-05 22:11:15 -06:00
|
|
|
def apply(game: Game): Funit =
|
|
|
|
getBotUserIds() flatMap { botUserIds =>
|
|
|
|
makeFastPgn(game, botUserIds) map {
|
|
|
|
_ foreach flowBuffer.apply
|
|
|
|
}
|
2018-10-18 01:31:34 -06:00
|
|
|
}
|
2016-02-18 06:04:15 -07:00
|
|
|
|
|
|
|
private object flowBuffer {
|
2016-02-22 22:24:32 -07:00
|
|
|
private val max = 30
|
2016-02-18 06:04:15 -07:00
|
|
|
private val buf = scala.collection.mutable.ArrayBuffer.empty[String]
|
2017-10-21 14:06:14 -06:00
|
|
|
def apply(pgn: String): Unit = {
|
2016-02-18 06:04:15 -07:00
|
|
|
buf += pgn
|
2016-02-18 22:39:25 -07:00
|
|
|
val startAt = nowMillis
|
2016-02-18 06:04:15 -07:00
|
|
|
if (buf.size >= max) {
|
2019-12-03 14:30:53 -07:00
|
|
|
ws.url(internalEndPointUrl).put(buf mkString separator) andThen {
|
2016-02-18 06:04:15 -07:00
|
|
|
case Success(res) if res.status == 200 =>
|
2019-12-10 14:01:18 -07:00
|
|
|
lila.mon.explorer.index.time.record((nowMillis - startAt) / max)
|
|
|
|
lila.mon.explorer.index.count(true).increment(max)
|
2016-03-10 11:21:04 -07:00
|
|
|
case Success(res) =>
|
|
|
|
logger.warn(s"[${res.status}]")
|
2019-12-10 14:01:18 -07:00
|
|
|
lila.mon.explorer.index.count(false).increment(max)
|
2016-03-10 11:21:04 -07:00
|
|
|
case Failure(err) =>
|
2016-06-12 02:39:49 -06:00
|
|
|
logger.warn(s"$err", err)
|
2019-12-10 14:01:18 -07:00
|
|
|
lila.mon.explorer.index.count(false).increment(max)
|
2016-02-18 06:04:15 -07:00
|
|
|
}
|
2020-07-07 02:34:48 -06:00
|
|
|
buf.clear()
|
2016-02-18 06:04:15 -07:00
|
|
|
}
|
2016-02-08 22:00:49 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private def valid(game: Game) =
|
|
|
|
game.finished &&
|
|
|
|
game.rated &&
|
|
|
|
game.turns >= 10 &&
|
2016-02-10 09:55:01 -07:00
|
|
|
game.variant != chess.variant.FromPosition &&
|
2016-09-04 11:07:21 -06:00
|
|
|
!Game.isOldHorde(game)
|
2016-02-08 22:00:49 -07:00
|
|
|
|
|
|
|
private def stableRating(player: Player) = player.rating ifFalse player.provisional
|
|
|
|
|
2016-02-09 02:34:10 -07:00
|
|
|
// probability of the game being indexed, between 0 and 1
|
|
|
|
private def probability(game: Game, rating: Int) = {
|
|
|
|
import lila.rating.PerfType._
|
|
|
|
game.perfType ?? {
|
2019-12-13 07:30:20 -07:00
|
|
|
case Correspondence => 1
|
2017-11-29 10:58:08 -07:00
|
|
|
case Rapid | Classical if rating >= 2000 => 1
|
|
|
|
case Rapid | Classical if rating >= 1800 => 2 / 5f
|
2019-12-13 07:30:20 -07:00
|
|
|
case Rapid | Classical => 1 / 8f
|
|
|
|
case Blitz if rating >= 2000 => 1
|
|
|
|
case Blitz if rating >= 1800 => 1 / 4f
|
|
|
|
case Blitz => 1 / 15f
|
|
|
|
case Bullet if rating >= 2300 => 1
|
|
|
|
case Bullet if rating >= 2200 => 4 / 5f
|
|
|
|
case Bullet if rating >= 2000 => 1 / 4f
|
|
|
|
case Bullet if rating >= 1800 => 1 / 7f
|
|
|
|
case Bullet => 1 / 20f
|
2020-05-10 11:03:20 -06:00
|
|
|
case _ if rating >= 1600 => 1 // variant games
|
2019-12-13 07:30:20 -07:00
|
|
|
case _ => 1 / 2f // noob variant games
|
2016-02-09 02:34:10 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-13 07:30:20 -07:00
|
|
|
private def makeFastPgn(game: Game, botUserIds: Set[User.ID]): Fu[Option[String]] =
|
|
|
|
~(for {
|
|
|
|
whiteRating <- stableRating(game.whitePlayer)
|
|
|
|
blackRating <- stableRating(game.blackPlayer)
|
|
|
|
minPlayerRating = if (game.variant.exotic) 1400 else 1500
|
|
|
|
minAverageRating = if (game.variant.exotic) 1520 else 1600
|
|
|
|
if whiteRating >= minPlayerRating
|
|
|
|
if blackRating >= minPlayerRating
|
|
|
|
averageRating = (whiteRating + blackRating) / 2
|
|
|
|
if averageRating >= minAverageRating
|
2020-07-07 02:34:48 -06:00
|
|
|
if probability(game, averageRating) > nextFloat()
|
2019-12-13 07:30:20 -07:00
|
|
|
if !game.userIds.exists(botUserIds.contains)
|
|
|
|
if valid(game)
|
|
|
|
} yield gameRepo initialFen game flatMap { initialFen =>
|
|
|
|
userRepo.usernamesByIds(game.userIds) map { usernames =>
|
|
|
|
def username(color: chess.Color) =
|
|
|
|
game.player(color).userId flatMap { id =>
|
|
|
|
usernames.find(_.toLowerCase == id)
|
|
|
|
} orElse game.player(color).userId getOrElse "?"
|
|
|
|
val fenTags = initialFen.?? { fen =>
|
|
|
|
List(s"[FEN $fen]")
|
|
|
|
}
|
|
|
|
val timeControl = Tag.timeControl(game.clock.map(_.config)).value
|
|
|
|
val otherTags = List(
|
|
|
|
s"[LichessID ${game.id}]",
|
|
|
|
s"[Variant ${game.variant.name}]",
|
|
|
|
s"[TimeControl $timeControl]",
|
|
|
|
s"[White ${username(chess.White)}]",
|
|
|
|
s"[Black ${username(chess.Black)}]",
|
|
|
|
s"[WhiteElo $whiteRating]",
|
|
|
|
s"[BlackElo $blackRating]",
|
|
|
|
s"[Result ${PgnDump.result(game)}]",
|
|
|
|
s"[Date ${pgnDateFormat.print(game.createdAt)}]"
|
|
|
|
)
|
|
|
|
val allTags = fenTags ::: otherTags
|
|
|
|
s"${allTags.mkString("\n")}\n\n${game.pgnMoves.take(maxPlies).mkString(" ")}".some
|
|
|
|
}
|
|
|
|
})
|
2016-02-08 22:00:49 -07:00
|
|
|
|
2016-03-20 03:31:09 -06:00
|
|
|
private val logger = lila.log("explorer")
|
2016-02-08 08:42:33 -07:00
|
|
|
}
|