deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

automatically detect relay format - WIP

relayDetect
Thibault Duplessis 2018-09-01 11:48:30 +02:00
parent 60908826bc
commit 4fc7adc1a5
9 changed files with 164 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/views/relay/inForm.scala.html
View File

@ -19,9 +19,6 @@ help = Html("Feature on /broadcast - for admins only").some) { field =>
@base.form.group(form("syncUrl"), Html("Source URL")) { field =>
@base.form.input(field, typ = "url", required = true)
}
@base.form.group(form("syncType"), Html("Source format")) { field =>
@base.form.select(field, lila.relay.RelayForm.syncTypes)
}
<div>
@base.form.group(form("startsAt"), Html("Start date <strong>UTC</strong>"), help = Html("Optional, if you know when the event starts").some, half = true) { field =>
@base.form.flatpickr(field)

2
build.sbt
View File

@ -290,7 +290,7 @@ lazy val study = module("study", Seq(
)
lazy val relay = module("relay", Seq(common, study)).settings(
libraryDependencies ++= provided(play.api, reactivemongo.driver)
libraryDependencies ++= Seq(scalaUri) ++ provided(play.api, reactivemongo.driver)
)
lazy val studySearch = module("studySearch", Seq(common, hub, study, search)).settings(

9
modules/relay/src/main/BSONHandlers.scala
View File

@ -11,14 +11,7 @@ object BSONHandlers {
import Relay.Sync
import Sync.Upstream
private implicit val SourceUpstreamHandler = new lila.db.BSON[Upstream] {
def reads(r: lila.db.BSON.Reader) = r.str("k") match {
case "dgt-one" => Upstream.DgtOneFile(r str "url")
case "dgt-many" => Upstream.DgtManyFiles(r str "url")
case k => sys error s"Invalid relay source upstream $k"
}
def writes(w: lila.db.BSON.Writer, u: Upstream) = $doc("k" -> u.key, "url" -> u.url)
}
implicit val upstreamHandler = Macros.handler[Upstream]
import SyncLog.Event
implicit val syncLogEventHandler = Macros.handler[Event]

3
modules/relay/src/main/Env.scala
View File

@ -8,6 +8,7 @@ final class Env(
config: Config,
db: lila.db.Env,
studyEnv: lila.study.Env,
asyncCache: lila.memo.AsyncCache.Builder,
system: ActorSystem
) {
@ -47,6 +48,7 @@ final class Env(
private val fetch = system.actorOf(Props(new RelayFetch(
sync = sync,
api = api,
formatApi = new RelayFormatApi(asyncCache),
chapterRepo = studyEnv.chapterRepo
)))
@ -66,6 +68,7 @@ object Env {
db = lila.db.Env.current,
config = lila.common.PlayApp loadConfig "relay",
studyEnv = lila.study.Env.current,
asyncCache = lila.memo.Env.current.asyncCache,
system = lila.common.PlayApp.system
)
}

8
modules/relay/src/main/Relay.scala
View File

@ -98,13 +98,7 @@ object Relay {
}
object Sync {
sealed abstract class Upstream(val key: String, val url: String, val heavy: Boolean) {
override def toString = s"$key $url"
}
object Upstream {
case class DgtOneFile(fileUrl: String) extends Upstream("dgt-one", fileUrl, false)
case class DgtManyFiles(dirUrl: String) extends Upstream("dgt-many", dirUrl, true)
}
case class Upstream(url: String) extends AnyVal
}
case class WithStudy(relay: Relay, study: Study)

109
modules/relay/src/main/RelayFetch.scala
View File

@ -1,19 +1,24 @@
package lila.relay
import akka.actor._
import com.github.blemale.scaffeine.{ Cache, Scaffeine }
import io.lemonlabs.uri.Url
import org.joda.time.DateTime
import play.api.libs.json._
import play.api.libs.ws.{ WS, WSResponse }
import play.api.Play.current
import scala.concurrent.duration._
import chess.format.pgn.Tags
import lila.base.LilaException
import lila.study.MultiPgn
import lila.tree.Node.Comments
import Relay.Sync.Upstream
private final class RelayFetch(
sync: RelaySync,
api: RelayApi,
formatApi: RelayFormatApi,
chapterRepo: lila.study.ChapterRepo
) extends Actor {
@ -57,7 +62,7 @@ private final class RelayFetch(
// no writing the relay; only reading!
def processRelay(relay: Relay): Fu[Relay] =
if (!relay.sync.playing) fuccess(relay.withSync(_.play))
else RelayFetch(relay) flatMap { games =>
else doProcess(relay) flatMap { games =>
sync(relay, games)
.chronometer.mon(_.relay.sync.duration.each).result
.withTimeout(1 second, SyncResult.Timeout)(context.system) map { res =>
@ -95,53 +100,80 @@ private final class RelayFetch(
(if (r.sync.log.alwaysFails) fuccess(60) else (r.sync.delay match {
case Some(delay) => fuccess(delay)
case None => api.getNbViewers(r) map { nb =>
if (r.sync.upstream.heavy) (18 - nb) atLeast 8
else (13 - nb) atLeast 5
(18 - nb) atLeast 8
}
})) map { seconds =>
r.withSync(_.copy(nextAt = DateTime.now plusSeconds {
seconds atLeast { if (r.sync.log.isOk) 5 else 15 }
} some))
}
}
private object RelayFetch {
import RelayFetch.GamesSeenBy
import Relay.Sync.Upstream
case class GamesSeenBy(games: Fu[RelayGames], seenBy: Set[Relay.Id])
def apply(relay: Relay): Fu[RelayGames] =
private def doProcess(relay: Relay): Fu[RelayGames] =
cache getIfPresent relay.sync.upstream match {
case Some(GamesSeenBy(games, seenBy)) if !seenBy(relay.id) =>
cache.put(relay.sync.upstream, GamesSeenBy(games, seenBy + relay.id))
games
case x =>
val games = doFetch(relay.sync.upstream, maxChapters(relay))
val games = doFetch(relay.sync.upstream, RelayFetch.maxChapters(relay))
cache.put(relay.sync.upstream, GamesSeenBy(games, Set(relay.id)))
games
}
def maxChapters(relay: Relay) =
lila.study.Study.maxChapters * (if (relay.official) 2 else 1)
// private def dgtManyFiles(dir: String, max: Int, format: DgtMany): Fu[MultiPgn] = {
// val indexFile = s"$dir/${format.indexFile}"
// httpGet(indexFile) flatMap {
// case res if res.status == 200 => roundReads reads res.json match {
// case JsError(err) => fufail(err.toString)
// case JsSuccess(round, _) => round.pairings.zipWithIndex.map {
// case (pairing, i) =>
// val number = i + 1
// val gameUrl = s"$dir/${format.gameFile(number)}"
// httpGet(gameUrl).flatMap {
// case res if res.status == 200 => fuccess(number -> format.toPgn(res, pairing))
// case res => fufail(s"[${res.status}] $gameUrl")
// }
// }.sequenceFu map { results =>
// MultiPgn(results.sortBy(_._1).map(_._2).toList)
// }
// }
// case res => fufail(s"[${res.status}] $indexFile")
// }
// }
import com.github.blemale.scaffeine.{ Cache, Scaffeine }
private val cache: Cache[Upstream, GamesSeenBy] = Scaffeine()
.expireAfterWrite(30.seconds)
.build[Upstream, GamesSeenBy]
private def doFetch(upstream: Upstream, max: Int): Fu[RelayGames] = (upstream match {
case Upstream.DgtOneFile(file) => dgtOneFile(file, max)
case Upstream.DgtManyFiles(dir) =>
dgtManyFiles(dir, max, DgtMany.RoundPgn) recoverWith {
case _: lila.base.LilaException => dgtManyFiles(dir, max, DgtMany.Indexjson)
private def doFetch(upstream: Upstream, max: Int): Fu[RelayGames] =
formatApi.get(upstream.url) flatMap {
_.fold[Fu[MultiPgn]](fufail("Cannot find any DGT compatible files")) {
case RelayFormat.SingleFile(doc) => httpGet(doc.url) map { body =>
doc.format match {
// all games in a single PGN file
case RelayFormat.DocFormat.Pgn => MultiPgn.split(body, max)
// maybe a single JSON game? Why not
case RelayFormat.DocFormat.Json => MultiPgn(List(RelayFetch.jsonToPgn(body)))
}
}
case f: RelayFormat.ManyFiles => ???
}
}) flatMap multiPgnToGames.apply
} flatMap RelayFetch.multiPgnToGames.apply
private def dgtOneFile(file: String, max: Int): Fu[MultiPgn] =
httpGet(file).flatMap {
case res if res.status == 200 => fuccess(MultiPgn.split(res.body, max))
private def httpGet(url: Url): Fu[String] =
WS.url(url.toString).withRequestTimeout(4.seconds.toMillis).get().flatMap {
case res if res.status == 200 => fuccess(res.body)
case res => fufail(s"[${res.status}]")
}
}
private object RelayFetch {
case class GamesSeenBy(games: Fu[RelayGames], seenBy: Set[Relay.Id])
def maxChapters(relay: Relay) =
lila.study.Study.maxChapters * (if (relay.official) 2 else 1)
private object DgtJson {
case class PairingPlayer(fname: Option[String], mname: Option[String], lname: Option[String], title: Option[String]) {
@ -172,7 +204,7 @@ private object RelayFetch {
private sealed abstract class DgtMany(val indexFile: String, val gameFile: Int => String, val toPgn: (WSResponse, RoundJsonPairing) => String)
private object DgtMany {
case object RoundPgn extends DgtMany("round.json", n => s"game-$n.pgn", (r, _) => r.body)
case object Indexjson extends DgtMany("index.json", n => s"game-$n.json", {
case object Indexjson extends DgtMany("index.json", n => s"game-$n.pgn", {
case (res, pairing) => res.json.validate[GameJson] match {
case JsSuccess(game, _) =>
val moves = game.moves.map(_ split ' ') map { move =>
@ -187,29 +219,18 @@ private object RelayFetch {
})
}
private def dgtManyFiles(dir: String, max: Int, format: DgtMany): Fu[MultiPgn] = {
val indexFile = s"$dir/${format.indexFile}"
httpGet(indexFile) flatMap {
case res if res.status == 200 => roundReads reads res.json match {
case JsError(err) => fufail(err.toString)
case JsSuccess(round, _) => round.pairings.zipWithIndex.map {
case (pairing, i) =>
val number = i + 1
val gameUrl = s"$dir/${format.gameFile(number)}"
httpGet(gameUrl).flatMap {
case res if res.status == 200 => fuccess(number -> format.toPgn(res, pairing))
case res => fufail(s"[${res.status}] $gameUrl")
}
}.sequenceFu map { results =>
MultiPgn(results.sortBy(_._1).map(_._2).toList)
}
}
case res => fufail(s"[${res.status}] $indexFile")
}
private def jsonToPgn(str: String, extraTags: Tags = Tags.empty) = Json.parse(str).validate[GameJson] match {
case JsSuccess(game, _) =>
val moves = game.moves.map(_ split ' ') map { move =>
chess.format.pgn.Move(
san = ~move.headOption,
secondsLeft = move.lift(1).map(_.takeWhile(_.isDigit)) flatMap parseIntOption
)
} mkString " "
s"${extraTags}\n\n$moves"
case JsError(err) => ""
}
private def httpGet(url: String) = WS.url(url).withRequestTimeout(4.seconds.toMillis).get()
private object multiPgnToGames {
import scala.util.{ Try, Success, Failure }

13
modules/relay/src/main/RelayForm.scala
View File

@ -12,16 +12,10 @@ object RelayForm {
import lila.common.Form.UTCDate._
val syncTypes = List(
"dgt-one" -> "DGT (traditional): all games in a single file",
"dgt-many" -> "DGT (new): one file per game"
)
val form = Form(mapping(
"name" -> nonEmptyText(minLength = 3, maxLength = 80),
"description" -> nonEmptyText(minLength = 3, maxLength = 4000),
"official" -> boolean,
"syncType" -> text.verifying(syncTypes.map(_._1).contains _),
"syncUrl" -> nonEmptyText,
"startsAt" -> optional(utcDate),
"throttle" -> optional(number(min = 0, max = 60))
@ -35,7 +29,6 @@ object RelayForm {
name: String,
description: String,
official: Boolean,
syncType: String,
syncUrl: String,
startsAt: Option[DateTime],
throttle: Option[Int]
@ -56,10 +49,7 @@ object RelayForm {
)
def makeSync = Relay.Sync(
upstream = syncType match {
case "dgt-one" => Relay.Sync.Upstream.DgtOneFile(cleanUrl)
case _ => Relay.Sync.Upstream.DgtManyFiles(cleanUrl)
},
upstream = Relay.Sync.Upstream(cleanUrl),
until = none,
nextAt = none,
delay = throttle,
@ -87,7 +77,6 @@ object RelayForm {
name = relay.name,
description = relay.description,
official = relay.official,
syncType = relay.sync.upstream.key,
syncUrl = relay.sync.upstream.url,
startsAt = relay.startsAt,
throttle = relay.sync.delay

91
modules/relay/src/main/RelayFormat.scala 100644
View File

@ -0,0 +1,91 @@
package lila.relay
import io.lemonlabs.uri._
import play.api.libs.json._
import play.api.libs.ws.{ WS, WSResponse }
import play.api.Play.current
import scala.concurrent.duration._
import lila.memo.AsyncCache
import lila.study.MultiPgn
private final class RelayFormatApi(
asyncCache: AsyncCache.Builder
) {
import RelayFormat._
def get(url: String): Fu[Option[RelayFormat]] = cache get Url.parse(url.pp).pp
private def guessFormat(url: Url): Fu[Option[RelayFormat]] = {
def guessSingleFile: Fu[Option[RelayFormat]] =
lila.common.Future.find(List(
url.some,
!url.path.parts.contains(mostCommonSingleFileName) option addPart(url, mostCommonSingleFileName)
).flatten.distinct)(looksLikePgn) map2 { (u: Url) =>
SingleFile(pgnDoc(u))
}
def guessManyFiles: Fu[Option[RelayFormat]] =
lila.common.Future.find(
List(url) ::: mostCommonIndexNames.filterNot(url.path.parts.contains).map(addPart(url, _))
)(looksLikeJson) map2 { (index: Url) =>
ManyFiles(index, i => pgnDoc(replaceLastPart(index, s"game-$i.pgn")))
}
guessSingleFile orElse guessManyFiles
} thenPp
private val cache = asyncCache.multi[Url, Option[RelayFormat]](
name = "relayFormat",
f = guessFormat,
expireAfter = _.ExpireAfterAccess(1 hour)
)
}
private sealed trait RelayFormat
private object RelayFormat {
sealed trait DocFormat
object DocFormat {
case object Json extends DocFormat
case object Pgn extends DocFormat
}
case class Doc(url: Url, format: DocFormat)
def jsonDoc(url: Url) = Doc(url, DocFormat.Json)
def pgnDoc(url: Url) = Doc(url, DocFormat.Pgn)
case class SingleFile(doc: Doc) extends RelayFormat
case class ManyFiles(jsonIndex: Url, game: Int => Doc) extends RelayFormat
def httpGet(url: Url): Fu[Option[String]] =
WS.url(url.toString.pp("httpGet")).withRequestTimeout(4.seconds.toMillis).get().map {
case res if res.status == 200 => res.body.some
case _ => none
}
def looksLikePgn(body: String): Boolean = MultiPgn.split(body, 1).value.headOption ?? { pgn =>
lila.study.PgnImport(pgn, Nil).isSuccess
}
def looksLikePgn(url: Url): Fu[Boolean] = httpGet(url).map { _ exists looksLikePgn }
def looksLikeJson(body: String): Boolean = Json.parse(body) != JsNull
def looksLikeJson(url: Url): Fu[Boolean] = httpGet(url).map { _ exists looksLikeJson }
def addPart(url: Url, part: String) = url.withPath(url.path addPart part)
def replaceLastPart(url: Url, withPart: String) =
if (url.path.isEmpty) addPart(url, withPart)
else url.withPath {
url.path.withParts {
url.path.parts.init :+ withPart
}
}
val mostCommonSingleFileName = "games.pgn"
val mostCommonIndexNames = List("round.json", "index.json")
}

1
project/Dependencies.scala
View File

@ -42,6 +42,7 @@ object Dependencies {
val specs2 = "org.specs2" %% "specs2-core" % "4.0.2" % "test"
val specs2Scalaz = "org.specs2" %% "specs2-scalaz" % "4.0.2" % "test"
val stm = "org.scala-stm" %% "scala-stm" % "0.8"
val scalaUri = "io.lemonlabs" %% "scala-uri" % "1.2.0"
object reactivemongo {
val version = "0.12.3"