deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

more work on indexing games

pull/867/merge
Thibault Duplessis 2015-09-01 19:03:51 +02:00
parent 228d436b2b
commit acdf4c52c0
11 changed files with 99 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/views/base/topmenu.scala.html
View File

@ -41,7 +41,7 @@
<a href="@routes.UserAnalysis.index">@trans.analysis()</a>
<a href="@routes.Editor.index">@trans.boardEditor()</a>
<a href="@routes.Importer.importGame">@trans.importGame()</a>
<!-- <a href="@routes.Search.index()">@trans.advancedSearch()</a> -->
<a href="@routes.Search.index()">@trans.advancedSearch()</a>
</div>
</section>
</div>

2
modules/common/src/main/Steroids.scala
View File

@ -47,7 +47,7 @@ trait JodaTimeSteroids {
def getSeconds: Long = date.getMillis / 1000
def getDate: java.util.Date = date.toDate
}
implicit def dateTimeOrdering: Ordering[DateTime] = Ordering.fromLessThan(_ isBefore _)
implicit val dateTimeOrdering: Ordering[DateTime] = Ordering.fromLessThan(_ isBefore _)
}
trait ListSteroids {

19
modules/gameSearch/src/main/Env.scala
View File

@ -16,13 +16,12 @@ final class Env(
private val PaginatorMaxPerPage = config getInt "paginator.max_per_page"
private val ActorName = config getString "actor.name"
private val client = makeClient(Index(IndexName))
private lazy val client = makeClient(Index(IndexName))
private def converter(ids: Seq[String]) =
$find.byOrderedIds[lila.game.Game](ids)
lazy val api = new GameSearchApi(client)
lazy val paginator = new PaginatorBuilder[lila.game.Game, Query](
searchApi = ???,
searchApi = api,
maxPerPage = PaginatorMaxPerPage)
lazy val forms = new DataForm
@ -31,14 +30,8 @@ final class Env(
import lila.game.actorApi.{ InsertGame, FinishGame }
context.system.lilaBus.subscribe(self, 'finishGame)
def receive = {
case FinishGame(game, _, _) => // self ! InsertGame(game)
// case lila.analyse.actorApi.AnalysisReady(game, analysis) =>
// assessApi.onAnalysisReady(game, analysis)
// case lila.game.actorApi.FinishGame(game, whiteUserOption, blackUserOption) =>
// (whiteUserOption |@| blackUserOption) apply {
// case (whiteUser, blackUser) => boosting.check(game, whiteUser, blackUser) >>
// assessApi.onGameReady(game, whiteUser, blackUser)
// }
case FinishGame(game, _, _) => self ! InsertGame(game)
case InsertGame(game) => api store game
}
}), name = ActorName)
@ -46,7 +39,7 @@ final class Env(
import akka.pattern.ask
private implicit def timeout = makeTimeout minutes 60
def process = {
case "game" :: "search" :: "reset" :: Nil => fuccess("done")
case "game" :: "search" :: "reset" :: Nil => api.reset inject "done"
}
}
}

79
modules/gameSearch/src/main/GameSearchApi.scala 100644
View File

@ -0,0 +1,79 @@
package lila.gameSearch
import lila.common.PimpedJson._
import lila.game.actorApi._
import lila.game.{ Game, GameRepo }
import lila.search._
import play.api.libs.json._
final class GameSearchApi(client: ESClient) extends SearchReadApi[Game, Query] {
def search(query: Query, from: From, size: Size) =
client.search(query, from, size) flatMap { res =>
import lila.db.api.$find
import lila.game.tube.gameTube
$find.byOrderedIds[lila.game.Game](res.ids)
}
def count(query: Query) =
client.count(query) map (_.count)
def store(game: Game) = storable(game) ?? {
GameRepo isAnalysed game.id flatMap { analysed =>
client.store(Id(game.id), toDoc(game, analysed))
}
}
private def storable(game: Game) = (game.finished || game.imported) && game.playedTurns > 4
private def toDoc(game: Game, analysed: Boolean) = Json.obj(
Fields.status -> (game.status match {
case s if s.is(_.Timeout) => chess.Status.Resign
case s if s.is(_.NoStart) => chess.Status.Resign
case s => game.status
}).id,
Fields.turns -> math.ceil(game.turns.toFloat / 2),
Fields.rated -> game.rated,
Fields.variant -> game.variant.id,
Fields.uids -> game.userIds.toArray.some.filterNot(_.isEmpty),
Fields.winner -> (game.winner flatMap (_.userId)),
Fields.winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)),
Fields.averageRating -> game.averageUsersRating,
Fields.ai -> game.aiLevel,
Fields.date -> (lila.search.Date.formatter print game.createdAt),
Fields.duration -> game.estimateTotalTime,
Fields.opening -> (game.opening map (_.code.toLowerCase)),
Fields.analysed -> analysed,
Fields.whiteUser -> game.whitePlayer.userId,
Fields.blackUser -> game.blackPlayer.userId
).noNull
def reset = client.putMapping >> {
import lila.db.api._
import lila.game.tube.gameTube
var nb = 0
var nbSkipped = 0
var started = nowMillis
for {
size <- $count($select.all)
batchSize = 1000
limit = 20 * 1000
_ <- $enumerate.bulk[Option[Game]]($query.all, batchSize, limit) { gameOptions =>
val games = gameOptions.flatten filter storable
val nbGames = games.size
(GameRepo filterAnalysed games.map(_.id).toSeq flatMap { analysedIds =>
client.storeBulk(games map { g =>
Id(g.id) -> toDoc(g, analysedIds(g.id))
})
}) >>- {
nb = nb + nbGames
nbSkipped = nbSkipped + gameOptions.size - nbGames
val perS = (batchSize * 1000) / math.max(1, (nowMillis - started))
started = nowMillis
loginfo("[game search] Indexed %d of %d, skipped %d, at %d/s".format(nb, size, nbSkipped, perS))
}
}
} yield ()
}
}

32
modules/gameSearch/src/main/Indexer.scala
View File

@ -94,36 +94,4 @@
// add alias indexName on tempIndexName
// }.await
// }
// private def storable(game: lila.game.Game) =
// (game.finished || game.imported) && game.playedTurns > 4
// private def store(inIndex: String, game: lila.game.Game, hasAnalyse: Boolean) = {
// import Fields._
// index into s"$inIndex/$typeName" fields {
// List(
// status -> (game.status match {
// case s if s.is(_.Timeout) => chess.Status.Resign
// case s if s.is(_.NoStart) => chess.Status.Resign
// case s => game.status
// }).id.some,
// turns -> math.ceil(game.turns.toFloat / 2).some,
// rated -> game.rated.some,
// variant -> game.variant.id.some,
// uids -> game.userIds.toArray.some.filterNot(_.isEmpty),
// winner -> (game.winner flatMap (_.userId)),
// winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)).some,
// averageRating -> game.averageUsersRating,
// ai -> game.aiLevel,
// date -> (ElasticSearch.Date.formatter print game.createdAt).some,
// duration -> game.estimateTotalTime.some,
// opening -> (game.opening map (_.code.toLowerCase)),
// analysed -> hasAnalyse.some,
// whiteUser -> game.whitePlayer.userId,
// blackUser -> game.blackPlayer.userId
// ).collect {
// case (key, Some(value)) => key -> value
// }: _*
// } id game.id
// }
// }

44
modules/gameSearch/src/main/Query.scala
View File

@ -4,7 +4,7 @@ import chess.{ Mode, Status, Openings }
import org.joda.time.DateTime
import lila.rating.RatingRange
import lila.search.{ ElasticSearch, Range }
import lila.search.Range
case class Query(
user1: Option[String] = None,
@ -26,8 +26,6 @@ case class Query(
whiteUser: Option[String] = None,
blackUser: Option[String] = None) {
import Fields._
def nonEmpty =
user1.nonEmpty ||
user2.nonEmpty ||
@ -43,46 +41,6 @@ case class Query(
opening.nonEmpty ||
date.nonEmpty ||
duration.nonEmpty
// def searchDef(from: Int = 0, size: Int = 10) =
// search in indexType query makeQuery sort sorting.definition start from size size
// def countDef = count from indexType query makeQuery
// private lazy val makeQuery = filteredQuery query matchall filter {
// List(
// usernames map { termFilter(Fields.uids, _) },
// toFilters(winner, Fields.winner),
// toFilters(winnerColor, Fields.winnerColor),
// turns filters Fields.turns,
// averageRating filters Fields.averageRating,
// duration map (60 *) filters Fields.duration,
// date map ElasticSearch.Date.formatter.print filters Fields.date,
// hasAiFilters,
// (hasAi | true).fold(aiLevel filters Fields.ai, Nil),
// toFilters(variant, Fields.variant),
// toFilters(rated, Fields.rated),
// toFilters(opening, Fields.opening),
// toFilters(status, Fields.status),
// toFilters(analysed, Fields.analysed),
// toFilters(whiteUser, Fields.whiteUser),
// toFilters(blackUser, Fields.blackUser)
// ).flatten match {
// case Nil => matchAllFilter
// case filters => must(filters: _*)
// }
// }
def usernames = List(user1, user2).flatten
// private def hasAiFilters = hasAi.toList map { a =>
// a.fold(existsFilter(Fields.ai), missingFilter(Fields.ai))
// }
// private def toFilters(query: Option[_], name: String) = query.toList map {
// case s: String => termFilter(name, s.toLowerCase)
// case x => termFilter(name, x)
// }
}
object Query {

13
modules/gameSearch/src/main/Sorting.scala
View File

@ -1,13 +1,6 @@
package lila.gameSearch
import lila.db.api.SortOrder
case class Sorting(f: String, order: String) {
// def definition =
// field sort (Sorting.fieldKeys contains f).fold(f, Sorting.default.f) order
// (order.toLowerCase == "asc").fold(SortOrder.Ascending, SortOrder.Descending)
}
case class Sorting(f: String, order: String)
object Sorting {
@ -16,9 +9,7 @@ object Sorting {
Fields.turns -> "Moves",
Fields.averageRating -> "Average Rating")
def fieldKeys = fields map (_._1)
val orders = List(SortOrder.Descending, SortOrder.Ascending) map { s => s.toString -> s.toString }
val orders = List("desc", "asc") map { s => s -> s }
val default = Sorting(Fields.date, "desc")
}

1
modules/search/src/main/ESClient.scala
View File

@ -48,6 +48,7 @@ final class ESClientHttp(endpoint: String, index: Index) extends ESClient {
private def HTTP[D: Writes, R](url: String, data: D, read: String => R): Fu[R] =
WS.url(s"$endpoint/$url").post(Json toJson data) flatMap {
case res if res.status == 200 => fuccess(read(res.body))
case res if res.status == 500 => fufail(s"$url ${res.status}")
case res => fufail(s"$url ${res.status} ${res.body}")
}
private def HTTP(url: String, data: JsObject): Funit = HTTP(url, data, _ => ())

18
modules/search/src/main/ElasticSearch.scala
View File

@ -1,18 +0,0 @@
package lila.search
import play.api.libs.json._
object ElasticSearch {
def decomposeTextQuery(text: String): List[String] =
text.trim.toLowerCase.replace("+", " ").split(" ").toList
object Date {
import org.joda.time.format.{ DateTimeFormat, DateTimeFormatter }
val format = "YYYY-MM-dd HH:mm:ss"
val formatter: DateTimeFormatter = DateTimeFormat forPattern format
}
}

6
modules/search/src/main/Range.scala
View File

@ -2,12 +2,6 @@ package lila.search
final class Range[A] private (val a: Option[A], val b: Option[A]) {
// def filters(name: String) = a.fold(b.toList map { bb => rangeFilter(name) lte bb.toString }) { aa =>
// b.fold(List(rangeFilter(name) gte aa.toString)) { bb =>
// List(rangeFilter(name) gte aa.toString lte bb.toString)
// }
// }
def map[B](f: A => B) = new Range(a map f, b map f)
def nonEmpty = a.nonEmpty || b.nonEmpty

9
modules/search/src/main/package.scala
View File

@ -1,3 +1,10 @@
package lila
package object search extends PackageObject with WithPlay
package object search extends PackageObject with WithPlay {
object Date {
import org.joda.time.format.{ DateTimeFormat, DateTimeFormatter }
val format = "YYYY-MM-dd HH:mm:ss"
val formatter: DateTimeFormatter = DateTimeFormat forPattern format
}
}