deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

Merge branch 'studyFlatChapter' 

* studyFlatChapter:
  parallelize study chapter migration
  hack the study multiboard - TODO really fix it
  fix tests
  study flat chapter WIP
  flat study chapter storage WIP
  flat study WIP
  study flat tree WIP
  study flat chapters WIP
pull/8077/head
Thibault Duplessis 2021-02-02 19:17:56 +01:00
parent c316938eda 93165b1927
commit d6a53fb6d3
19 changed files with 365 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
bin/mongodb/study-flat-chapter-migrate.js 100644
View File

@ -0,0 +1,82 @@
print(`parallelism: ${parallelism}, instance: ${instance}`);
const idChars = 'abcdefghyjklmnopqrstuvwxyzABCDEFGHYJKLMNOPQRSTUVWXYZ0123456789'.split('');
const sliceSize = idChars.length / parallelism;
const charSlice = idChars.slice(sliceSize * (instance -1), sliceSize * instance);
const firstCharRegex = new RegExp('^[' + charSlice.join('') + ']');
print(firstCharRegex);
const coll = db.study_chapter_flat;
coll.drop();
coll.createIndex({studyId:1,order:1})
function flattenEach(parentPath, node, nodes) {
const path = parentPath + encodePath(node.i || '');
if (path.length > 400 * 2) throw 'Too deep!';
if (node.n.length > 1) node.o = node.n.map(child => child.i);
node.n.forEach(child => flattenEach(path, child, nodes));
delete node.n;
delete node.i;
nodes[path] = node;
}
const dotRegex = /\./g;
const dollarRegex = /\$/g;
function encodePath(path) {
return path ? path
.replace(dotRegex, String.fromCharCode(144))
.replace(dollarRegex, String.fromCharCode(145)) : '_';
}
let i = 0;
let lastAt = Date.now();
let sumSizeFrom = 0;
let sumSizeTo = 0;
let sumMoves = 0;
let tooBigNb = 0;
let tooDeepNb = 0;
let batch = [];
const batchSize = 1000;
const totalNb = db.study_chapter_backup.count() / parallelism;
db.study_chapter_backup.find({_id:firstCharRegex}).forEach(c => {
try {
sumSizeFrom += Object.bsonsize(c);
} catch (e) {
tooDeepNb++;
}
const tree = {};
try {
flattenEach('', c.root, tree);
} catch (e) {
print(`ERROR ${c._id} ${e}`);
return;
}
c.root = tree;
sumSizeTo += Object.bsonsize(c);
const nbMoves = Object.keys(c.root).length;
if (nbMoves > 3000) tooBigNb++;
else batch.push(c);
i++;
sumMoves += nbMoves;
if (i % batchSize == 0) {
coll.insertMany(batch, {
ordered: false,
writeConcern: { w: 0, j: false }
});
batch = [];
const at = Date.now();
const perSecond = Math.round(batchSize / (at - lastAt) * 1000);
const percent = 100 * i / totalNb;
const minutesLeft = Math.round((totalNb - i) / perSecond / 60);
print(`${i} ${percent.toFixed(2)}% ${perSecond}/s ETA ${minutesLeft} minutes | size:${(sumSizeFrom / batchSize).toFixed(0)}->${(sumSizeTo / batchSize).toFixed(0)} moves:${(sumMoves / batchSize).toFixed(0)} big:${tooBigNb} deep:${tooDeepNb}`);
lastAt = at;
sumSizeFrom = 0;
sumSizeTo = 0;
sumMoves = 0;
tooBigNb = 0;
tooDeepNb = 0;
}
});

4
modules/challenge/src/test/JoinerTest.scala
View File

@ -19,7 +19,7 @@ final class JoinerTest extends Specification {
destUser = None,
rematchOf = None
)
Joiner.createGame(challenge, None, None, None) must beLike { case g: Game =>
ChallengeJoiner.createGame(challenge, None, None, None) must beLike { case g: Game =>
g.chess.startedAtTurn must_== 0
}
}
@ -35,7 +35,7 @@ final class JoinerTest extends Specification {
destUser = None,
rematchOf = None
)
Joiner.createGame(challenge, None, None, None) must beLike { case g: Game =>
ChallengeJoiner.createGame(challenge, None, None, None) must beLike { case g: Game =>
g.chess.startedAtTurn must_== 6
}
}

9
modules/common/src/main/mon.scala
View File

@ -597,6 +597,15 @@ object mon {
def request(hit: Boolean) = counter("fishnet.http.acquire").withTag("hit", hit)
}
}
object study {
object tree {
val read = timer("study.tree.read").withoutTags()
val write = timer("study.tree.write").withoutTags()
}
object sequencer {
val chapterTime = timer("study.sequencer.chapter.time").withoutTags()
}
}
object api {
val userGames = counter("api.cost").withTag("endpoint", "userGames")
val users = counter("api.cost").withTag("endpoint", "users")

6
modules/db/src/main/CollExt.scala
View File

@ -198,6 +198,12 @@ trait CollExt { self: dsl with QueryBuilderExt =>
def unsetField(selector: Bdoc, field: String, multi: Boolean = false) =
coll.update.one(selector, $unset(field), multi = multi)
def updateOrUnsetField[V: BSONWriter](selector: Bdoc, field: String, value: Option[V]): Fu[Int] =
value match {
case None => unsetField(selector, field).dmap(_.n)
case Some(v) => updateField(selector, field, v).dmap(_.n)
}
def fetchUpdate[D: BSONDocumentHandler](selector: Bdoc)(update: D => Bdoc): Funit =
one[D](selector) flatMap {
_ ?? { doc =>

2
modules/relay/src/main/RelaySync.scala
View File

@ -82,7 +82,7 @@ final private class RelaySync(
toMainline = true
)(who) >> chapterRepo.setRelayPath(chapter.id, path)
} >> newNode.?? { node =>
lila.common.Future.fold(node.mainline)(Position(chapter, path).ref) { case (position, n) =>
lila.common.Future.fold(node.mainline.toList)(Position(chapter, path).ref) { case (position, n) =>
studyApi.addNode(
studyId = study.id,
position = position,

159
modules/study/src/main/BSONHandlers.scala
View File

@ -9,6 +9,7 @@ import reactivemongo.api.bson._
import scala.util.Success
import scala.util.Try
import lila.common.Iso
import lila.db.BSON
import lila.db.BSON.{ Reader, Writer }
import lila.db.dsl._
@ -16,8 +17,6 @@ import lila.tree.Eval
import lila.tree.Eval.Score
import lila.tree.Node.{ Comment, Comments, Gamebook, Shape, Shapes }
import lila.common.Iso
object BSONHandlers {
import Chapter._
@ -159,92 +158,86 @@ object BSONHandlers {
)
}
implicit val ChildrenBSONHandler: BSONHandler[Node.Children] = tryHandler[Node.Children](
{ case arr: BSONArray =>
Try {
Node.Children(arr.values.view.flatMap(NodeBSONHandler.readOpt).toVector)
}
},
children =>
BSONArray(children.nodes map { n =>
NodeBSONHandler.writeTry(n).get
})
)
implicit lazy val NodeBSONHandler: BSON[Node] = new BSON[Node] {
def reads(r: Reader) =
Node(
id = r.get[UciCharPair]("i"),
ply = r int "p",
move = WithSan(r.get[Uci]("u"), r.str("s")),
fen = r.get[FEN]("f"),
check = r boolD "c",
shapes = r.getO[Shapes]("h") | Shapes.empty,
comments = r.getO[Comments]("co") | Comments.empty,
gamebook = r.getO[Gamebook]("ga"),
glyphs = r.getO[Glyphs]("g") | Glyphs.empty,
score = r.getO[Score]("e"),
crazyData = r.getO[Crazyhouse.Data]("z"),
clock = r.getO[Centis]("l"),
children =
try {
r.get[Node.Children]("n")
} catch {
case _: StackOverflowError =>
logger.warn(s"study ChildrenBSONHandler StackOverflowError")
Node.emptyChildren
},
forceVariation = r boolD "fv"
)
def writes(w: Writer, s: Node) =
$doc(
"i" -> s.id,
"p" -> s.ply,
"u" -> s.move.uci,
"s" -> s.move.san,
"f" -> s.fen,
"c" -> w.boolO(s.check),
"h" -> s.shapes.value.nonEmpty.option(s.shapes),
"co" -> s.comments.value.nonEmpty.option(s.comments),
"ga" -> s.gamebook,
"g" -> s.glyphs.nonEmpty,
"e" -> s.score,
"l" -> s.clock,
"z" -> s.crazyData,
"n" -> (if (s.ply < Node.MAX_PLIES) s.children else Node.emptyChildren),
"fv" -> w.boolO(s.forceVariation)
)
def readNode(doc: Bdoc, id: UciCharPair): Node = {
import Node.BsonFields._
val r = new Reader(doc)
Node(
id = id,
ply = r int ply,
move = WithSan(r.get[Uci](uci), r.str(san)),
fen = r.get[FEN](fen),
check = r boolD check,
shapes = r.getO[Shapes](shapes) | Shapes.empty,
comments = r.getO[Comments](comments) | Comments.empty,
gamebook = r.getO[Gamebook](gamebook),
glyphs = r.getO[Glyphs](glyphs) | Glyphs.empty,
score = r.getO[Score](score),
crazyData = r.getO[Crazyhouse.Data](crazy),
clock = r.getO[Centis](clock),
children = Node.emptyChildren,
forceVariation = r boolD forceVariation
)
}
def writeNode(n: Node) = {
import Node.BsonFields._
val w = new Writer
$doc(
ply -> n.ply,
uci -> n.move.uci,
san -> n.move.san,
fen -> n.fen,
check -> w.boolO(n.check),
shapes -> n.shapes.value.nonEmpty.option(n.shapes),
comments -> n.comments.value.nonEmpty.option(n.comments),
gamebook -> n.gamebook,
glyphs -> n.glyphs.nonEmpty,
score -> n.score,
clock -> n.clock,
crazy -> n.crazyData,
forceVariation -> w.boolO(n.forceVariation),
order -> {
(n.children.nodes.sizeIs > 1) option n.children.nodes.map(_.id)
}
)
}
import Node.Root
implicit private[study] lazy val NodeRootBSONHandler: BSON[Root] = new BSON[Root] {
def reads(r: Reader) =
import Node.BsonFields._
def reads(fullReader: Reader) = {
val rootNode = fullReader.doc.getAsOpt[Bdoc](Path.rootDbKey) err "Missing root"
val r = new Reader(rootNode)
Root(
ply = r int "p",
fen = r.get[FEN]("f"),
check = r boolD "c",
shapes = r.getO[Shapes]("h") | Shapes.empty,
comments = r.getO[Comments]("co") | Comments.empty,
gamebook = r.getO[Gamebook]("ga"),
glyphs = r.getO[Glyphs]("g") | Glyphs.empty,
score = r.getO[Score]("e"),
clock = r.getO[Centis]("l"),
crazyData = r.getO[Crazyhouse.Data]("z"),
children = r.get[Node.Children]("n")
)
def writes(w: Writer, s: Root) =
$doc(
"p" -> s.ply,
"f" -> s.fen,
"c" -> w.boolO(s.check),
"h" -> s.shapes.value.nonEmpty.option(s.shapes),
"co" -> s.comments.value.nonEmpty.option(s.comments),
"ga" -> s.gamebook,
"g" -> s.glyphs.nonEmpty,
"e" -> s.score,
"l" -> s.clock,
"z" -> s.crazyData,
"n" -> s.children
ply = r int ply,
fen = r.get[FEN](fen),
check = r boolD check,
shapes = r.getO[Shapes](shapes) | Shapes.empty,
comments = r.getO[Comments](comments) | Comments.empty,
gamebook = r.getO[Gamebook](gamebook),
glyphs = r.getO[Glyphs](glyphs) | Glyphs.empty,
score = r.getO[Score](score),
clock = r.getO[Centis](clock),
crazyData = r.getO[Crazyhouse.Data](crazy),
children = StudyFlatTree.reader.rootChildren(fullReader.doc)
)
}
def writes(w: Writer, r: Root) = $doc(
StudyFlatTree.writer.rootChildren(r) appended {
Path.rootDbKey -> $doc(
ply -> r.ply,
fen -> r.fen,
check -> r.check.some.filter(identity),
shapes -> r.shapes.value.nonEmpty.option(r.shapes),
comments -> r.comments.value.nonEmpty.option(r.comments),
gamebook -> r.gamebook,
glyphs -> r.glyphs.nonEmpty,
score -> r.score,
clock -> r.clock,
crazy -> r.crazyData
)
}
)
}
implicit val PathBSONHandler = BSONStringHandler.as[Path](Path.apply, _.toString)

52
modules/study/src/main/ChapterRepo.scala
View File

@ -126,40 +126,38 @@ final class ChapterRepo(val coll: Coll)(implicit
def setScore(score: Option[lila.tree.Eval.Score]) = setNodeValue("e", score) _
def setChildren(children: Node.Children) = setNodeValue("n", children.some) _
// overrides all children sub-nodes in DB! Make the tree merge beforehand.
def setChildren(children: Node.Children)(chapter: Chapter, path: Path): Funit = {
val set: Bdoc = {
(children.nodes.sizeIs > 1) ?? $doc(
pathToField(path, Node.BsonFields.order) -> children.nodes.map(_.id)
)
} ++ $doc(childrenTreeToBsonElements(path, children))
coll.update.one($id(chapter.id), $set(set)).void
}
private def childrenTreeToBsonElements(parentPath: Path, children: Node.Children): Vector[(String, Bdoc)] =
children.nodes.flatMap { node =>
val path = parentPath + node
childrenTreeToBsonElements(path, node.children) appended (path.toDbField -> writeNode(node))
}
private def setNodeValue[A: BSONWriter](
field: String,
value: Option[A]
)(chapter: Chapter, path: Path): Funit =
pathToField(chapter, path, field) match {
case None =>
logger.warn(s"Can't setNodeValue ${chapter.id} $path $field")
funit
case Some(field) =>
(value match {
case None => coll.unsetField($id(chapter.id), field)
case Some(v) => coll.updateField($id(chapter.id), field, v)
}) void
}
// root.n.0.n.0.n.1.n.0.n.2.subField
private def pathToField(chapter: Chapter, path: Path, subField: String): Option[String] =
if (path.isEmpty) s"root.$subField".some
else
chapter.root.children.pathToIndexes(path) map { indexes =>
s"root.n.${indexes.mkString(".n.")}.$subField"
coll
.updateOrUnsetField($id(chapter.id), pathToField(path, field), value)
.addEffect {
case 0 => logger.warn(s"Can't setNodeValue ${chapter.id} $path $field, no node matched!")
case _ =>
}
.void
private[study] def setChild(chapter: Chapter, path: Path, child: Node): Funit =
pathToField(chapter, path, "n") ?? { parentChildrenPath =>
coll.update.one(
$id(chapter.id) ++ $doc(s"$parentChildrenPath.i" -> child.id),
$set(s"$parentChildrenPath.$$" -> child)
) flatMap { res =>
(res.n == 0) ?? coll.update.one($id(chapter.id), $push(parentChildrenPath -> child)).void
}
}
// root.path.subField
private def pathToField(path: Path, subField: String): String = s"${path.toDbField}.$subField"
private[study] def idNamesByStudyIds(
studyIds: Seq[Study.Id],

2
modules/study/src/main/Env.scala
View File

@ -46,7 +46,7 @@ final class Env(
private val socket = wire[StudySocket]
lazy val studyRepo = new StudyRepo(db(CollName("study")))
lazy val chapterRepo = new ChapterRepo(db(CollName("study_chapter")))
lazy val chapterRepo = new ChapterRepo(db(CollName("study_chapter_flat")))
private lazy val topicRepo = new StudyTopicRepo(db(CollName("study_topic")))
private lazy val userTopicRepo = new StudyUserTopicRepo(db(CollName("study_user_topic")))

6
modules/study/src/main/GifExport.scala
View File

@ -28,7 +28,7 @@ final class GifExport(
chapter.tags(_.Black),
chapter.tags(_.BlackElo).map(elo => s"($elo)")
).flatten.mkString(" "),
"frames" -> framesRec(chapter.root :: chapter.root.mainline, Json.arr())
"frames" -> framesRec(chapter.root +: chapter.root.mainline, Json.arr())
)
)
.stream() flatMap {
@ -39,10 +39,10 @@ final class GifExport(
}
@annotation.tailrec
private def framesRec(nodes: List[RootOrNode], arr: JsArray): JsArray =
private def framesRec(nodes: Vector[RootOrNode], arr: JsArray): JsArray =
nodes match {
case Nil => arr
case node :: tail =>
case node +: tail =>
framesRec(
tail,
arr :+ Json

36
modules/study/src/main/Node.scala
View File

@ -22,7 +22,7 @@ sealed trait RootOrNode {
val score: Option[Score]
def addChild(node: Node): RootOrNode
def fullMoveNumber = 1 + ply / 2
def mainline: List[Node]
def mainline: Vector[Node]
def color = chess.Color.fromPly(ply)
def moveOption: Option[Uci.WithSan]
}
@ -70,7 +70,7 @@ case class Node(
def toggleGlyph(glyph: Glyph) = copy(glyphs = glyphs toggle glyph)
def mainline: List[Node] = this :: children.first.??(_.mainline)
def mainline: Vector[Node] = this +: children.first.??(_.mainline)
def updateMainlineLast(f: Node => Node): Node =
children.first.fold(f(this)) { main =>
@ -120,10 +120,11 @@ object Node {
case (head, tail) => get(head) flatMap (_.children nodeAt tail)
}
def nodesOn(path: Path): List[(Node, Path)] =
// select all nodes on that path
def nodesOn(path: Path): Vector[(Node, Path)] =
path.split ?? { case (head, tail) =>
get(head) ?? { first =>
(first, Path(List(head))) :: first.children.nodesOn(tail).map { case (n, p) =>
(first, Path(Vector(head))) +: first.children.nodesOn(tail).map { case (n, p) =>
(n, p prepend head)
}
}
@ -216,14 +217,6 @@ object Node {
case x => x
})
// List(0, 0, 1, 0, 2)
def pathToIndexes(path: Path): Option[List[Int]] =
path.split.fold(List.empty[Int].some) { case (head, tail) =>
getNodeAndIndex(head) flatMap { case (node, index) =>
node.children.pathToIndexes(tail).map(rest => index :: rest)
}
}
def countRecursive: Int =
nodes.foldLeft(nodes.size) { case (count, n) =>
count + n.children.countRecursive
@ -302,7 +295,7 @@ object Node {
copy(children = children.update(main updateMainlineLast f))
}
lazy val mainline: List[Node] = children.first.??(_.mainline)
lazy val mainline: Vector[Node] = children.first.??(_.mainline)
def lastMainlinePly = Chapter.Ply(mainline.lastOption.??(_.ply))
@ -363,4 +356,21 @@ object Node {
children = Children(b.children.view.map(fromBranch).toVector),
forceVariation = false
)
object BsonFields {
val ply = "p"
val uci = "u"
val san = "s"
val fen = "f"
val check = "c"
val shapes = "h"
val comments = "co"
val gamebook = "ga"
val glyphs = "g"
val score = "e"
val clock = "l"
val crazy = "z"
val forceVariation = "fv"
val order = "o"
}
}

32
modules/study/src/main/Path.scala
View File

@ -2,22 +2,23 @@ package lila.study
import chess.format.UciCharPair
case class Path(ids: List[UciCharPair]) extends AnyVal {
case class Path(ids: Vector[UciCharPair]) extends AnyVal {
def head: Option[UciCharPair] = ids.headOption
def tail: Path = Path(ids drop 1)
// def tail: Path = Path(ids drop 1)
def init: Path = Path(ids take (ids.length - 1))
def parent: Path = Path(ids dropRight 1)
def split: Option[(UciCharPair, Path)] = head.map(_ -> tail)
def split: Option[(UciCharPair, Path)] = head.map(_ -> Path(ids.drop(1)))
def isEmpty = ids.isEmpty
def +(node: Node): Path = Path(ids :+ node.id)
def +(more: Path): Path = Path(ids ::: more.ids)
def +(id: UciCharPair): Path = Path(ids appended id)
def +(node: Node): Path = Path(ids appended node.id)
def +(more: Path): Path = Path(ids appendedAll more.ids)
def prepend(id: UciCharPair) = Path(id :: ids)
def prepend(id: UciCharPair) = Path(ids prepended id)
def intersect(other: Path): Path =
Path {
@ -26,6 +27,10 @@ case class Path(ids: List[UciCharPair]) extends AnyVal {
} map (_._1)
}
def toDbField =
if (ids.isEmpty) s"root.${Path.rootDbKey}"
else s"root.${Path encodeDbKey this}"
override def toString = ids.mkString
}
@ -40,11 +45,22 @@ object Path {
UciCharPair(p(0), b)
}
}
.toList
.toVector
}
def fromDbKey(key: String): Path = apply(decodeDbKey(key))
val root = Path("")
// mongodb objects don't support empty keys
val rootDbKey = "_"
// mongodb objects don't support '.' and '$' in keys
def encodeDbKey(path: Path): String = encodeDbKey(path.ids.mkString)
def encodeDbKey(pair: UciCharPair): String = encodeDbKey(pair.toString)
def encodeDbKey(pathStr: String): String = pathStr.replace('.', 144.toChar).replace('$', 145.toChar)
def decodeDbKey(key: String): String = key.replace(144.toChar, '.').replace(145.toChar, '$')
def isMainline(node: RootOrNode, path: Path): Boolean =
path.split.fold(true) { case (id, rest) =>
node.children.first ?? { child =>

25
modules/study/src/main/PgnDump.scala
View File

@ -26,7 +26,7 @@ final class PgnDump(
def ofChapter(study: Study, flags: WithFlags)(chapter: Chapter) =
Pgn(
tags = makeTags(study, chapter),
turns = toTurns(chapter.root)(flags),
turns = toTurns(chapter.root)(flags).toList,
initial = Initial(
chapter.root.comments.list.map(_.text.value) ::: shapeComment(chapter.root.shapes).toList
)
@ -105,7 +105,7 @@ object PgnDump {
result = none,
variations = flags.variations ?? {
variations.view.map { child =>
toTurns(child.mainline, noVariations)
toTurns(child.mainline, noVariations).toList
}.toList
},
secondsLeft = flags.clocks ?? node.clock.map(_.roundSeconds)
@ -138,34 +138,37 @@ object PgnDump {
black = second map { node2move(_, first.children.variations) }
)
def toTurns(root: Node.Root)(implicit flags: WithFlags): List[chessPgn.Turn] =
def toTurns(root: Node.Root)(implicit flags: WithFlags): Vector[chessPgn.Turn] =
toTurns(root.mainline, root.children.variations)
def toTurns(line: List[Node], variations: Variations)(implicit flags: WithFlags): List[chessPgn.Turn] = {
def toTurns(
line: Vector[Node],
variations: Variations
)(implicit flags: WithFlags): Vector[chessPgn.Turn] = {
line match {
case Nil => Nil
case first :: rest if first.ply % 2 == 0 =>
case Vector() => Vector()
case first +: rest if first.ply % 2 == 0 =>
chessPgn.Turn(
number = 1 + (first.ply - 1) / 2,
white = none,
black = node2move(first, variations).some
) :: toTurnsFromWhite(rest, first.children.variations)
) +: toTurnsFromWhite(rest, first.children.variations)
case l => toTurnsFromWhite(l, variations)
}
}.filterNot(_.isEmpty)
def toTurnsFromWhite(line: List[Node], variations: Variations)(implicit
def toTurnsFromWhite(line: Vector[Node], variations: Variations)(implicit
flags: WithFlags
): List[chessPgn.Turn] =
): Vector[chessPgn.Turn] =
line
.grouped(2)
.foldLeft(variations -> List.empty[chessPgn.Turn]) { case variations ~ turns ~ pair =>
.foldLeft(variations -> Vector.empty[chessPgn.Turn]) { case variations ~ turns ~ pair =>
pair.headOption.fold(variations -> turns) { first =>
pair
.lift(1)
.getOrElse(first)
.children
.variations -> (toTurn(first, pair lift 1, variations) :: turns)
.variations -> (toTurn(first, pair lift 1, variations) +: turns)
}
}
._2

10
modules/study/src/main/ServerEval.scala
View File

@ -57,7 +57,7 @@ object ServerEval {
case Study.WithChapter(_, chapter) =>
(complete ?? chapterRepo.completeServerEval(chapter)) >> {
lila.common.Future
.fold(chapter.root.mainline zip analysis.infoAdvices)(Path.root) {
.fold(chapter.root.mainline.zip(analysis.infoAdvices).toList)(Path.root) {
case (path, (node, (info, advOpt))) =>
info.eval.score
.ifTrue {
@ -78,10 +78,10 @@ object ServerEval {
node.glyphs merge Glyphs.fromList(List(adv.judgment.glyph))
)(chapter, path + node) >> {
chapter.root.nodeAt(path).flatMap { parent =>
analysisLine(parent, chapter.setup.variant, info) flatMap { child =>
parent.addChild(child).children.get(child.id)
}
} ?? { chapterRepo.setChild(chapter, path, _) }
analysisLine(parent, chapter.setup.variant, info) map parent.addChild
} ?? { parentWithNewChildren =>
chapterRepo.setChildren(parentWithNewChildren.children)(chapter, path)
}
}
}
} inject path + node

3
modules/study/src/main/StudyApi.scala
View File

@ -296,7 +296,8 @@ final class StudyApi(
}
}
def promote(studyId: Study.Id, position: Position.Ref, toMainline: Boolean)(who: Who) =
// rewrites the whole chapter because of `forceVariation`. Very inefficient.
def promote(studyId: Study.Id, position: Position.Ref, toMainline: Boolean)(who: Who): Funit =
sequenceStudyWithChapter(studyId, position.chapterId) { case Study.WithChapter(study, chapter) =>
Contribute(who.u, study) {
chapter.updateRoot { root =>

66
modules/study/src/main/StudyFlatTree.scala 100644
View File

@ -0,0 +1,66 @@
package lila.study
import BSONHandlers._
import Node.Children
import lila.common.Chronometer
import lila.db.dsl._
import lila.tree.Eval
import lila.tree.Eval.Score
private object StudyFlatTree {
private case class FlatNode(path: Path, data: Bdoc) {
val depth = path.ids.size
def toNodeWithChildren(children: Option[Children]): Node = {
readNode(data, path.ids.last)
}.copy(children = children | Node.emptyChildren)
}
object reader {
def rootChildren(flatTree: Bdoc): Children =
Chronometer.syncMon(_.study.tree.read) {
traverse {
flatTree.elements.toList
.collect {
case el if el.name != Path.rootDbKey =>
FlatNode(Path.fromDbKey(el.name), el.value.asOpt[Bdoc].get)
}
.sortBy(-_.depth)
}
}
private def traverse(children: List[FlatNode]): Children =
children
.foldLeft(Map.empty[Path, Children]) { case (allChildren, flat) =>
update(allChildren, flat)
}
.get(Path.root) | Node.emptyChildren
// assumes that node has a greater depth than roots (sort beforehand)
private def update(roots: Map[Path, Children], flat: FlatNode): Map[Path, Children] = {
val node = flat.toNodeWithChildren(roots get flat.path)
roots.removed(flat.path).updatedWith(flat.path.parent) {
case None => Children(Vector(node)).some
case Some(siblings) => siblings.addNode(node).some
}
}
}
object writer {
def rootChildren(root: Node.Root): Vector[(String, Bdoc)] =
Chronometer.syncMon(_.study.tree.write) {
root.children.nodes.flatMap { traverse(_, Path.root) }
}
private def traverse(node: Node, parentPath: Path): Vector[(String, Bdoc)] = {
val path = parentPath + node.id
node.children.nodes.flatMap {
traverse(_, path)
} appended (Path.encodeDbKey(path) -> writeNode(node))
}
}
}

18
modules/study/src/main/StudyMultiBoard.scala
View File

@ -54,6 +54,9 @@ final class StudyMultiBoard(
def nbResults: Fu[Int] = chapterRepo.coll.secondaryPreferred.countSel(selector)
/* TODO fix
* printjson(db.study_chapter_flat.aggregate([{$match:{studyId:'6IzKWsfb'}},{$project:{root:{$objectToArray:'$root'}}},{$unwind:'$root'},{$project:{'root.v.f':1,size:{$strLenBytes:'$root.k'}}},{$sort:{size:-1}},{$limit:1}]).toArray())
* */
def slice(offset: Int, length: Int): Fu[Seq[ChapterPreview]] =
chapterRepo.coll
.aggregateList(length, readPreference = ReadPreference.secondaryPreferred) { framework =>
@ -65,11 +68,16 @@ final class StudyMultiBoard(
Project(
$doc(
"comp" -> $doc(
"$function" -> $doc(
"lang" -> "js",
"args" -> $arr("$root", "$tags"),
"body" -> """function(node, tags) { tags = tags.filter(t => t.startsWith('White') || t.startsWith('Black') || t.startsWith('Result')); if (tags.length) while(child = node.n[0]) { node = child }; return {node:{fen:node.f,uci:node.u},tags} }"""
)
// "$function" -> $doc(
// "lang" -> "js",
// "args" -> $arr("$root", "$tags"),
// "body" -> """function(node, tags) { tags = tags.filter(t => t.startsWith('White') || t.startsWith('Black') || t.startsWith('Result')); if (tags.length) while(child = node.n[0]) { node = child }; return {node:{fen:node.f,uci:node.u},tags} }"""
// )
// {node:{fen:node.f,uci:node.u},tags}
"node" -> $doc(
"fen" -> "$root._.f"
),
"tags" -> true
),
"orientation" -> "$setup.orientation",
"name" -> true

11
modules/study/src/main/StudySequencer.scala
View File

@ -27,10 +27,13 @@ final private class StudySequencer(
f: Study.WithChapter => Funit
): Funit =
sequenceStudy(studyId) { study =>
chapterRepo.byId(chapterId) flatMap {
_.filter(_.studyId == studyId) ?? { chapter =>
f(Study.WithChapter(study, chapter))
chapterRepo
.byId(chapterId)
.flatMap {
_.filter(_.studyId == studyId) ?? { chapter =>
f(Study.WithChapter(study, chapter))
}
}
}
.mon(_.study.sequencer.chapterTime)
}
}

10
modules/study/src/test/PgnDumpTest.scala
View File

@ -35,7 +35,7 @@ class PgnDumpTest extends Specification {
}
"one move" in {
val tree = root.copy(children = children(node(1, "e2e4", "e4")))
P.toTurns(tree) must beLike { case List(Turn(1, Some(move), None)) =>
P.toTurns(tree) must beLike { case Vector(Turn(1, Some(move), None)) =>
move.san must_== "e4"
move.variations must beEmpty
}
@ -47,7 +47,7 @@ class PgnDumpTest extends Specification {
node(1, "g1f3", "Nf3")
)
)
P.toTurns(tree) must beLike { case List(Turn(1, Some(move), None)) =>
P.toTurns(tree) must beLike { case Vector(Turn(1, Some(move), None)) =>
move.san must_== "e4"
move.variations must beLike { case List(List(Turn(1, Some(move), None))) =>
move.san must_== "Nf3"
@ -69,7 +69,7 @@ class PgnDumpTest extends Specification {
node(1, "g1f3", "Nf3")
)
)
P.toTurns(tree) must beLike { case List(Turn(1, Some(white), Some(black))) =>
P.toTurns(tree) must beLike { case Vector(Turn(1, Some(white), Some(black))) =>
white.san must_== "e4"
white.variations must beLike { case List(List(Turn(1, Some(move), None))) =>
move.san must_== "Nf3"
@ -97,7 +97,7 @@ class PgnDumpTest extends Specification {
P.toTurns(tree).mkString(" ").toString must_==
"1. e4 (1. Nf3) 1... d5 (1... Nf6)"
P.toTurns(tree) must beLike { case List(Turn(1, Some(white), Some(black))) =>
P.toTurns(tree) must beLike { case Vector(Turn(1, Some(white), Some(black))) =>
white.san must_== "e4"
white.variations must beLike { case List(List(Turn(1, Some(move), None))) =>
move.san must_== "Nf3"
@ -158,7 +158,7 @@ class PgnDumpTest extends Specification {
P.toTurns(tree).mkString(" ").toString must_==
"1. e4 (1. Nf3 a6 (1... b6 2. c4)) 1... d5 (1... Nf6 2. h4) 2. a3 (2. b3)"
P.toTurns(tree) must beLike { case List(Turn(1, Some(w1), Some(b1)), Turn(2, Some(w2), None)) =>
P.toTurns(tree) must beLike { case Vector(Turn(1, Some(w1), Some(b1)), Turn(2, Some(w2), None)) =>
w1.san must_== "e4"
w1.variations must beLike { case List(List(Turn(1, Some(w), Some(b)))) =>
w.san must_== "Nf3"

2
project/Dependencies.scala
View File

@ -8,7 +8,7 @@ object Dependencies {
val scalalib = "com.github.ornicar" %% "scalalib" % "7.0.2"
val hasher = "com.roundeights" %% "hasher" % "1.2.1"
val jodaTime = "joda-time" % "joda-time" % "2.10.9"
val chess = "org.lichess" %% "scalachess" % "10.1.7"
val chess = "org.lichess" %% "scalachess" % "10.2.0"
val compression = "org.lichess" %% "compression" % "1.6"
val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.3.1-THIB"
val prismic = "io.prismic" %% "scala-kit" % "1.2.19-THIB213"