Add mongodb migration and optimize for storage compaction

chess/src/main/scala/Clock.scala

@@ -9,7 +9,7 @@ sealed trait Clock {
   val color: Color
   val whiteTime: Float
   val blackTime: Float
-  val timer: Double
+  val timerOption: Option[Double]
 
   def time(c: Color) = if (c == White) whiteTime else blackTime
 
@@ -31,6 +31,8 @@ sealed trait Clock {
 
   def step: RunningClock
 
+  def stop: PausedClock
+
   def addTime(c: Color, t: Float): Clock
 
   def giveTime(c: Color, t: Float): Clock
@@ -46,6 +48,8 @@ case class RunningClock(
     blackTime: Float = 0f,
     timer: Double = 0d) extends Clock {
 
+  val timerOption = Some(timer)
+
   override def elapsedTime(c: Color) = time(c) + {
     if (c == color) now - timer else 0
   }.toFloat
@@ -61,6 +65,13 @@ case class RunningClock(
       )
   }
 
+  def stop = PausedClock(
+    limit = limit,
+    increment = increment,
+    color = color,
+    whiteTime = whiteTime,
+    blackTime = blackTime)
+
   def addTime(c: Color, t: Float): RunningClock = c match {
     case White ⇒ copy(whiteTime = whiteTime + t)
     case Black ⇒ copy(blackTime = blackTime + t)
@@ -76,7 +87,7 @@ case class PausedClock(
     whiteTime: Float = 0f,
     blackTime: Float = 0f) extends Clock {
 
-  val timer = 0d
+  val timerOption = None
 
   def step = RunningClock(
     color = color,
@@ -86,6 +97,8 @@ case class PausedClock(
     limit = limit,
     timer = now).giveTime(White, increment).step
 
+  def stop = this
+
   def addTime(c: Color, t: Float): PausedClock = c match {
     case White ⇒ copy(whiteTime = whiteTime + t)
     case Black ⇒ copy(blackTime = blackTime + t)

cli/src/main/scala/ImportDb.scala

@@ -0,0 +1,116 @@
+package lila.cli
+
+import lila.system.model._
+import lila.system.db.GameRepo
+import com.mongodb.casbah.MongoCollection
+import com.mongodb.casbah.Imports._
+import scalaz.effects._
+
+case class ImportDb(mongodb: MongoDB, gameRepo: GameRepo) {
+
+  val oldGames = mongodb("game2")
+  val newGames = mongodb("game")
+  val bulkSize = 200
+  val max = Some(200000)
+  //val max: Option[Int] = None
+
+  def apply: IO[Unit] = for {
+    _ <- putStrLn("- Drop indexes")
+    _ <- gameRepo.dropIndexes
+    _ <- putStrLn("- Import games")
+    _ <- importGames
+    _ <- putStrLn("- Ensure indexes")
+    _ <- gameRepo.ensureIndexes
+  } yield ()
+
+  private def importGames: IO[Unit] = io {
+    val total = oldGames.count
+    println("%d games to import" format total)
+    println("drop db.game")
+    newGames.drop()
+    println("start import")
+    var it = 0
+    val pool = max.fold(
+      m ⇒ oldGames.find().limit(m).grouped(bulkSize),
+      oldGames.grouped(bulkSize))
+    for (olds ← pool) {
+      newGames insert (olds.toList map convert).flatten
+      it = it + bulkSize
+      if (it % 5000 == 0) {
+        val percentage = it / total.toFloat * 100
+        println("%d/%d %2.1f%%".format(it, total, percentage))
+      }
+    }
+    println("unset userIds: []")
+    newGames.update(
+      DBObject("userIds" -> DBList()),
+      $unset("userIds"),
+      upsert = false,
+      multi = true)
+    println("unset winnerUserId: \"\"")
+    newGames.update(
+      DBObject("winnerUserId" -> ""),
+      $unset("winnerUserId"),
+      upsert = false,
+      multi = true)
+    println("done")
+  }
+
+  def convert(game: DBObject): Option[DBObject] = try {
+    if (game.as[String]("_id").size != 8) None
+    else {
+      val status = Status(game.as[Int]("status")).get
+      game.put("players", List(0, 1) map { it ⇒
+        val player = game.expand[BasicDBObject]("players." + it).get
+        player.put("ps", playerPs(player.getString("ps")))
+        player.put("c", player.as[String]("color"))
+        player.removeField("color")
+        if (player.containsField("isWinner")) {
+          player.put("w", player.as[Boolean]("isWinner"))
+          player.removeField("isWinner")
+        }
+        if (status >= Aborted) {
+          player.removeField("isOfferingDraw")
+          player.removeField("isOfferingRematch")
+          player.removeField("lastDrawOffer")
+          player.removeField("evts")
+        }
+        player
+      })
+      game.getAs[DBObject]("clock") map { clock ⇒
+        val times = clock.as[DBObject]("times")
+        clock.removeField("times")
+        clock.put("w", times.as[String]("white"))
+        clock.put("b", times.as[String]("black"))
+        clock.put("i", clock.as[Int]("increment"))
+        clock.removeField("increment")
+        clock.put("l", clock.as[Int]("limit"))
+        clock.removeField("limit")
+        clock.put("c", clock.as[String]("color"))
+        clock.removeField("color")
+        if (status >= Aborted) {
+          clock.removeField("timer")
+        }
+        game.put("clock", clock)
+      }
+      if (game.containsField("creatorColor")) {
+        game.put("cc", game.as[String]("creatorColor"))
+        game -= "creatorColor"
+      }
+      if (game.containsField("variant")) {
+        game.put("v", game.as[Int]("variant"))
+        game -= "variant"
+      }
+      else game.put("v", 1)
+      Some(game)
+    }
+  }
+  catch {
+    case e ⇒ {
+      println("%s - %s".format(game.as[String]("_id"), e.getMessage))
+      None
+    }
+  }
+
+  def playerPs(ps: String) = ps.toString.split(' ').map(_ take 2).mkString(" ")
+}

cli/src/main/scala/Main.scala

@@ -0,0 +1,18 @@
+package lila.cli
+
+import lila.system.SystemEnv
+
+object Main {
+
+  lazy val env = SystemEnv()
+
+  def main(args: Array[String]): Unit = sys exit {
+    args.toList match {
+      case "import-db" :: args ⇒ {
+        ImportDb(env.mongodb, env.gameRepo).apply.unsafePerformIO
+      }
+      case _ ⇒ println("Usage: run command args")
+    }
+    0
+  }
+}

cli/src/main/scala/package.scala

@@ -0,0 +1,16 @@
+package lila
+
+import ornicar.scalalib._
+
+package object cli
+    extends OrnicarValidation
+    with OrnicarCommon
+    with scalaz.NonEmptyLists
+    with scalaz.Strings
+    with scalaz.Lists
+    with scalaz.Booleans {
+
+  implicit def addPP[A](a: A) = new {
+    def pp[A] = a ~ println
+  }
+}

conf/application.conf

@@ -3,7 +3,7 @@ mongo {
   port = 27017
   dbName = lichess
   collection {
-    game = game2
+    game = game
     user = user
     hook = hook
     entry = lobby_entry

project/Build.scala

@@ -23,6 +23,7 @@ trait Dependencies {
   val jodaTime = "joda-time" % "joda-time" % "2.0"
   val jodaConvert = "org.joda" % "joda-convert" % "1.2"
   val scalaTime = "org.scala-tools.time" %% "time" % "0.5"
+  val slf4jNop = "org.slf4j" % "slf4j-nop" % "1.6.4"
 
   // benchmark
   val instrumenter = "com.google.code.java-allocation-instrumenter" % "java-allocation-instrumenter" % "2.0"
@@ -49,7 +50,7 @@ object ApplicationBuild extends Build with Resolvers with Dependencies {
   ) dependsOn (system)
 
   lazy val cli = Project("cli", file("cli"), settings = buildSettings).settings(
-    libraryDependencies ++= Seq(scalaz)
+    libraryDependencies ++= Seq(slf4jNop, scalaz)
   ) dependsOn (system)
 
   lazy val system = Project("system", file("system"), settings = buildSettings).settings(

system/src/main/scala/db/GameRepo.scala

@@ -57,15 +57,15 @@ class GameRepo(collection: MongoCollection)
     for (i ← 0 to 1) {
       val name = "players." + i + "."
       d(name + "ps", _.players(i).ps)
-      d(name + "isWinner", _.players(i).isWinner)
+      d(name + "w", _.players(i).w)
       d(name + "evts", _.players(i).evts)
       d(name + "lastDrawOffer", _.players(i).lastDrawOffer)
       d(name + "isOfferingDraw", _.players(i).isOfferingDraw)
     }
     a.clock foreach { c ⇒
-      d("clock.color", _.clock.get.color)
-      d("clock.white", _.clock.get.white)
-      d("clock.black", _.clock.get.black)
+      d("clock.c", _.clock.get.c)
+      d("clock.w", _.clock.get.w)
+      d("clock.w", _.clock.get.b)
       d("clock.timer", _.clock.get.timer)
     }
 
@@ -79,4 +79,18 @@ class GameRepo(collection: MongoCollection)
   def decode(raw: RawDbGame): Option[DbGame] = raw.decode
 
   def encode(dbGame: DbGame): RawDbGame = RawDbGame encode dbGame
+
+  def ensureIndexes: IO[Unit] = io {
+    collection.ensureIndex(DBObject("status" -> 1))
+    collection.ensureIndex(DBObject("userIds" -> 1))
+    collection.ensureIndex(DBObject("winnerUserId" -> 1))
+    collection.ensureIndex(DBObject("turns" -> 1))
+    collection.ensureIndex(DBObject("updatedAt" -> -1))
+    collection.ensureIndex(DBObject("createdAt" -> -1))
+    collection.ensureIndex(DBObject("createdAt" -> -1, "userIds" ->1))
+  }
+
+  def dropIndexes: IO[Unit] = io {
+    collection.dropIndexes()
+  }
 }

system/src/main/scala/model/DbGame.scala

@@ -174,7 +174,8 @@ case class DbGame(
     winnerId = winner flatMap (player(_).userId),
     whitePlayer = whitePlayer finish (winner == Some(White)),
     blackPlayer = blackPlayer finish (winner == Some(Black)),
-    positionHashes = ""
+    positionHashes = "",
+    clock = clock map (_.stop)
   ) withEvents List(EndEvent())
 
   def rated = isRated

system/src/main/scala/model/RawDbClock.scala

@@ -5,29 +5,31 @@ import lila.chess._
 import scala.math.round
 
 case class RawDbClock(
-    color: String,
-    increment: Int,
-    limit: Int,
-    white: Float,
-    black: Float,
-    timer: Double = 0d) {
+    c: String,
+    i: Int,
+    l: Int,
+    w: Float,
+    b: Float,
+    timer: Option[Double] = None) {
 
   def decode: Option[Clock] = for {
-    trueColor ← Color(color)
+    trueColor ← Color(c)
   } yield {
-    if (timer == 0l) PausedClock(
-      color = trueColor,
-      increment = increment,
-      limit = limit,
-      whiteTime = white,
-      blackTime = black)
-    else RunningClock(
-      color = trueColor,
-      increment = increment,
-      limit = limit,
-      whiteTime = white,
-      blackTime = black,
-      timer = timer)
+    timer.fold(
+      t ⇒ RunningClock(
+        color = trueColor,
+        increment = i,
+        limit = l,
+        whiteTime = w,
+        blackTime = b,
+        timer = t),
+      PausedClock(
+        color = trueColor,
+        increment = i,
+        limit = l,
+        whiteTime = w,
+        blackTime = b)
+    )
   }
 }
 
@@ -36,12 +38,12 @@ object RawDbClock {
   def encode(clock: Clock): RawDbClock = {
     import clock._
     RawDbClock(
-      color = color.name,
-      increment = increment,
-      limit = limit,
-      white = whiteTime,
-      black = blackTime,
-      timer = timer
+      c = color.name,
+      i = increment,
+      l = limit,
+      w = whiteTime,
+      b = blackTime,
+      timer = timerOption
     )
   }
 }

system/src/main/scala/model/RawDbGame.scala

@@ -16,19 +16,19 @@ case class RawDbGame(
     clock: Option[RawDbClock],
     lastMove: Option[String],
     check: Option[String],
-    creatorColor: String = "white",
+    cc: String = "white",
     positionHashes: String = "",
     castles: String = "KQkq",
     isRated: Boolean = false,
-    variant: Int = 1,
+    v: Int = 1,
     winnerId: Option[String] = None) {
 
   def decode: Option[DbGame] = for {
-    whitePlayer ← players find (_.color == "white") flatMap (_.decode)
-    blackPlayer ← players find (_.color == "black") flatMap (_.decode)
+    whitePlayer ← players find (_.c == "white") flatMap (_.decode)
+    blackPlayer ← players find (_.c == "black") flatMap (_.decode)
     trueStatus ← Status(status)
-    trueCreatorColor ← Color(creatorColor)
-    trueVariant ← Variant(variant)
+    trueCreatorColor ← Color(cc)
+    trueVariant ← Variant(v)
     validClock = clock flatMap (_.decode)
     if validClock.isDefined == clock.isDefined
   } yield DbGame(
@@ -63,11 +63,11 @@ object RawDbGame {
       clock = clock map RawDbClock.encode,
       lastMove = lastMove,
       check = check map (_.key),
-      creatorColor = creatorColor.name,
+      cc = creatorColor.name,
       positionHashes = positionHashes,
       castles = castles,
       isRated = isRated,
-      variant = variant.id,
+      v = variant.id,
       winnerId = winnerId
     )
   }

system/src/main/scala/model/RawDbPlayer.scala

@@ -7,10 +7,10 @@ import com.mongodb.DBRef
 
 case class RawDbPlayer(
     id: String,
-    color: String,
+    c: String,
     ps: String,
     aiLevel: Option[Int],
-    isWinner: Option[Boolean],
+    w: Option[Boolean],
     evts: String = "",
     elo: Option[Int],
     isOfferingDraw: Option[Boolean],
@@ -18,13 +18,13 @@ case class RawDbPlayer(
     user: Option[DBRef]) {
 
   def decode: Option[DbPlayer] = for {
-    trueColor ← Color(color)
+    trueColor ← Color(c)
   } yield DbPlayer(
     id = id,
     color = trueColor,
     ps = ps,
     aiLevel = aiLevel,
-    isWinner = isWinner,
+    isWinner = w,
     evts = evts,
     elo = elo,
     isOfferingDraw = isOfferingDraw getOrElse false,
@@ -39,10 +39,10 @@ object RawDbPlayer {
     import dbPlayer._
     RawDbPlayer(
       id = id,
-      color = color.name,
+      c = color.name,
       ps = ps,
       aiLevel = aiLevel,
-      isWinner = isWinner,
+      w = isWinner,
       evts = evts,
       elo = elo,
       isOfferingDraw = if (isOfferingDraw) Some(true) else None,