wiki fetcher with acturius markdown implementation

app/core/Settings.scala

@@ -107,6 +107,7 @@ final class Settings(config: Config) {
   val MessageCollectionThread = getString("message.collection.thread")
 
   val WikiCollectionPage = getString("wiki.collection.page")
+  val WikiGitUrl = getString("wiki.git_url")
 
   val BookmarkCollectionBookmark = getString("bookmark.collection.bookmark")
 

app/wiki/Fetch.scala

@@ -0,0 +1,47 @@
+package lila
+package wiki
+
+import java.io.File
+import com.google.common.io.Files
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.lib.Repository
+import org.eclipse.jgit.storage.file.FileRepository
+import scalaz.effects._
+import scala.collection.JavaConversions._
+import eu.henkelmann.actuarius.ActuariusTransformer
+
+final class Fetch(
+    gitUrl: String,
+    pageRepo: PageRepo) {
+
+  def apply: IO[Unit] = for {
+    files ← getFiles
+    pages = files.map(filePage).flatten
+    _ ← pageRepo.clear
+    _ ← pages.map(pageRepo.saveIO).sequence
+  } yield ()
+
+  private def filePage(file: File): Option[Page] = {
+    val name = """^(.+)\.md$""".r.replaceAllIn(file.getName, _ group 1)
+    (name != "Home") option Page(name, toHtml(fileContent(file)))
+  }
+
+  private def getFiles: IO[List[File]] = io {
+    val dir = Files.createTempDir
+    dir.deleteOnExit
+    Git.cloneRepository
+      .setURI("/home/thib/lichess.wiki/.git")
+      //.setURI(gitUrl)
+      .setDirectory(dir)
+      .setBare(false)
+      .call
+    dir.listFiles.toList filter (_.isFile) sortBy (_.getName)
+  }
+
+  private def fileContent(file: File) = 
+    scala.io.Source.fromFile(file.getCanonicalPath).mkString
+
+  private def toHtml(input: String): String = 
+    new ActuariusTransformer() apply input
+
+}

app/wiki/Page.scala

@@ -6,11 +6,31 @@ import user.User
 import org.joda.time.DateTime
 import com.novus.salat.annotations.Key
 import ornicar.scalalib.OrnicarRandom
+import java.text.Normalizer
 
 case class Page(
     @Key("_id") id: String,
+    name: String,
     title: String,
     body: String) {
 
-      def slug = id
+  def slug = id
+}
+
+object Page {
+
+  def apply(name: String, body: String): Page = new Page(
+    id = dropNumber(slugify(name)),
+    name = name,
+    title = dropNumber(name.replace("-", " ")),
+    body = body)
+
+  private def slugify(input: String) = {
+    val nowhitespace = input.replace(" ", "_")
+    val normalized = Normalizer.normalize(nowhitespace, Normalizer.Form.NFD)
+    """[^\w-]""".r.replaceAllIn(normalized, "")
+  }
+
+  private def dropNumber(input: String) = 
+    """^\d+_(.+)$""".r.replaceAllIn(input, _ group 1)
 }

app/wiki/PageRepo.scala

@@ -17,4 +17,12 @@ final class PageRepo(
   val all: IO[List[Page]] = io {
     find(DBObject()).sort(DBObject("name" -> 1)).toList
   }
+
+  def saveIO(page: Page): IO[Unit] = io {
+    save(page)
+  }
+
+  val clear: IO[Unit] = io {
+    remove(DBObject())
+  }
 }

app/wiki/WikiEnv.scala

@@ -13,6 +13,9 @@ final class WikiEnv(
 
   lazy val pageRepo = new PageRepo(mongodb(WikiCollectionPage))
 
-  lazy val api = new Api(
+  lazy val api = new Api(pageRepo = pageRepo)
+  
+  lazy val fetch = new Fetch(
+    gitUrl = WikiGitUrl,
     pageRepo = pageRepo)
 }

cli/src/main/scala/Main.scala

@@ -17,6 +17,7 @@ object Main {
     def titivate = env.titivate
     def forum = Forum(env.forum)
     def infos = Infos(env)
+    def wiki = Wiki(env.wiki)
 
     args.toList match {
       case "average-elo" :: Nil               ⇒ infos.averageElo
@@ -33,6 +34,7 @@ object Main {
       case "game-finish" :: Nil               ⇒ titivate.finishByClock
       case "game-per-day" :: Nil              ⇒ games.perDay(30)
       case "game-per-day" :: days :: Nil      ⇒ games.perDay(parseIntOption(days) err "days: Int")
+      case "wiki-fetch" :: Nil                ⇒ wiki.fetch
       case _ ⇒
         putStrLn("Unknown command: " + args.mkString(" "))
     }

cli/src/main/scala/Wiki.scala

@@ -0,0 +1,12 @@
+package lila.cli
+
+import lila.wiki.WikiEnv
+import scalaz.effects._
+
+case class Wiki(env: WikiEnv) {
+
+  def fetch: IO[Unit] = for {
+    _ ← putStrLn("Fetching wiki from github")
+    _ ← env.fetch.apply
+  } yield ()
+}

conf/application.conf

@@ -136,6 +136,7 @@ session {
 }
 wiki {
   collection.page = wiki
+  git_url = "git://github.com/ornicar/lichess.wiki.git"
 }
 
 # trust proxy X-Forwarded-For header

project/Build.scala

@@ -11,6 +11,7 @@ trait Resolvers {
   val t2v = "t2v.jp repo" at "http://www.t2v.jp/maven-repo/"
   val guice = "guice-maven" at "http://guice-maven.googlecode.com/svn/trunk"
   val jgitMaven = "jgit-maven" at "http://download.eclipse.org/jgit/maven"
+  val christophs = "Christophs Maven Repo" at "http://maven.henkelmann.eu/"
 }
 
 trait Dependencies {
@@ -30,6 +31,7 @@ trait Dependencies {
   val csv = "com.github.tototoshi" %% "scala-csv" % "0.3"
   val hasher = "com.roundeights" % "hasher" % "0.3" from "http://cloud.github.com/downloads/Nycto/Hasher/hasher_2.9.1-0.3.jar"
   val jgit = "org.eclipse.jgit" % "org.eclipse.jgit" % "1.3.0.201202151440-r"
+  val actuarius = "eu.henkelmann" %% "actuarius" % "0.2.3"
 }
 
 object ApplicationBuild extends Build with Resolvers with Dependencies {
@@ -38,7 +40,7 @@ object ApplicationBuild extends Build with Resolvers with Dependencies {
     organization := "com.github.ornicar",
     version := "1.0",
     scalaVersion := "2.9.1",
-    resolvers := Seq(iliaz, codahale, sonatype, sonatypeS, typesafe, t2v, guice, jgitMaven),
+    resolvers := Seq(iliaz, codahale, sonatype, sonatypeS, typesafe, t2v, guice, jgitMaven, christophs),
     libraryDependencies := Seq(scalaz, scalalib, hasher),
     libraryDependencies in test := Seq(specs2),
     shellPrompt := {
@@ -59,7 +61,8 @@ object ApplicationBuild extends Build with Resolvers with Dependencies {
       paginator,
       paginatorSalat,
       csv,
-      jgit),
+      jgit,
+      actuarius),
     templatesImport ++= Seq(
       "lila.game.{ DbGame, DbPlayer, Pov }",
       "lila.user.User",