wiki fetcher with acturius markdown implementation
This commit is contained in:
parent
4a7be5fd3f
commit
0adc964955
|
@ -107,6 +107,7 @@ final class Settings(config: Config) {
|
|||
val MessageCollectionThread = getString("message.collection.thread")
|
||||
|
||||
val WikiCollectionPage = getString("wiki.collection.page")
|
||||
val WikiGitUrl = getString("wiki.git_url")
|
||||
|
||||
val BookmarkCollectionBookmark = getString("bookmark.collection.bookmark")
|
||||
|
||||
|
|
47
app/wiki/Fetch.scala
Normal file
47
app/wiki/Fetch.scala
Normal file
|
@ -0,0 +1,47 @@
|
|||
package lila
|
||||
package wiki
|
||||
|
||||
import java.io.File
|
||||
import com.google.common.io.Files
|
||||
import org.eclipse.jgit.api.Git
|
||||
import org.eclipse.jgit.lib.Repository
|
||||
import org.eclipse.jgit.storage.file.FileRepository
|
||||
import scalaz.effects._
|
||||
import scala.collection.JavaConversions._
|
||||
import eu.henkelmann.actuarius.ActuariusTransformer
|
||||
|
||||
final class Fetch(
|
||||
gitUrl: String,
|
||||
pageRepo: PageRepo) {
|
||||
|
||||
def apply: IO[Unit] = for {
|
||||
files ← getFiles
|
||||
pages = files.map(filePage).flatten
|
||||
_ ← pageRepo.clear
|
||||
_ ← pages.map(pageRepo.saveIO).sequence
|
||||
} yield ()
|
||||
|
||||
private def filePage(file: File): Option[Page] = {
|
||||
val name = """^(.+)\.md$""".r.replaceAllIn(file.getName, _ group 1)
|
||||
(name != "Home") option Page(name, toHtml(fileContent(file)))
|
||||
}
|
||||
|
||||
private def getFiles: IO[List[File]] = io {
|
||||
val dir = Files.createTempDir
|
||||
dir.deleteOnExit
|
||||
Git.cloneRepository
|
||||
.setURI("/home/thib/lichess.wiki/.git")
|
||||
//.setURI(gitUrl)
|
||||
.setDirectory(dir)
|
||||
.setBare(false)
|
||||
.call
|
||||
dir.listFiles.toList filter (_.isFile) sortBy (_.getName)
|
||||
}
|
||||
|
||||
private def fileContent(file: File) =
|
||||
scala.io.Source.fromFile(file.getCanonicalPath).mkString
|
||||
|
||||
private def toHtml(input: String): String =
|
||||
new ActuariusTransformer() apply input
|
||||
|
||||
}
|
|
@ -6,11 +6,31 @@ import user.User
|
|||
import org.joda.time.DateTime
|
||||
import com.novus.salat.annotations.Key
|
||||
import ornicar.scalalib.OrnicarRandom
|
||||
import java.text.Normalizer
|
||||
|
||||
case class Page(
|
||||
@Key("_id") id: String,
|
||||
name: String,
|
||||
title: String,
|
||||
body: String) {
|
||||
|
||||
def slug = id
|
||||
}
|
||||
|
||||
object Page {
|
||||
|
||||
def apply(name: String, body: String): Page = new Page(
|
||||
id = dropNumber(slugify(name)),
|
||||
name = name,
|
||||
title = dropNumber(name.replace("-", " ")),
|
||||
body = body)
|
||||
|
||||
private def slugify(input: String) = {
|
||||
val nowhitespace = input.replace(" ", "_")
|
||||
val normalized = Normalizer.normalize(nowhitespace, Normalizer.Form.NFD)
|
||||
"""[^\w-]""".r.replaceAllIn(normalized, "")
|
||||
}
|
||||
|
||||
private def dropNumber(input: String) =
|
||||
"""^\d+_(.+)$""".r.replaceAllIn(input, _ group 1)
|
||||
}
|
||||
|
|
|
@ -17,4 +17,12 @@ final class PageRepo(
|
|||
val all: IO[List[Page]] = io {
|
||||
find(DBObject()).sort(DBObject("name" -> 1)).toList
|
||||
}
|
||||
|
||||
def saveIO(page: Page): IO[Unit] = io {
|
||||
save(page)
|
||||
}
|
||||
|
||||
val clear: IO[Unit] = io {
|
||||
remove(DBObject())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -13,6 +13,9 @@ final class WikiEnv(
|
|||
|
||||
lazy val pageRepo = new PageRepo(mongodb(WikiCollectionPage))
|
||||
|
||||
lazy val api = new Api(
|
||||
lazy val api = new Api(pageRepo = pageRepo)
|
||||
|
||||
lazy val fetch = new Fetch(
|
||||
gitUrl = WikiGitUrl,
|
||||
pageRepo = pageRepo)
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@ object Main {
|
|||
def titivate = env.titivate
|
||||
def forum = Forum(env.forum)
|
||||
def infos = Infos(env)
|
||||
def wiki = Wiki(env.wiki)
|
||||
|
||||
args.toList match {
|
||||
case "average-elo" :: Nil ⇒ infos.averageElo
|
||||
|
@ -33,6 +34,7 @@ object Main {
|
|||
case "game-finish" :: Nil ⇒ titivate.finishByClock
|
||||
case "game-per-day" :: Nil ⇒ games.perDay(30)
|
||||
case "game-per-day" :: days :: Nil ⇒ games.perDay(parseIntOption(days) err "days: Int")
|
||||
case "wiki-fetch" :: Nil ⇒ wiki.fetch
|
||||
case _ ⇒
|
||||
putStrLn("Unknown command: " + args.mkString(" "))
|
||||
}
|
||||
|
|
12
cli/src/main/scala/Wiki.scala
Normal file
12
cli/src/main/scala/Wiki.scala
Normal file
|
@ -0,0 +1,12 @@
|
|||
package lila.cli
|
||||
|
||||
import lila.wiki.WikiEnv
|
||||
import scalaz.effects._
|
||||
|
||||
case class Wiki(env: WikiEnv) {
|
||||
|
||||
def fetch: IO[Unit] = for {
|
||||
_ ← putStrLn("Fetching wiki from github")
|
||||
_ ← env.fetch.apply
|
||||
} yield ()
|
||||
}
|
|
@ -136,6 +136,7 @@ session {
|
|||
}
|
||||
wiki {
|
||||
collection.page = wiki
|
||||
git_url = "git://github.com/ornicar/lichess.wiki.git"
|
||||
}
|
||||
|
||||
# trust proxy X-Forwarded-For header
|
||||
|
|
|
@ -11,6 +11,7 @@ trait Resolvers {
|
|||
val t2v = "t2v.jp repo" at "http://www.t2v.jp/maven-repo/"
|
||||
val guice = "guice-maven" at "http://guice-maven.googlecode.com/svn/trunk"
|
||||
val jgitMaven = "jgit-maven" at "http://download.eclipse.org/jgit/maven"
|
||||
val christophs = "Christophs Maven Repo" at "http://maven.henkelmann.eu/"
|
||||
}
|
||||
|
||||
trait Dependencies {
|
||||
|
@ -30,6 +31,7 @@ trait Dependencies {
|
|||
val csv = "com.github.tototoshi" %% "scala-csv" % "0.3"
|
||||
val hasher = "com.roundeights" % "hasher" % "0.3" from "http://cloud.github.com/downloads/Nycto/Hasher/hasher_2.9.1-0.3.jar"
|
||||
val jgit = "org.eclipse.jgit" % "org.eclipse.jgit" % "1.3.0.201202151440-r"
|
||||
val actuarius = "eu.henkelmann" %% "actuarius" % "0.2.3"
|
||||
}
|
||||
|
||||
object ApplicationBuild extends Build with Resolvers with Dependencies {
|
||||
|
@ -38,7 +40,7 @@ object ApplicationBuild extends Build with Resolvers with Dependencies {
|
|||
organization := "com.github.ornicar",
|
||||
version := "1.0",
|
||||
scalaVersion := "2.9.1",
|
||||
resolvers := Seq(iliaz, codahale, sonatype, sonatypeS, typesafe, t2v, guice, jgitMaven),
|
||||
resolvers := Seq(iliaz, codahale, sonatype, sonatypeS, typesafe, t2v, guice, jgitMaven, christophs),
|
||||
libraryDependencies := Seq(scalaz, scalalib, hasher),
|
||||
libraryDependencies in test := Seq(specs2),
|
||||
shellPrompt := {
|
||||
|
@ -59,7 +61,8 @@ object ApplicationBuild extends Build with Resolvers with Dependencies {
|
|||
paginator,
|
||||
paginatorSalat,
|
||||
csv,
|
||||
jgit),
|
||||
jgit,
|
||||
actuarius),
|
||||
templatesImport ++= Seq(
|
||||
"lila.game.{ DbGame, DbPlayer, Pov }",
|
||||
"lila.user.User",
|
||||
|
|
Loading…
Reference in a new issue