fetch videos from google spreadsheet, save them in mongodb

app/Env.scala

@@ -69,7 +69,8 @@ final class Env(
       Env.chat,
       Env.puzzle,
       Env.tv,
-      Env.blog)
+      Env.blog,
+      Env.video)
     loginfo("[boot] Preloading complete")
   }
 

modules/video/src/main/Env.scala

@@ -12,6 +12,7 @@ final class Env(
   private val settings = new {
     val CollectionVideo = config getString "collection.video"
     val CollectionView = config getString "collection.view"
+    val SheetUrl = config getString "sheet.url"
   }
   import settings._
 
@@ -19,13 +20,19 @@ final class Env(
     videoColl = videoColl,
     viewColl = viewColl)
 
+  private lazy val fetch = new FetchSheet(
+    url = SheetUrl,
+    api = api)
+
+  fetch.apply.thenPp
+
   private[video] lazy val videoColl = db(CollectionVideo)
   private[video] lazy val viewColl = db(CollectionView)
 }
 
 object Env {
 
-  lazy val current: Env = "[boot] opening" describes new Env(
-    config = lila.common.PlayApp loadConfig "opening",
+  lazy val current: Env = "[boot] video" describes new Env(
+    config = lila.common.PlayApp loadConfig "video",
     db = lila.db.Env.current)
 }

modules/video/src/main/FetchSheet.scala

@@ -0,0 +1,82 @@
+package lila.video
+
+import org.joda.time.DateTime
+import play.api.libs.functional.syntax._
+import play.api.libs.json._
+import play.api.libs.ws.WS
+import play.api.Play.current
+
+private[video] final class FetchSheet(
+    url: String,
+    api: VideoApi) {
+
+  import FetchSheet._
+
+  private implicit val readGStr = Json.reads[GStr]
+  private implicit val readEntry = Json.reads[Entry]
+  private implicit val readEntries: Reads[Seq[Entry]] =
+    (__ \ "feed" \ "entry").read(Reads seq readEntry)
+
+  def apply: Funit = fetch flatMap { entries =>
+    entries.map { entry =>
+      api.video.find(entry.youtubeId) flatMap {
+        case Some(video) => api.video.save(video.copy(
+          title = entry.title,
+          author = entry.author,
+          targets = entry.targets,
+          tags = entry.tags,
+          lang = entry.lang,
+          lichess = entry.lichess,
+          ads = entry.ads,
+          updatedAt = entry.updatedAt))
+        case None => api.video.save(Video(
+          id = entry.youtubeId,
+          title = entry.title,
+          author = entry.author,
+          targets = entry.targets,
+          tags = entry.tags,
+          lang = entry.lang,
+          lichess = entry.lichess,
+          ads = entry.ads,
+          createdAt = DateTime.now,
+          updatedAt = entry.updatedAt))
+      }
+    }.sequenceFu.void
+  }
+
+  private def fetch: Fu[List[Entry]] = WS.url(url).get() flatMap {
+    case res if res.status == 200 => readEntries reads res.json match {
+      case JsError(err)          => fufail(err.toString)
+      case JsSuccess(entries, _) => fuccess(entries.toList)
+    }
+    case res => fufail(s"[video] fetch sheet ${res.status}")
+  }
+}
+
+object FetchSheet {
+
+  case class GStr(`$t`: String) {
+    override def toString = `$t`
+  }
+
+  case class Entry(
+      `gsx$youtubeid`: GStr,
+      `gsx$youtubeauthor`: GStr,
+      `gsx$title`: GStr,
+      `gsx$target`: GStr,
+      `gsx$tags`: GStr,
+      `gsx$language`: GStr,
+      `gsx$useslichess`: GStr,
+      `gsx$ads`: GStr,
+      updated: GStr) {
+    def youtubeId = `gsx$youtubeid`.toString
+    def author = `gsx$youtubeauthor`.toString
+    def title = `gsx$title`.toString
+    def targets = `gsx$target`.toString.split(';').map(_.trim).toList flatMap parseIntOption
+    def tags = `gsx$tags`.toString.split(';').map(_.trim.toLowerCase).toList
+    def lang = `gsx$language`.toString
+    def lichess = `gsx$useslichess`.toString == "yes"
+    def ads = `gsx$ads`.toString == "yes"
+    def updatedAt = new DateTime(updated.toString)
+  }
+}

modules/video/src/main/Video.scala

@@ -11,7 +11,8 @@ case class Video(
     lang: Lang,
     lichess: Boolean,
     ads: Boolean,
-    createdAt: DateTime) {
+    createdAt: DateTime,
+    updatedAt: DateTime) {
 }
 
 object Target {