fetch videos from google spreadsheet, save them in mongodb
This commit is contained in:
parent
8a314bd150
commit
6e7d9862e5
|
@ -69,7 +69,8 @@ final class Env(
|
|||
Env.chat,
|
||||
Env.puzzle,
|
||||
Env.tv,
|
||||
Env.blog)
|
||||
Env.blog,
|
||||
Env.video)
|
||||
loginfo("[boot] Preloading complete")
|
||||
}
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ final class Env(
|
|||
private val settings = new {
|
||||
val CollectionVideo = config getString "collection.video"
|
||||
val CollectionView = config getString "collection.view"
|
||||
val SheetUrl = config getString "sheet.url"
|
||||
}
|
||||
import settings._
|
||||
|
||||
|
@ -19,13 +20,19 @@ final class Env(
|
|||
videoColl = videoColl,
|
||||
viewColl = viewColl)
|
||||
|
||||
private lazy val fetch = new FetchSheet(
|
||||
url = SheetUrl,
|
||||
api = api)
|
||||
|
||||
fetch.apply.thenPp
|
||||
|
||||
private[video] lazy val videoColl = db(CollectionVideo)
|
||||
private[video] lazy val viewColl = db(CollectionView)
|
||||
}
|
||||
|
||||
object Env {
|
||||
|
||||
lazy val current: Env = "[boot] opening" describes new Env(
|
||||
config = lila.common.PlayApp loadConfig "opening",
|
||||
lazy val current: Env = "[boot] video" describes new Env(
|
||||
config = lila.common.PlayApp loadConfig "video",
|
||||
db = lila.db.Env.current)
|
||||
}
|
||||
|
|
82
modules/video/src/main/FetchSheet.scala
Normal file
82
modules/video/src/main/FetchSheet.scala
Normal file
|
@ -0,0 +1,82 @@
|
|||
package lila.video
|
||||
|
||||
import org.joda.time.DateTime
|
||||
import play.api.libs.functional.syntax._
|
||||
import play.api.libs.json._
|
||||
import play.api.libs.ws.WS
|
||||
import play.api.Play.current
|
||||
|
||||
private[video] final class FetchSheet(
|
||||
url: String,
|
||||
api: VideoApi) {
|
||||
|
||||
import FetchSheet._
|
||||
|
||||
private implicit val readGStr = Json.reads[GStr]
|
||||
private implicit val readEntry = Json.reads[Entry]
|
||||
private implicit val readEntries: Reads[Seq[Entry]] =
|
||||
(__ \ "feed" \ "entry").read(Reads seq readEntry)
|
||||
|
||||
def apply: Funit = fetch flatMap { entries =>
|
||||
entries.map { entry =>
|
||||
api.video.find(entry.youtubeId) flatMap {
|
||||
case Some(video) => api.video.save(video.copy(
|
||||
title = entry.title,
|
||||
author = entry.author,
|
||||
targets = entry.targets,
|
||||
tags = entry.tags,
|
||||
lang = entry.lang,
|
||||
lichess = entry.lichess,
|
||||
ads = entry.ads,
|
||||
updatedAt = entry.updatedAt))
|
||||
case None => api.video.save(Video(
|
||||
id = entry.youtubeId,
|
||||
title = entry.title,
|
||||
author = entry.author,
|
||||
targets = entry.targets,
|
||||
tags = entry.tags,
|
||||
lang = entry.lang,
|
||||
lichess = entry.lichess,
|
||||
ads = entry.ads,
|
||||
createdAt = DateTime.now,
|
||||
updatedAt = entry.updatedAt))
|
||||
}
|
||||
}.sequenceFu.void
|
||||
}
|
||||
|
||||
private def fetch: Fu[List[Entry]] = WS.url(url).get() flatMap {
|
||||
case res if res.status == 200 => readEntries reads res.json match {
|
||||
case JsError(err) => fufail(err.toString)
|
||||
case JsSuccess(entries, _) => fuccess(entries.toList)
|
||||
}
|
||||
case res => fufail(s"[video] fetch sheet ${res.status}")
|
||||
}
|
||||
}
|
||||
|
||||
object FetchSheet {
|
||||
|
||||
case class GStr(`$t`: String) {
|
||||
override def toString = `$t`
|
||||
}
|
||||
|
||||
case class Entry(
|
||||
`gsx$youtubeid`: GStr,
|
||||
`gsx$youtubeauthor`: GStr,
|
||||
`gsx$title`: GStr,
|
||||
`gsx$target`: GStr,
|
||||
`gsx$tags`: GStr,
|
||||
`gsx$language`: GStr,
|
||||
`gsx$useslichess`: GStr,
|
||||
`gsx$ads`: GStr,
|
||||
updated: GStr) {
|
||||
def youtubeId = `gsx$youtubeid`.toString
|
||||
def author = `gsx$youtubeauthor`.toString
|
||||
def title = `gsx$title`.toString
|
||||
def targets = `gsx$target`.toString.split(';').map(_.trim).toList flatMap parseIntOption
|
||||
def tags = `gsx$tags`.toString.split(';').map(_.trim.toLowerCase).toList
|
||||
def lang = `gsx$language`.toString
|
||||
def lichess = `gsx$useslichess`.toString == "yes"
|
||||
def ads = `gsx$ads`.toString == "yes"
|
||||
def updatedAt = new DateTime(updated.toString)
|
||||
}
|
||||
}
|
|
@ -11,7 +11,8 @@ case class Video(
|
|||
lang: Lang,
|
||||
lichess: Boolean,
|
||||
ads: Boolean,
|
||||
createdAt: DateTime) {
|
||||
createdAt: DateTime,
|
||||
updatedAt: DateTime) {
|
||||
}
|
||||
|
||||
object Target {
|
||||
|
|
Loading…
Reference in a new issue