From bd5fb84e6cd390e5f885352e68f67b6f413048cb Mon Sep 17 00:00:00 2001 From: Thibault Duplessis Date: Sat, 30 Nov 2019 01:34:21 -0600 Subject: [PATCH] security module migration --- conf/base.conf | 9 +- modules/common/src/main/config.scala | 29 ++- modules/common/src/main/model.scala | 6 - modules/db/src/main/Env.scala | 2 +- modules/event/src/main/Env.scala | 25 +-- modules/event/src/main/EventApi.scala | 8 +- modules/memo/src/main/Env.scala | 2 +- modules/oauth/src/main/Env.scala | 11 +- modules/oauth/src/main/OAuthServer.scala | 6 +- .../security/src/main/AutomaticEmail.scala | 11 +- modules/security/src/main/CheckMail.scala | 22 +- modules/security/src/main/Cli.scala | 8 +- modules/security/src/main/DataForm.scala | 3 +- .../src/main/DisposableEmailDomain.scala | 8 +- modules/security/src/main/DnsApi.scala | 15 +- .../src/main/EmailAddressValidator.scala | 7 +- modules/security/src/main/EmailChange.scala | 10 +- modules/security/src/main/EmailConfirm.scala | 35 ++-- modules/security/src/main/Env.scala | 193 +++++------------- modules/security/src/main/Firewall.scala | 36 ++-- modules/security/src/main/GeoIP.scala | 15 +- modules/security/src/main/IpIntel.scala | 11 +- modules/security/src/main/LoginToken.scala | 7 +- modules/security/src/main/MagicLink.scala | 8 +- modules/security/src/main/Mailgun.scala | 72 ++++--- modules/security/src/main/PasswordReset.scala | 10 +- modules/security/src/main/PrintBan.scala | 2 +- modules/security/src/main/Recaptcha.scala | 39 ++-- modules/security/src/main/SecurityApi.scala | 44 ++-- .../security/src/main/SecurityConfig.scala | 68 ++++++ modules/security/src/main/Store.scala | 52 ++--- modules/security/src/main/StringToken.scala | 8 +- modules/security/src/main/Tor.scala | 9 +- modules/security/src/main/UserSpy.scala | 39 ++-- modules/user/src/main/Env.scala | 2 +- modules/user/src/main/PasswordHasher.scala | 2 +- modules/user/src/main/UserRepo.scala | 2 +- 37 files changed, 434 insertions(+), 402 deletions(-) create mode 100644 modules/security/src/main/SecurityConfig.scala diff --git a/conf/base.conf b/conf/base.conf index 44ce2e12fd..fa51cf8873 100644 --- a/conf/base.conf +++ b/conf/base.conf @@ -199,14 +199,8 @@ geoip { security { collection.security = security collection.print_ban = print_ban + collection.firewall = firewall flood.duration = 60 seconds - firewall { - cookie { - enabled = false - name=fEKHA4zI74ZrZrom - } - collection.firewall = firewall - } geoip = ${geoip} password_reset { secret = "???" @@ -254,7 +248,6 @@ oauth { access_token = oauth_access_token app = oauth_client } - base_url = ${net.protocol}oauth.${net.domain}/ } recaptcha { endpoint = "https://www.google.com/recaptcha/api/siteverify" diff --git a/modules/common/src/main/config.scala b/modules/common/src/main/config.scala index 54dbbcf9a5..f8ff338acd 100644 --- a/modules/common/src/main/config.scala +++ b/modules/common/src/main/config.scala @@ -1,15 +1,32 @@ package lila.common +import io.methvin.play.autoconfig._ import play.api.ConfigLoader object config { - implicit val maxPerPageLoader: ConfigLoader[MaxPerPage] = - ConfigLoader(_.getInt).map(MaxPerPage.apply) + case class CollName(value: String) extends AnyVal with StringValue - implicit val collNameLoader: ConfigLoader[CollName] = - ConfigLoader(_.getString).map(CollName.apply) + case class Secret(value: String) extends AnyVal { + override def toString = "Secret(****)" + } - implicit val SecretLoader: ConfigLoader[Secret] = - ConfigLoader(_.getString).map(Secret.apply) + case class BaseUrl(value: String) extends AnyVal with StringValue + + case class NetConfig( + domain: String, + protocol: String, + @ConfigName("base_url") baseUrl: BaseUrl, + email: String + ) + + implicit val maxPerPageLoader = intLoader(MaxPerPage.apply) + implicit val collNameLoader = strLoader(CollName.apply) + implicit val secretLoader = strLoader(Secret.apply) + implicit val baseUrlLoader = strLoader(BaseUrl.apply) + implicit val emailAddressLoader = strLoader(EmailAddress.apply) + implicit val netLoader = AutoConfig.loader[NetConfig] + + private def strLoader[A](f: String => A): ConfigLoader[A] = ConfigLoader(_.getString) map f + private def intLoader[A](f: Int => A): ConfigLoader[A] = ConfigLoader(_.getInt) map f } diff --git a/modules/common/src/main/model.scala b/modules/common/src/main/model.scala index 474b35a567..3cce462164 100644 --- a/modules/common/src/main/model.scala +++ b/modules/common/src/main/model.scala @@ -28,12 +28,6 @@ case class MaxPerSecond(value: Int) extends AnyVal with IntValue case class IpAddress(value: String) extends AnyVal with StringValue -case class CollName(value: String) extends AnyVal with StringValue - -case class Secret(value: String) extends AnyVal { - override def toString = "Secret(****)" -} - object IpAddress { // http://stackoverflow.com/questions/106179/regular-expression-to-match-hostname-or-ip-address private val ipv4Regex = """^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$""".r diff --git a/modules/db/src/main/Env.scala b/modules/db/src/main/Env.scala index b8b4e74501..3693dd804a 100644 --- a/modules/db/src/main/Env.scala +++ b/modules/db/src/main/Env.scala @@ -10,7 +10,7 @@ import scala.concurrent.{ Await, ExecutionContext, Future } import scala.util.{ Failure, Success } import dsl.Coll -import lila.common.{ CollName, Chronometer } +import lila.common.Chronometer import lila.common.config._ case class DbConfig( diff --git a/modules/event/src/main/Env.scala b/modules/event/src/main/Env.scala index 0dce223767..b8d71ddf5b 100644 --- a/modules/event/src/main/Env.scala +++ b/modules/event/src/main/Env.scala @@ -1,28 +1,17 @@ package lila.event -import akka.actor._ -import com.typesafe.config.Config +import play.api.Configuration + +import lila.common.CollName +import lila.common.config._ final class Env( - config: Config, + appConfig: Configuration, db: lila.db.Env, - asyncCache: lila.memo.AsyncCache.Builder, - system: ActorSystem + asyncCache: lila.memo.AsyncCache.Builder ) { - private val CollectionEvent = config getString "collection.event" - - private lazy val eventColl = db(CollectionEvent) + private lazy val eventColl = db(appConfig.get[CollName]("event.collection.event")) lazy val api = new EventApi(coll = eventColl, asyncCache = asyncCache) } - -object Env { - - lazy val current = "event" boot new Env( - config = lila.common.PlayApp loadConfig "event", - db = lila.db.Env.current, - asyncCache = lila.memo.Env.current.asyncCache, - system = lila.common.PlayApp.system - ) -} diff --git a/modules/event/src/main/EventApi.scala b/modules/event/src/main/EventApi.scala index 4042c3215a..a991f01322 100644 --- a/modules/event/src/main/EventApi.scala +++ b/modules/event/src/main/EventApi.scala @@ -29,14 +29,14 @@ final class EventApi( expireAfter = _.ExpireAfterWrite(5 minutes) ) - def fetchPromotable: Fu[List[Event]] = coll.find($doc( + def fetchPromotable: Fu[List[Event]] = coll.ext.find($doc( "enabled" -> true, "startsAt" $gt DateTime.now.minusDays(1) $lt DateTime.now.plusDays(1) )).sort($doc("startsAt" -> 1)).list[Event](10).map { _.filter(_.featureNow) take 3 } - def list = coll.find($empty).sort($doc("startsAt" -> -1)).list[Event](50) + def list = coll.ext.find($empty).sort($doc("startsAt" -> -1)).list[Event](50) def oneEnabled(id: String) = coll.byId[Event](id).map(_.filter(_.enabled)) @@ -47,13 +47,13 @@ final class EventApi( } def update(old: Event, data: EventForm.Data) = - coll.update($id(old.id), data update old) >>- promotable.refresh + coll.update.one($id(old.id), data update old) >>- promotable.refresh def createForm = EventForm.form def create(data: EventForm.Data, userId: String): Fu[Event] = { val event = data make userId - coll.insert(event) >>- promotable.refresh inject event + coll.insert.one(event) >>- promotable.refresh inject event } def clone(old: Event) = old.copy( diff --git a/modules/memo/src/main/Env.scala b/modules/memo/src/main/Env.scala index aac0b672a3..a7524e07ae 100644 --- a/modules/memo/src/main/Env.scala +++ b/modules/memo/src/main/Env.scala @@ -3,8 +3,8 @@ package lila.memo import com.softwaremill.macwire._ import io.methvin.play.autoconfig._ import play.api.Configuration + import lila.db.dsl.Coll -import lila.common.CollName import lila.common.config._ case class MemoConfig( diff --git a/modules/oauth/src/main/Env.scala b/modules/oauth/src/main/Env.scala index fea7752848..93d13bce0f 100644 --- a/modules/oauth/src/main/Env.scala +++ b/modules/oauth/src/main/Env.scala @@ -6,7 +6,6 @@ import io.methvin.play.autoconfig._ import play.api.Configuration import scala.concurrent.duration._ -import lila.common.CollName import lila.common.config._ import lila.db.DbConfig import lila.db.dsl.Coll @@ -14,7 +13,6 @@ import lila.db.Env.configLoader private case class OauthConfig( mongodb: DbConfig, - @ConfigName("base_url") baseUrl: String, @ConfigName("collection.access_token") tokenColl: CollName, @ConfigName("collection.app") appColl: CollName ) @@ -35,11 +33,20 @@ final class Env( lazy val appApi = new OAuthAppApi(appColl) + // #TODO lila should be able to start without it lazy val server = { val mk = (coll: Coll) => wire[OAuthServer] mk(tokenColl) } + lazy val tryServer: OAuthServer.Try = () => scala.concurrent.Future { + server.some + }.withTimeoutDefault(50 millis, none)(system) recover { + case e: Exception => + lila.log("security").warn("oauth", e) + none + } + lazy val tokenApi = new PersonalTokenApi( tokenColl = tokenColl ) diff --git a/modules/oauth/src/main/OAuthServer.scala b/modules/oauth/src/main/OAuthServer.scala index ece8c0dd6e..4bba9ec2fc 100644 --- a/modules/oauth/src/main/OAuthServer.scala +++ b/modules/oauth/src/main/OAuthServer.scala @@ -52,13 +52,13 @@ final class OAuthServer( ) private def fetchAccessToken(tokenId: AccessToken.Id): Fu[Option[AccessToken.ForAuth]] = - tokenColl.findAndUpdate( + tokenColl.ext.findAndUpdate( selector = $doc(F.id -> tokenId), update = $set(F.usedAt -> DateTime.now), fields = AccessToken.forAuthProjection.some ).map(_.value) map { - _ ?? AccessToken.ForAuthBSONReader.readOpt - } + _ ?? AccessToken.ForAuthBSONReader.readOpt + } } object OAuthServer { diff --git a/modules/security/src/main/AutomaticEmail.scala b/modules/security/src/main/AutomaticEmail.scala index 92ed04e68b..3982e035ff 100644 --- a/modules/security/src/main/AutomaticEmail.scala +++ b/modules/security/src/main/AutomaticEmail.scala @@ -7,6 +7,7 @@ import lila.i18n.I18nKeys.{ emails => trans } import lila.user.{ User, UserRepo } final class AutomaticEmail( + userRepo: UserRepo, mailgun: Mailgun, baseUrl: String ) { @@ -31,8 +32,8 @@ ${Mailgun.txt.serviceNote} } def onTitleSet(username: String)(implicit lang: Lang): Funit = for { - user <- UserRepo named username flatten s"No such user $username" - emailOption <- UserRepo email user.id + user <- userRepo named username orFail s"No such user $username" + emailOption <- userRepo email user.id } yield for { title <- user.title email <- emailOption @@ -62,7 +63,7 @@ ${Mailgun.txt.serviceNote} } def onBecomeCoach(user: User)(implicit lang: Lang): Funit = - UserRepo email user.id flatMap { + userRepo email user.id flatMap { _ ?? { email => val body = s"""Hello, @@ -88,8 +89,8 @@ ${Mailgun.txt.serviceNote} } def onFishnetKey(userId: User.ID, key: String)(implicit lang: Lang): Funit = for { - user <- UserRepo named userId flatten s"No such user $userId" - emailOption <- UserRepo email user.id + user <- userRepo named userId orFail s"No such user $userId" + emailOption <- userRepo email user.id } yield emailOption ?? { email => val body = s"""Hello, diff --git a/modules/security/src/main/CheckMail.scala b/modules/security/src/main/CheckMail.scala index b54549a07b..3f874938bf 100644 --- a/modules/security/src/main/CheckMail.scala +++ b/modules/security/src/main/CheckMail.scala @@ -3,9 +3,9 @@ package lila.security import scala.concurrent.duration._ import play.api.libs.json._ -import play.api.libs.ws.{ WS, WSResponse } -import play.api.Play.current +import play.api.libs.ws.WSClient +import lila.common.config.Secret import lila.common.Domain import lila.db.dsl._ @@ -13,13 +13,13 @@ import lila.db.dsl._ * Only hit after trying everything else (DnsApi) * and save the result forever. */ private final class CheckMail( - url: String, - key: String, + ws: WSClient, + config: SecurityConfig.CheckMail, mongoCache: lila.memo.MongoCache.Builder )(implicit system: akka.actor.ActorSystem) { def apply(domain: Domain.Lower): Fu[Boolean] = - if (key.isEmpty) fuccess(true) + if (config.key.value.isEmpty) fuccess(true) else cache(domain).withTimeoutDefault(2.seconds, true) recover { case e: Exception => lila.mon.security.checkMailApi.error() @@ -27,12 +27,12 @@ private final class CheckMail( true } - private[security] def fetchAllBlocked: Fu[List[String]] = cache.coll.distinct[String, List]( + private[security] def fetchAllBlocked: Fu[List[String]] = cache.coll.distinctEasy[String, List]( "_id", $doc( "_id" $regex s"^$prefix:", "v" -> false - ).some + ) ) map { ids => val dropSize = prefix.size + 1 ids.map(_ drop dropSize) @@ -48,9 +48,9 @@ private final class CheckMail( ) private def fetch(domain: Domain.Lower): Fu[Boolean] = - WS.url(url) - .withQueryString("domain" -> domain.value, "disable_test_connection" -> "true") - .withHeaders("x-rapidapi-key" -> key) + ws.url(config.url) + .withQueryStringParameters("domain" -> domain.value, "disable_test_connection" -> "true") + .withHttpHeaders("x-rapidapi-key" -> config.key.value) .get withTimeout 15.seconds map { case res if res.status == 200 => val valid = ~(res.json \ "valid").asOpt[Boolean] @@ -63,6 +63,6 @@ private final class CheckMail( if (!ok) lila.mon.security.checkMailApi.block() ok case res => - throw lila.base.LilaException(s"$url $domain ${res.status} ${res.body take 200}") + throw lila.base.LilaException(s"${config.url} $domain ${res.status} ${res.body take 200}") } } diff --git a/modules/security/src/main/Cli.scala b/modules/security/src/main/Cli.scala index 5bd2ba1e9a..9547f7f8ca 100644 --- a/modules/security/src/main/Cli.scala +++ b/modules/security/src/main/Cli.scala @@ -2,22 +2,22 @@ package lila.security import lila.user.{ User, UserRepo } -private[security] final class Cli extends lila.common.Cli { +private[security] final class Cli(userRepo: UserRepo) extends lila.common.Cli { def process = { case "security" :: "roles" :: uid :: Nil => - UserRepo named uid map { + userRepo named uid map { _.fold("User %s not found" format uid)(_.roles mkString " ") } case "security" :: "grant" :: uid :: roles => perform(uid, user => - UserRepo.setRoles(user.id, roles map (_.toUpperCase)).void) + userRepo.setRoles(user.id, roles map (_.toUpperCase)).void) } private def perform(username: String, op: User => Funit): Fu[String] = - UserRepo named username flatMap { userOption => + userRepo named username flatMap { userOption => userOption.fold(fufail[String]("User %s not found" format username)) { u => op(u) inject "User %s successfully updated".format(username) } diff --git a/modules/security/src/main/DataForm.scala b/modules/security/src/main/DataForm.scala index 9928023394..86fc935e0b 100644 --- a/modules/security/src/main/DataForm.scala +++ b/modules/security/src/main/DataForm.scala @@ -9,6 +9,7 @@ import lila.user.{ User, TotpSecret, UserRepo } import User.{ ClearPassword, TotpToken } final class DataForm( + userRepo: UserRepo, val captcher: akka.actor.ActorSelection, authenticator: lila.user.Authenticator, emailValidator: EmailAddressValidator @@ -61,7 +62,7 @@ final class DataForm( error = "usernameCharsInvalid" ) ).verifying("usernameUnacceptable", u => !LameName.username(u)) - .verifying("usernameAlreadyUsed", u => !UserRepo.nameExists(u).awaitSeconds(4)) + .verifying("usernameAlreadyUsed", u => !userRepo.nameExists(u).awaitSeconds(4)) private val agreementBool = boolean.verifying(b => b) diff --git a/modules/security/src/main/DisposableEmailDomain.scala b/modules/security/src/main/DisposableEmailDomain.scala index 2eb58f2166..d86ed7d149 100644 --- a/modules/security/src/main/DisposableEmailDomain.scala +++ b/modules/security/src/main/DisposableEmailDomain.scala @@ -1,11 +1,11 @@ package lila.security -import play.api.libs.ws.WS -import play.api.Play.current +import play.api.libs.ws.WSClient import lila.common.Domain final class DisposableEmailDomain( + ws: WSClient, providerUrl: String, checkMailBlocked: () => Fu[List[String]] ) { @@ -15,14 +15,14 @@ final class DisposableEmailDomain( private var regex = finalizeRegex(staticRegex) private[security] def refresh: Unit = for { - blacklist <- WS.url(providerUrl).get().map(_.body.lines) recover { + blacklist <- ws.url(providerUrl).get().map(_.body.linesIterator) recover { case e: Exception => logger.warn("DisposableEmailDomain.refresh", e) Iterator.empty } checked <- checkMailBlocked() } { - val regexStr = s"${toRegexStr(blacklist)}|${toRegexStr(checked.toIterator)}" + val regexStr = s"${toRegexStr(blacklist)}|${toRegexStr(checked.iterator)}" val nbDomains = regexStr.count('|' ==) lila.mon.email.disposableDomain(nbDomains) regex = finalizeRegex(s"$staticRegex|$regexStr") diff --git a/modules/security/src/main/DnsApi.scala b/modules/security/src/main/DnsApi.scala index c84acf2de9..17a0e97e25 100644 --- a/modules/security/src/main/DnsApi.scala +++ b/modules/security/src/main/DnsApi.scala @@ -2,16 +2,15 @@ package lila.security import com.github.blemale.scaffeine.{ AsyncLoadingCache, Scaffeine } import play.api.libs.json._ -import play.api.libs.ws.{ WS, WSResponse } -import play.api.Play.current +import play.api.libs.ws.WSClient import scala.concurrent.duration._ import lila.base.LilaException import lila.common.{ Chronometer, Domain } private final class DnsApi( - resolverUrl: String, - fetchTimeout: FiniteDuration + ws: WSClient, + config: SecurityConfig.DnsApi )(implicit system: akka.actor.ActorSystem) { // only valid email domains that are not whitelisted should make it here @@ -38,10 +37,10 @@ private final class DnsApi( }) private def fetch[A](domain: Domain.Lower, tpe: String)(f: List[JsObject] => A): Fu[A] = - WS.url(resolverUrl) - .withQueryString("name" -> domain.value, "type" -> tpe) - .withHeaders("Accept" -> "application/dns-json") - .get withTimeout fetchTimeout map { + ws.url(config.url) + .withQueryStringParameters("name" -> domain.value, "type" -> tpe) + .withHttpHeaders("Accept" -> "application/dns-json") + .get withTimeout config.timeout map { case res if res.status == 200 || res.status == 404 => f(~(res.json \ "Answer").asOpt[List[JsObject]]) case res => throw LilaException(s"Status ${res.status}") } diff --git a/modules/security/src/main/EmailAddressValidator.scala b/modules/security/src/main/EmailAddressValidator.scala index 765ecf7124..4aeee8739a 100644 --- a/modules/security/src/main/EmailAddressValidator.scala +++ b/modules/security/src/main/EmailAddressValidator.scala @@ -4,12 +4,13 @@ import play.api.data.validation._ import scala.concurrent.duration._ import lila.common.{ EmailAddress, Domain } -import lila.user.User +import lila.user.{ User, UserRepo } /** * Validate and normalize emails */ final class EmailAddressValidator( + userRepo: UserRepo, disposable: DisposableEmailDomain, dnsApi: DnsApi, checkMail: CheckMail @@ -29,14 +30,14 @@ final class EmailAddressValidator( * @return */ private def isTakenBySomeoneElse(email: EmailAddress, forUser: Option[User]): Fu[Boolean] = - lila.user.UserRepo.idByEmail(email.normalize) map (_ -> forUser) map { + userRepo.idByEmail(email.normalize) map (_ -> forUser) map { case (None, _) => false case (Some(userId), Some(user)) => userId != user.id case (_, _) => true } private def wasUsedTwiceRecently(email: EmailAddress): Fu[Boolean] = - lila.user.UserRepo.countRecentByPrevEmail(email.normalize).map(1<) + userRepo.countRecentByPrevEmail(email.normalize).map(1<) val acceptableConstraint = Constraint[String]("constraint.email_acceptable") { e => if (isAcceptable(EmailAddress(e))) Valid diff --git a/modules/security/src/main/EmailChange.scala b/modules/security/src/main/EmailChange.scala index 6c50e1a3ec..43df7224ba 100644 --- a/modules/security/src/main/EmailChange.scala +++ b/modules/security/src/main/EmailChange.scala @@ -2,14 +2,16 @@ package lila.security import scalatags.Text.all._ +import lila.common.config._ import lila.common.{ Lang, EmailAddress } import lila.i18n.I18nKeys.{ emails => trans } import lila.user.{ User, UserRepo } final class EmailChange( + userRepo: UserRepo, mailgun: Mailgun, - baseUrl: String, - tokenerSecret: String + baseUrl: BaseUrl, + tokenerSecret: Secret ) { import Mailgun.html._ @@ -45,7 +47,7 @@ ${Mailgun.txt.serviceNote} tokener read token map (_.flatten) flatMap { _ ?? { case TokenPayload(userId, email) => - UserRepo.setEmail(userId, email).nevermind >> UserRepo.byId(userId) + userRepo.setEmail(userId, email).nevermind >> userRepo.byId(userId) } } @@ -65,7 +67,7 @@ ${Mailgun.txt.serviceNote} private val tokener = new StringToken[Option[TokenPayload]]( secret = tokenerSecret, getCurrentValue = p => p ?? { - case TokenPayload(userId, _) => UserRepo email userId map (_.??(_.value)) + case TokenPayload(userId, _) => userRepo email userId map (_.??(_.value)) } ) } diff --git a/modules/security/src/main/EmailConfirm.scala b/modules/security/src/main/EmailConfirm.scala index fc48709b88..ee0a54576d 100644 --- a/modules/security/src/main/EmailConfirm.scala +++ b/modules/security/src/main/EmailConfirm.scala @@ -2,7 +2,8 @@ package lila.security import scalatags.Text.all._ -import lila.common.{ Lang, EmailAddress } +import lila.common.config._ +import lila.common.{ Lang, EmailAddress, LilaCookie } import lila.i18n.I18nKeys.{ emails => trans } import lila.user.{ User, UserRepo } @@ -15,19 +16,20 @@ trait EmailConfirm { def confirm(token: String): Fu[EmailConfirm.Result] } -object EmailConfirmSkip extends EmailConfirm { +final class EmailConfirmSkip(userRepo: UserRepo) extends EmailConfirm { def effective = false - def send(user: User, email: EmailAddress)(implicit lang: Lang) = UserRepo setEmailConfirmed user.id void + def send(user: User, email: EmailAddress)(implicit lang: Lang) = userRepo setEmailConfirmed user.id void def confirm(token: String): Fu[EmailConfirm.Result] = fuccess(EmailConfirm.Result.NotFound) } final class EmailConfirmMailgun( + userRepo: UserRepo, mailgun: Mailgun, - baseUrl: String, - tokenerSecret: String + baseUrl: BaseUrl, + tokenerSecret: Secret ) extends EmailConfirm { import Mailgun.html._ @@ -70,11 +72,11 @@ ${trans.emailConfirm_ignore.literalTxtTo(lang, List("https://lichess.org"))} import EmailConfirm.Result def confirm(token: String): Fu[Result] = tokener read token flatMap { - _ ?? UserRepo.enabledById + _ ?? userRepo.enabledById } flatMap { _.fold[Fu[Result]](fuccess(Result.NotFound)) { user => - UserRepo.mustConfirmEmail(user.id) flatMap { - case true => (UserRepo setEmailConfirmed user.id) inject Result.JustConfirmed(user) + userRepo.mustConfirmEmail(user.id) flatMap { + case true => (userRepo setEmailConfirmed user.id) inject Result.JustConfirmed(user) case false => fuccess(Result.AlreadyConfirmed(user)) } } @@ -82,7 +84,7 @@ ${trans.emailConfirm_ignore.literalTxtTo(lang, List("https://lichess.org"))} private val tokener = new StringToken[User.ID]( secret = tokenerSecret, - getCurrentValue = id => UserRepo email id map (_.??(_.value)) + getCurrentValue = id => userRepo email id map (_.??(_.value)) ) } @@ -97,7 +99,7 @@ object EmailConfirm { case class UserEmail(username: String, email: EmailAddress) - object cookie { + final class CookieApi(lilaCookie: LilaCookie) { import play.api.mvc.{ Cookie, RequestHeader } @@ -108,10 +110,11 @@ object EmailConfirm { case Array(username, email) => UserEmail(username, EmailAddress(email)) } - def make(user: User, email: EmailAddress)(implicit req: RequestHeader): Cookie = lila.common.LilaCookie.session( - name = name, - value = s"${user.username}$sep${email.value}" - ) + def make(user: User, email: EmailAddress)(implicit req: RequestHeader): Cookie = + lilaCookie.session( + name = name, + value = s"${user.username}$sep${email.value}" + ) } import scala.concurrent.duration._ @@ -171,11 +174,11 @@ object EmailConfirm { )) ) - def getStatus(username: String): Fu[Status] = UserRepo withEmails username flatMap { + def getStatus(userRepo: UserRepo, username: String): Fu[Status] = userRepo withEmails username flatMap { case None => fuccess(NoSuchUser(username)) case Some(User.WithEmails(user, emails)) => if (!user.enabled) fuccess(Closed(username)) - else UserRepo mustConfirmEmail user.id map { + else userRepo mustConfirmEmail user.id map { case true => emails.current match { case None => NoEmail(user.username) case Some(email) => EmailSent(user.username, email) diff --git a/modules/security/src/main/Env.scala b/modules/security/src/main/Env.scala index 9d2805018d..8374ae8c76 100644 --- a/modules/security/src/main/Env.scala +++ b/modules/security/src/main/Env.scala @@ -1,99 +1,63 @@ package lila.security -import lila.common.{ EmailAddress, Strings, Iso } +import akka.actor._ +import com.softwaremill.macwire._ +import io.methvin.play.autoconfig._ +import play.api.Configuration +import play.api.libs.ws.WSClient +import scala.concurrent.duration._ + +import lila.common.config._ +import lila.common.{ Bus, Strings, Iso, EmailAddress } import lila.memo.SettingStore.Formable.stringsFormable import lila.memo.SettingStore.Strings._ import lila.oauth.OAuthServer - -import akka.actor._ -import com.typesafe.config.Config -import scala.concurrent.duration._ -import lila.common.Bus +import lila.user.{ UserRepo, Authenticator } final class Env( - config: Config, + appConfig: Configuration, + ws: WSClient, captcher: ActorSelection, - authenticator: lila.user.Authenticator, + userRepo: UserRepo, + authenticator: Authenticator, slack: lila.slack.SlackApi, asyncCache: lila.memo.AsyncCache.Builder, settingStore: lila.memo.SettingStore.Builder, tryOAuthServer: OAuthServer.Try, mongoCache: lila.memo.MongoCache.Builder, system: ActorSystem, - scheduler: lila.common.Scheduler, + scheduler: Scheduler, db: lila.db.Env, lifecycle: play.api.inject.ApplicationLifecycle ) { - private val MailgunApiUrl = config getString "mailgun.api.url" - private val MailgunApiKey = config getString "mailgun.api.key" - private val MailgunSender = config getString "mailgun.sender" - private val MailgunReplyTo = config getString "mailgun.reply_to" - private val CollectionSecurity = config getString "collection.security" - private val CollectionPrintBan = config getString "collection.print_ban" - private val FirewallCookieName = config getString "firewall.cookie.name" - private val FirewallCookieEnabled = config getBoolean "firewall.cookie.enabled" - private val FirewallCollectionFirewall = config getString "firewall.collection.firewall" - private val FloodDuration = config duration "flood.duration" - private val GeoIPFile = config getString "geoip.file" - private val GeoIPCacheTtl = config duration "geoip.cache_ttl" - private val EmailConfirmSecret = config getString "email_confirm.secret" - private val EmailConfirmEnabled = config getBoolean "email_confirm.enabled" - private val PasswordResetSecret = config getString "password_reset.secret" - private val EmailChangeSecret = config getString "email_change.secret" - private val LoginTokenSecret = config getString "login_token.secret" - private val TorProviderUrl = config getString "tor.provider_url" - private val TorRefreshDelay = config duration "tor.refresh_delay" - private val DisposableEmailProviderUrl = config getString "disposable_email.provider_url" - private val DisposableEmailRefreshDelay = config duration "disposable_email.refresh_delay" - private val RecaptchaPrivateKey = config getString "recaptcha.private_key" - private val RecaptchaEndpoint = config getString "recaptcha.endpoint" - private val NetBaseUrl = config getString "net.base_url" - private val NetDomain = config getString "net.domain" - private val IpIntelEmail = EmailAddress(config getString "ipintel.email") - private val DnsApiUrl = config getString "dns_api.url" - private val DnsApiTimeout = config duration "dns_api.timeout" - private val CheckMailUrl = config getString "check_mail_api.url" - private val CheckMailKey = config getString "check_mail_api.key" + private val config = appConfig.get[SecurityConfig.Root]("security")(SecurityConfig.loader) - val recaptchaPublicConfig = RecaptchaPublicConfig( - key = config getString "recaptcha.public_key", - enabled = config getBoolean "recaptcha.enabled" - ) + val recaptchaPublicConfig = config.recaptcha.public lazy val firewall = new Firewall( - coll = firewallColl, - cookieName = FirewallCookieName.some filter (_ => FirewallCookieEnabled), - system = system + coll = db(config.collection.firewall), + scheduler = scheduler ) - lazy val flood = new Flood(FloodDuration) + lazy val flood = new Flood(config.floodDuration) lazy val recaptcha: Recaptcha = - if (recaptchaPublicConfig.enabled) new RecaptchaGoogle( - privateKey = RecaptchaPrivateKey, - endpoint = RecaptchaEndpoint, - lichessHostname = NetDomain - ) + if (recaptchaPublicConfig.enabled) wire[RecaptchaGoogle] else RecaptchaSkip - lazy val forms = new DataForm( - captcher = captcher, - authenticator = authenticator, - emailValidator = emailAddressValidator - ) + lazy val forms = wire[DataForm] - lazy val geoIP = new GeoIP( - file = GeoIPFile, - cacheTtl = GeoIPCacheTtl - ) + lazy val geoIP = wire[GeoIP] - lazy val userSpyApi = new UserSpyApi(firewall, geoIP, storeColl) - def userSpy = userSpyApi.apply _ + lazy val userSpyApi = wire[UserSpyApi] - def store = Store + lazy val store = new Store(db(config.collection.security)) - lazy val ipIntel = new IpIntel(asyncCache, IpIntelEmail) + lazy val ipIntel = { + def mk = (email: EmailAddress) => wire[IpIntel] + mk(config.ipIntelEmail) + } lazy val ugcArmedSetting = settingStore[Boolean]( "ugcArmed", @@ -101,24 +65,14 @@ final class Env( text = "Enable the user garbage collector".some ) - lazy val printBan = new PrintBan(printBanColl) + lazy val printBan = new PrintBan(db(config.collection.printBan)) - lazy val garbageCollector = new GarbageCollector( - userSpyApi, - ipTrust, - printBan, - slack, - ugcArmedSetting.get, - system - ) + lazy val garbageCollector = { + def mk = (isArmed: () => Boolean) => wire[GarbageCollector] + mk(ugcArmedSetting.get) + } - private lazy val mailgun = new Mailgun( - apiUrl = MailgunApiUrl, - apiKey = MailgunApiKey, - from = MailgunSender, - replyTo = MailgunReplyTo, - system = system - ) + private lazy val mailgun = wire[Mailgun] lazy val emailConfirm: EmailConfirm = if (EmailConfirmEnabled) new EmailConfirmMailgun( @@ -128,41 +82,34 @@ final class Env( ) else EmailConfirmSkip - lazy val passwordReset = new PasswordReset( - mailgun = mailgun, - baseUrl = NetBaseUrl, - tokenerSecret = PasswordResetSecret - ) + lazy val passwordReset = { + def mk = (u: BaseUrl, s: Secret) => wire[PasswordReset] + mk(config.net.baseUrl, config.passwordResetSecret) + } - lazy val magicLink = new MagicLink( - mailgun = mailgun, - baseUrl = NetBaseUrl, - tokenerSecret = PasswordResetSecret - ) + lazy val magicLink = { + def mk = (u: BaseUrl, s: Secret) => wire[MagicLink] + mk(config.net.baseUrl, config.passwordResetSecret) + } - lazy val emailChange = new EmailChange( - mailgun = mailgun, - baseUrl = NetBaseUrl, - tokenerSecret = EmailChangeSecret - ) + lazy val emailChange = { + def mk = (u: BaseUrl, s: Secret) => wire[EmailChange] + mk(config.net.baseUrl, config.emailChangeSecret) + } - lazy val loginToken = new LoginToken( - secret = LoginTokenSecret - ) + lazy val loginToken = new LoginToken(config.loginTokenSecret, userRepo) - lazy val automaticEmail = new AutomaticEmail( - mailgun = mailgun, - baseUrl = NetBaseUrl - ) + lazy val automaticEmail = wire[AutomaticEmail] - private lazy val dnsApi = new DnsApi(DnsApiUrl, DnsApiTimeout)(system) + private lazy val dnsApi = wire[DnsApi] - private lazy val checkMail = new CheckMail(CheckMailUrl, CheckMailKey, mongoCache)(system) + private lazy val checkMail: CheckMail = wire[CheckMail] - lazy val emailAddressValidator = new EmailAddressValidator(disposableEmailDomain, dnsApi, checkMail) + lazy val emailAddressValidator = wire[EmailAddressValidator] private lazy val disposableEmailDomain = new DisposableEmailDomain( - providerUrl = DisposableEmailProviderUrl, + ws = ws, + providerUrl = config.disposableEmail.providerUrl, checkMailBlocked = () => checkMail.fetchAllBlocked ) @@ -178,7 +125,7 @@ final class Env( lazy val spam = new Spam(spamKeywordsSetting.get) scheduler.once(30 seconds)(disposableEmailDomain.refresh) - scheduler.effect(DisposableEmailRefreshDelay, "Refresh disposable email domains")(disposableEmailDomain.refresh) + scheduler.effect(config.disposableEmail.refreshDelay, "Refresh disposable email domains")(disposableEmailDomain.refresh) lazy val tor = new Tor(TorProviderUrl) scheduler.once(31 seconds)(tor.refresh(_ => funit)) @@ -196,34 +143,4 @@ final class Env( case lila.hub.actorApi.fishnet.NewKey(userId, key) => automaticEmail.onFishnetKey(userId, key)(lila.i18n.defaultLang) } - - private[security] lazy val storeColl = db(CollectionSecurity) - private[security] lazy val printBanColl = db(CollectionPrintBan) - private[security] lazy val firewallColl = db(FirewallCollectionFirewall) -} - -object Env { - - private lazy val system = lila.common.PlayApp.system - - lazy val current = "security" boot new Env( - config = lila.common.PlayApp loadConfig "security", - db = lila.db.Env.current, - authenticator = lila.user.Env.current.authenticator, - slack = lila.slack.Env.current.api, - asyncCache = lila.memo.Env.current.asyncCache, - settingStore = lila.memo.Env.current.settingStore, - tryOAuthServer = () => scala.concurrent.Future { - lila.oauth.Env.current.server.some - }.withTimeoutDefault(50 millis, none)(system) recover { - case e: Exception => - lila.log("security").warn("oauth", e) - none - }, - mongoCache = lila.memo.Env.current.mongoCache, - system = system, - scheduler = lila.common.PlayApp.scheduler, - captcher = lila.hub.Env.current.captcher, - lifecycle = lila.common.PlayApp.lifecycle - ) } diff --git a/modules/security/src/main/Firewall.scala b/modules/security/src/main/Firewall.scala index 2b60b3ecf0..7402610e14 100644 --- a/modules/security/src/main/Firewall.scala +++ b/modules/security/src/main/Firewall.scala @@ -1,9 +1,9 @@ package lila.security -import scala.concurrent.duration._ - import org.joda.time.DateTime -import play.api.mvc.{ RequestHeader, Cookies } +import play.api.mvc.RequestHeader +import scala.concurrent.duration._ +import scala.concurrent.Future import lila.common.IpAddress import lila.db.BSON.BSONJodaDateTimeHandler @@ -11,48 +11,46 @@ import lila.db.dsl._ final class Firewall( coll: Coll, - cookieName: Option[String], - system: akka.actor.ActorSystem + scheduler: akka.actor.Scheduler ) { private var current: Set[String] = Set.empty - system.scheduler.scheduleOnce(10 minutes)(loadFromDb) + scheduler.scheduleOnce(10 minutes)(loadFromDb) def blocksIp(ip: IpAddress): Boolean = current contains ip.value def blocks(req: RequestHeader): Boolean = { val v = blocksIp { lila.common.HTTPRequest lastRemoteAddress req - } || cookieName.?? { blocksCookies(req.cookies, _) } + } if (v) lila.mon.security.firewall.block() v } def accepts(req: RequestHeader): Boolean = !blocks(req) - def blockIps(ips: List[IpAddress]): Funit = ips.map { ip => - validIp(ip) ?? { - coll.update( - $id(ip), - $doc("_id" -> ip, "date" -> DateTime.now), - upsert = true - ).void + def blockIps(ips: List[IpAddress]): Funit = Future.sequence { + ips.map { ip => + validIp(ip) ?? { + coll.update( + $id(ip), + $doc("_id" -> ip, "date" -> DateTime.now), + upsert = true + ).void + } } - }.sequenceFu >> loadFromDb + } >> loadFromDb def unblockIps(ips: Iterable[IpAddress]): Funit = coll.remove($inIds(ips.filter(validIp))).void >>- loadFromDb private def loadFromDb: Funit = - coll.distinct[String, Set]("_id", none).map { ips => + coll.distinctEasy[String, Set]("_id", $empty).map { ips => current = ips lila.mon.security.firewall.ip(ips.size) } - private def blocksCookies(cookies: Cookies, name: String) = - (cookies get name).isDefined - private def validIp(ip: IpAddress) = (IpAddress.isv4(ip) && ip.value != "127.0.0.1" && ip.value != "0.0.0.0") || (IpAddress.isv6(ip) && ip.value != "0:0:0:0:0:0:0:1" && ip.value != "0:0:0:0:0:0:0:0") diff --git a/modules/security/src/main/GeoIP.scala b/modules/security/src/main/GeoIP.scala index 4c3e1cd991..8ed6058a82 100644 --- a/modules/security/src/main/GeoIP.scala +++ b/modules/security/src/main/GeoIP.scala @@ -2,16 +2,17 @@ package lila.security import com.github.blemale.scaffeine.{ LoadingCache, Scaffeine } import com.sanoma.cda.geoip.{ MaxMindIpGeo, IpLocation } +import io.methvin.play.autoconfig._ import scala.concurrent.duration._ import lila.common.IpAddress -final class GeoIP(file: String, cacheTtl: FiniteDuration) { +final class GeoIP(config: GeoIP.Config) { - private val geoIp = MaxMindIpGeo(file, 0) + private val geoIp = MaxMindIpGeo(config.file, 0) private val cache: LoadingCache[IpAddress, Option[Location]] = Scaffeine() - .expireAfterAccess(cacheTtl) + .expireAfterAccess(config.cacheTtl) .build(compute) private def compute(ip: IpAddress): Option[Location] = @@ -22,6 +23,14 @@ final class GeoIP(file: String, cacheTtl: FiniteDuration) { def orUnknown(ip: IpAddress): Location = apply(ip) | Location.unknown } +object GeoIP { + case class Config( + file: String, + @ConfigName("cache_ttl") cacheTtl: FiniteDuration + ) + implicit val configLoader = AutoConfig.loader[Config] +} + case class Location( country: String, region: Option[String], diff --git a/modules/security/src/main/IpIntel.scala b/modules/security/src/main/IpIntel.scala index cc906ec79d..7959304d67 100644 --- a/modules/security/src/main/IpIntel.scala +++ b/modules/security/src/main/IpIntel.scala @@ -1,12 +1,15 @@ package lila.security -import play.api.libs.ws.WS -import play.api.Play.current +import play.api.libs.ws.WSClient import scala.concurrent.duration._ import lila.common.{ IpAddress, EmailAddress } -final class IpIntel(asyncCache: lila.memo.AsyncCache.Builder, contactEmail: EmailAddress) { +final class IpIntel( + ws: WSClient, + asyncCache: lila.memo.AsyncCache.Builder, + contactEmail: EmailAddress +) { def apply(ip: IpAddress): Fu[Int] = failable(ip) recover { case e: Exception => @@ -22,7 +25,7 @@ final class IpIntel(asyncCache: lila.memo.AsyncCache.Builder, contactEmail: Emai name = "ipIntel", f = ip => { val url = s"https://check.getipintel.net/check.php?ip=$ip&contact=${contactEmail.value}" - WS.url(url).get().map(_.body).mon(_.security.proxy.request.time).flatMap { str => + ws.url(url).get().map(_.body).mon(_.security.proxy.request.time).flatMap { str => str.toFloatOption.fold[Fu[Int]](fufail(s"Invalid ratio ${str.take(140)}")) { ratio => if (ratio < 0) fufail(s"IpIntel error $ratio on $url") else fuccess((ratio * 100).toInt) diff --git a/modules/security/src/main/LoginToken.scala b/modules/security/src/main/LoginToken.scala index bca42da07d..5526db7b6f 100644 --- a/modules/security/src/main/LoginToken.scala +++ b/modules/security/src/main/LoginToken.scala @@ -4,21 +4,22 @@ import org.joda.time.DateTime import scala.concurrent.duration._ import lila.common.Iso +import lila.common.config.Secret import lila.user.{ User, UserRepo } -final class LoginToken(secret: String) { +final class LoginToken(secret: Secret, userRepo: UserRepo) { def generate(user: User): Fu[String] = tokener make user.id def consume(token: String): Fu[Option[User]] = - tokener read token flatMap { _ ?? UserRepo.byId } + tokener read token flatMap { _ ?? userRepo.byId } private val tokener = LoginToken.makeTokener(secret, 1 minute) } private object LoginToken { - def makeTokener(secret: String, lifetime: FiniteDuration) = new StringToken[User.ID]( + def makeTokener(secret: Secret, lifetime: FiniteDuration) = new StringToken[User.ID]( secret = secret, getCurrentValue = _ => fuccess(DateStr toStr DateTime.now), currentValueHashSize = none, diff --git a/modules/security/src/main/MagicLink.scala b/modules/security/src/main/MagicLink.scala index 214fa6b506..d56901003f 100644 --- a/modules/security/src/main/MagicLink.scala +++ b/modules/security/src/main/MagicLink.scala @@ -4,13 +4,15 @@ import scala.concurrent.duration._ import scalatags.Text.all._ import lila.common.{ Lang, EmailAddress } +import lila.common.config._ import lila.i18n.I18nKeys.{ emails => trans } import lila.user.{ User, UserRepo } final class MagicLink( mailgun: Mailgun, - baseUrl: String, - tokenerSecret: String + userRepo: UserRepo, + baseUrl: BaseUrl, + tokenerSecret: Secret ) { import Mailgun.html._ @@ -40,7 +42,7 @@ ${Mailgun.txt.serviceNote} } def confirm(token: String): Fu[Option[User]] = - tokener read token flatMap { _ ?? UserRepo.byId } + tokener read token flatMap { _ ?? userRepo.byId } private val tokener = LoginToken.makeTokener(tokenerSecret, 10 minutes) } diff --git a/modules/security/src/main/Mailgun.scala b/modules/security/src/main/Mailgun.scala index e0e3b72899..d9514aeff8 100644 --- a/modules/security/src/main/Mailgun.scala +++ b/modules/security/src/main/Mailgun.scala @@ -3,58 +3,64 @@ package lila.security import scala.concurrent.duration.{ span => _, _ } import akka.actor.ActorSystem -import play.api.libs.ws.{ WS, WSAuthScheme } -import play.api.Play.current +import io.methvin.play.autoconfig._ +import play.api.libs.ws.{ WSClient, WSAuthScheme } import scalatags.Text.all._ -import lila.common.String.html.escapeHtml -import lila.common.String.html.nl2brUnsafe +import lila.common.config.Secret +import lila.common.String.html.{ escapeHtml, nl2brUnsafe } import lila.common.{ Lang, EmailAddress } import lila.i18n.I18nKeys.{ emails => trans } final class Mailgun( - apiUrl: String, - apiKey: String, - from: String, - replyTo: String, - system: ActorSystem -) { + ws: WSClient, + config: Mailgun.Config +)(implicit system: ActorSystem) { def send(msg: Mailgun.Message): Funit = - if (apiUrl.isEmpty) { + if (config.apiUrl.isEmpty) { println(msg, "No mailgun API URL") funit } else { lila.mon.email.actions.send() - WS.url(s"$apiUrl/messages").withAuth("api", apiKey, WSAuthScheme.BASIC).post(Map( - "from" -> Seq(msg.from | from), - "to" -> Seq(msg.to.value), - "h:Reply-To" -> Seq(msg.replyTo | replyTo), - "o:tag" -> msg.tag.toSeq, - "subject" -> Seq(msg.subject), - "text" -> Seq(msg.text) - ) ++ msg.htmlBody.?? { body => - Map("html" -> Seq(Mailgun.html.wrap(msg.subject, body).render)) - }).void addFailureEffect { - case e: java.net.ConnectException => lila.mon.http.mailgun.timeout() - case _ => - } recoverWith { - case e if msg.retriesLeft > 0 => { - lila.mon.email.actions.retry() - akka.pattern.after(15 seconds, system.scheduler) { - send(msg.copy(retriesLeft = msg.retriesLeft - 1)) + ws.url(s"${config.apiUrl}/messages") + .withAuth("api", config.apiKey.value, WSAuthScheme.BASIC).post(Map( + "from" -> Seq(msg.from | config.sender), + "to" -> Seq(msg.to.value), + "h:Reply-To" -> Seq(msg.replyTo | config.replyTo), + "o:tag" -> msg.tag.toSeq, + "subject" -> Seq(msg.subject), + "text" -> Seq(msg.text) + ) ++ msg.htmlBody.?? { body => + Map("html" -> Seq(Mailgun.html.wrap(msg.subject, body).render)) + }).void addFailureEffect { + case e: java.net.ConnectException => lila.mon.http.mailgun.timeout() + case _ => + } recoverWith { + case e if msg.retriesLeft > 0 => { + lila.mon.email.actions.retry() + akka.pattern.after(15 seconds, system.scheduler) { + send(msg.copy(retriesLeft = msg.retriesLeft - 1)) + } + } + case e => { + lila.mon.email.actions.fail() + fufail(e) } } - case e => { - lila.mon.email.actions.fail() - fufail(e) - } - } } } object Mailgun { + case class Config( + @ConfigName("api.url") apiUrl: String, + @ConfigName("api.key") apiKey: Secret, + sender: String, + @ConfigName("reply_to") replyTo: String + ) + implicit val configLoader = AutoConfig.loader[Config] + case class Message( to: EmailAddress, subject: String, diff --git a/modules/security/src/main/PasswordReset.scala b/modules/security/src/main/PasswordReset.scala index 19d5c7ff5a..538e3974c7 100644 --- a/modules/security/src/main/PasswordReset.scala +++ b/modules/security/src/main/PasswordReset.scala @@ -3,13 +3,15 @@ package lila.security import scalatags.Text.all._ import lila.common.{ Lang, EmailAddress } +import lila.common.config._ import lila.i18n.I18nKeys.{ emails => trans } import lila.user.{ User, UserRepo } final class PasswordReset( mailgun: Mailgun, - baseUrl: String, - tokenerSecret: String + userRepo: UserRepo, + baseUrl: BaseUrl, + tokenerSecret: Secret ) { import Mailgun.html._ @@ -42,10 +44,10 @@ ${Mailgun.txt.serviceNote} } def confirm(token: String): Fu[Option[User]] = - tokener read token flatMap { _ ?? UserRepo.byId } + tokener read token flatMap { _ ?? userRepo.byId } private val tokener = new StringToken[User.ID]( secret = tokenerSecret, - getCurrentValue = id => UserRepo getPasswordHash id map (~_) + getCurrentValue = id => userRepo getPasswordHash id map (~_) ) } diff --git a/modules/security/src/main/PrintBan.scala b/modules/security/src/main/PrintBan.scala index 3d646b4d1f..b0f0fd5f53 100644 --- a/modules/security/src/main/PrintBan.scala +++ b/modules/security/src/main/PrintBan.scala @@ -22,7 +22,7 @@ final class PrintBan(coll: Coll) { } >> loadFromDb private def loadFromDb: Funit = - coll.distinct[String, Set]("_id", none).map { hashes => + coll.distinctEasy[String, Set]("_id", $empty).map { hashes => current = hashes lila.mon.security.firewall.prints(hashes.size) } diff --git a/modules/security/src/main/Recaptcha.scala b/modules/security/src/main/Recaptcha.scala index a868c3e634..fe89ac2b4e 100644 --- a/modules/security/src/main/Recaptcha.scala +++ b/modules/security/src/main/Recaptcha.scala @@ -1,26 +1,41 @@ package lila.security -import play.api.libs.ws.WS -import play.api.mvc.RequestHeader -import play.api.Play.current import play.api.libs.json._ +import play.api.libs.ws.WSClient +import play.api.mvc.RequestHeader +import io.methvin.play.autoconfig._ import lila.common.HTTPRequest +import lila.common.config.Secret trait Recaptcha { def verify(response: String, req: RequestHeader): Fu[Boolean] } +private object Recaptcha { + + case class Config( + endpoint: String, + @ConfigName("public_key") publicKey: String, + @ConfigName("private_key") privateKey: Secret, + enabled: Boolean, + netDomain: String + ) { + def public = RecaptchaPublicConfig(publicKey, enabled) + } + implicit val configLoader = AutoConfig.loader[Config] +} + object RecaptchaSkip extends Recaptcha { def verify(response: String, req: RequestHeader) = fuTrue + } final class RecaptchaGoogle( - endpoint: String, - privateKey: String, - lichessHostname: String + ws: WSClient, + config: Recaptcha.Config ) extends Recaptcha { private case class Response( @@ -31,18 +46,18 @@ final class RecaptchaGoogle( private implicit val responseReader = Json.reads[Response] def verify(response: String, req: RequestHeader): Fu[Boolean] = { - WS.url(endpoint).post(Map( - "secret" -> Seq(privateKey), - "response" -> Seq(response), - "remoteip" -> Seq(HTTPRequest lastRemoteAddress req value) + ws.url(config.endpoint).post(Map( + "secret" -> config.privateKey.value, + "response" -> response, + "remoteip" -> HTTPRequest.lastRemoteAddress(req).value )) flatMap { case res if res.status == 200 => res.json.validate[Response] match { case JsSuccess(res, _) => fuccess { - res.success && res.hostname == lichessHostname + res.success && res.hostname == config.netDomain } case JsError(err) => - fufail(s"$err ${~res.body.lines.toList.headOption}") + fufail(s"$err ${~res.body.linesIterator.to(LazyList).headOption}") } case res => fufail(s"${res.status} ${res.body}") } recover { diff --git a/modules/security/src/main/SecurityApi.scala b/modules/security/src/main/SecurityApi.scala index 22c17abd29..b1d3e27a5e 100644 --- a/modules/security/src/main/SecurityApi.scala +++ b/modules/security/src/main/SecurityApi.scala @@ -6,8 +6,8 @@ import play.api.data._ import play.api.data.Forms._ import play.api.data.validation.{ Constraint, Valid => FormValid, Invalid, ValidationError } import play.api.mvc.RequestHeader -import reactivemongo.api.ReadPreference import reactivemongo.api.bson._ +import reactivemongo.api.ReadPreference import scala.concurrent.duration._ import lila.common.{ ApiVersion, IpAddress, EmailAddress, HTTPRequest } @@ -18,8 +18,10 @@ import lila.user.{ User, UserRepo } import User.LoginCandidate final class SecurityApi( + userRepo: UserRepo, coll: Coll, firewall: Firewall, + store: Store, geoIP: GeoIP, authenticator: lila.user.Authenticator, emailValidator: EmailAddressValidator, @@ -69,25 +71,25 @@ final class SecurityApi( } def saveAuthentication(userId: User.ID, apiVersion: Option[ApiVersion])(implicit req: RequestHeader): Fu[String] = - UserRepo mustConfirmEmail userId flatMap { + userRepo mustConfirmEmail userId flatMap { case true => fufail(SecurityApi MustConfirmEmail userId) case false => val sessionId = Random secureString 22 - Store.save(sessionId, userId, req, apiVersion, up = true, fp = none) inject sessionId + store.save(sessionId, userId, req, apiVersion, up = true, fp = none) inject sessionId } def saveSignup(userId: User.ID, apiVersion: Option[ApiVersion], fp: Option[FingerPrint])(implicit req: RequestHeader): Funit = { val sessionId = Random secureString 22 - Store.save(s"SIG-$sessionId", userId, req, apiVersion, up = false, fp = fp) + store.save(s"SIG-$sessionId", userId, req, apiVersion, up = false, fp = fp) } def restoreUser(req: RequestHeader): Fu[Option[FingerPrintedUser]] = firewall.accepts(req) ?? { reqSessionId(req) ?? { sessionId => - Store userIdAndFingerprint sessionId flatMap { + store userIdAndFingerprint sessionId flatMap { _ ?? { d => - if (d.isOld) Store.setDateToNow(sessionId) - UserRepo byId d.user map { _ map { FingerPrintedUser(_, d.fp) } } + if (d.isOld) store.setDateToNow(sessionId) + userRepo byId d.user map { _ map { FingerPrintedUser(_, d.fp) } } } } } @@ -104,17 +106,17 @@ final class SecurityApi( } def locatedOpenSessions(userId: User.ID, nb: Int): Fu[List[LocatedSession]] = - Store.openSessions(userId, nb) map { + store.openSessions(userId, nb) map { _.map { session => LocatedSession(session, geoIP(session.ip)) } } def dedup(userId: User.ID, req: RequestHeader): Funit = - reqSessionId(req) ?? { Store.dedup(userId, _) } + reqSessionId(req) ?? { store.dedup(userId, _) } def setFingerPrint(req: RequestHeader, fp: FingerPrint): Fu[Option[FingerHash]] = - reqSessionId(req) ?? { Store.setFingerPrint(_, fp) map some } + reqSessionId(req) ?? { store.setFingerPrint(_, fp) map some } val sessionIdKey = "sessionId" @@ -123,24 +125,22 @@ final class SecurityApi( def userIdsSharingIp = userIdsSharingField("ip") _ - def recentByIpExists(ip: IpAddress): Fu[Boolean] = Store recentByIpExists ip + def recentByIpExists(ip: IpAddress): Fu[Boolean] = store recentByIpExists ip - def recentByPrintExists(fp: FingerPrint): Fu[Boolean] = Store recentByPrintExists fp + def recentByPrintExists(fp: FingerPrint): Fu[Boolean] = store recentByPrintExists fp private def userIdsSharingField(field: String)(userId: User.ID): Fu[List[User.ID]] = - coll.distinctWithReadPreference[User.ID, List]( + coll.secondaryPreferred.distinctEasy[User.ID, List]( field, - $doc("user" -> userId, field $exists true).some, - readPreference = ReadPreference.secondaryPreferred + $doc("user" -> userId, field $exists true) ).flatMap { case Nil => fuccess(Nil) - case values => coll.distinctWithReadPreference[User.ID, List]( + case values => coll.secondaryPreferred.distinctEasy[User.ID, List]( "user", $doc( field $in values, "user" $ne userId - ).some, - ReadPreference.secondaryPreferred + ) ) } @@ -149,7 +149,7 @@ final class SecurityApi( def recentUserIdsByIp(ip: IpAddress) = recentUserIdsByField("ip")(ip.value) def shareIpOrPrint(u1: User.ID, u2: User.ID): Fu[Boolean] = - Store.ipsAndFps(List(u1, u2), max = 100) map { ipsAndFps => + store.ipsAndFps(List(u1, u2), max = 100) map { ipsAndFps => val u1s: Set[String] = ipsAndFps.filter(_.user == u1).flatMap { x => List(x.ip.value, ~x.fp) }.toSet @@ -161,15 +161,15 @@ final class SecurityApi( } def printUas(fh: FingerHash): Fu[List[String]] = - coll.distinct[String, List]("ua", $doc("fp" -> fh.value).some) + coll.distinctEasy[String, List]("ua", $doc("fp" -> fh.value)) private def recentUserIdsByField(field: String)(value: String): Fu[List[User.ID]] = - coll.distinct[User.ID, List]( + coll.distinctEasy[User.ID, List]( "user", $doc( field -> value, "date" $gt DateTime.now.minusYears(1) - ).some + ) ) } diff --git a/modules/security/src/main/SecurityConfig.scala b/modules/security/src/main/SecurityConfig.scala new file mode 100644 index 0000000000..a232fefb50 --- /dev/null +++ b/modules/security/src/main/SecurityConfig.scala @@ -0,0 +1,68 @@ +package lila.security + +import io.methvin.play.autoconfig._ +import scala.concurrent.duration.FiniteDuration + +import lila.common.config._ +import lila.common.EmailAddress + +private object SecurityConfig { + + case class Root( + collection: Collection, + floodDuration: FiniteDuration, + geoIP: GeoIP.Config, + @ConfigName("password_reset.secret") passwordResetSecret: Secret, + emailConfirm: EmailConfirm, + @ConfigName("email_change.secret") emailChangeSecret: Secret, + @ConfigName("login_token.secret") loginTokenSecret: Secret, + tor: Tor, + @ConfigName("disposable_email") disposableEmail: DisposableEmail, + @ConfigName("dns_api") dnsApi: DnsApi, + @ConfigName("check_mail_api") checkMail: CheckMail, + recaptcha: Recaptcha.Config, + mailgun: Mailgun.Config, + net: NetConfig, + @ConfigName("ipintel.email") ipIntelEmail: EmailAddress + ) + + case class Collection( + security: CollName, + printBan: CollName, + firewall: CollName + ) + implicit val collectionLoader = AutoConfig.loader[Collection] + + case class EmailConfirm( + enabled: Boolean, + secret: Secret, + cookie: String + ) + implicit val emailConfirmLoader = AutoConfig.loader[EmailConfirm] + + case class Tor( + @ConfigName("provider_url") providerUrl: String, + @ConfigName("refresh_delay") refreshDelay: FiniteDuration + ) + implicit val torLoader = AutoConfig.loader[Tor] + + case class DisposableEmail( + @ConfigName("provider_url") providerUrl: String, + @ConfigName("refresh_delay") refreshDelay: FiniteDuration + ) + implicit val disposableLoader = AutoConfig.loader[DisposableEmail] + + case class DnsApi( + url: String, + timeout: FiniteDuration + ) + implicit val dnsLoader = AutoConfig.loader[DnsApi] + + case class CheckMail( + url: String, + key: Secret + ) + implicit val checkMailLoader = AutoConfig.loader[CheckMail] + + implicit val loader = AutoConfig.loader[Root] +} diff --git a/modules/security/src/main/Store.scala b/modules/security/src/main/Store.scala index 4bd0c27ca4..6cc6426c0b 100644 --- a/modules/security/src/main/Store.scala +++ b/modules/security/src/main/Store.scala @@ -2,24 +2,21 @@ package lila.security import org.joda.time.DateTime import play.api.mvc.RequestHeader -import reactivemongo.api.ReadPreference import reactivemongo.api.bson.Macros +import reactivemongo.api.ReadPreference import lila.common.{ HTTPRequest, ApiVersion, IpAddress } import lila.db.BSON.BSONJodaDateTimeHandler import lila.db.dsl._ import lila.user.User -object Store { +private final class Store(coll: Coll) { - // dirty - private val coll = Env.current.storeColl - - private implicit val fingerHashBSONHandler = stringIsoHandler[FingerHash] + import Store._ private val localhost = IpAddress("127.0.0.1") - private[security] def save( + def save( sessionId: String, userId: User.ID, req: RequestHeader, @@ -39,7 +36,7 @@ object Store { "date" -> DateTime.now, "up" -> up, "api" -> apiVersion.map(_.value), - "fp" -> fp.flatMap(FingerHash.apply) + "fp" -> fp.flatMap(FingerHash.apply).flatMap(fingerHashBSONHandler.writeOpt) )).void private val userIdFingerprintProjection = $doc( @@ -105,17 +102,6 @@ object Store { case Some(hash) => coll.updateField($doc("_id" -> id), "fp", hash) inject hash } - case class Info(ip: IpAddress, ua: String, fp: Option[FingerHash], date: DateTime) { - def datedIp = Dated(ip, date) - def datedFp = fp.map { Dated(_, date) } - def datedUa = Dated(ua, date) - } - private implicit val InfoReader = Macros.reader[Info] - - case class Dated[V](value: V, date: DateTime) extends Ordered[Dated[V]] { - def compare(other: Dated[V]) = other.date compareTo date - } - def chronoInfoByUser(userId: User.ID): Fu[List[Info]] = coll.find( $doc( @@ -123,7 +109,7 @@ object Store { "date" $gt DateTime.now.minusYears(2) ), $doc("_id" -> false, "ip" -> true, "ua" -> true, "fp" -> true, "date" -> true) - ).sort($sort desc "date").list[Info]() + ).sort($sort desc "date").list[Info]()(InfoReader) private case class DedupInfo(_id: String, ip: String, ua: String) { def compositeKey = s"$ip $ua" @@ -147,16 +133,30 @@ object Store { coll.find($doc("user" $in userIds)).list[IpAndFp](max, ReadPreference.secondaryPreferred) private[security] def recentByIpExists(ip: IpAddress): Fu[Boolean] = - coll.exists( - $doc("ip" -> ip, "date" -> $gt(DateTime.now minusDays 7)), - readPreference = ReadPreference.secondaryPreferred + coll.secondaryPreferred.exists( + $doc("ip" -> ip, "date" -> $gt(DateTime.now minusDays 7)) ) private[security] def recentByPrintExists(fp: FingerPrint): Fu[Boolean] = FingerHash(fp) ?? { hash => - coll.exists( - $doc("fp" -> hash, "date" -> $gt(DateTime.now minusDays 7)), - readPreference = ReadPreference.secondaryPreferred + coll.secondaryPreferred.exists( + $doc("fp" -> hash, "date" -> $gt(DateTime.now minusDays 7)) ) } } + +object Store { + + case class Dated[V](value: V, date: DateTime) extends Ordered[Dated[V]] { + def compare(other: Dated[V]) = other.date compareTo date + } + + case class Info(ip: IpAddress, ua: String, fp: Option[FingerHash], date: DateTime) { + def datedIp = Dated(ip, date) + def datedFp = fp.map { Dated(_, date) } + def datedUa = Dated(ua, date) + } + + implicit val fingerHashBSONHandler = stringIsoHandler[FingerHash] + implicit val InfoReader = Macros.reader[Info] +} diff --git a/modules/security/src/main/StringToken.scala b/modules/security/src/main/StringToken.scala index b462c9cfdc..9b71229118 100644 --- a/modules/security/src/main/StringToken.scala +++ b/modules/security/src/main/StringToken.scala @@ -1,13 +1,15 @@ package lila.security import com.roundeights.hasher.Algo -import lila.common.String.base64 import org.mindrot.BCrypt +import lila.common.String.base64 +import lila.common.config.Secret + import StringToken.ValueChecker private[security] final class StringToken[A]( - secret: String, + secret: Secret, getCurrentValue: A => Fu[String], valueChecker: ValueChecker = ValueChecker.Same, fullHashSize: Int = 14, @@ -36,7 +38,7 @@ private[security] final class StringToken[A]( } } - private def makeHash(msg: String) = Algo.hmac(secret).sha1(msg).hex take fullHashSize + private def makeHash(msg: String) = Algo.hmac(secret.value).sha1(msg).hex take fullHashSize private def hashCurrentValue(payload: A) = getCurrentValue(payload) map { v => currentValueHashSize.fold(v)(makeHash(v) take _) diff --git a/modules/security/src/main/Tor.scala b/modules/security/src/main/Tor.scala index 07541709bf..d193f0250c 100644 --- a/modules/security/src/main/Tor.scala +++ b/modules/security/src/main/Tor.scala @@ -2,16 +2,15 @@ package lila.security import lila.common.IpAddress -import play.api.libs.ws.WS -import play.api.Play.current +import play.api.libs.ws.WSClient -final class Tor(providerUrl: String) { +final class Tor(ws: WSClient, config: SecurityConfig.Tor) { private var ips = Set.empty[IpAddress] private[security] def refresh(withIps: Iterable[IpAddress] => Funit): Unit = { - WS.url(providerUrl).get() map { res => - ips = res.body.lines.filterNot(_ startsWith "#").map(IpAddress.apply).toSet + ws.url(config.providerUrl).get() map { res => + ips = res.body.linesIterator.filterNot(_ startsWith "#").map(IpAddress.apply).toSet withIps(ips) lila.mon.security.tor.node(ips.size) } diff --git a/modules/security/src/main/UserSpy.scala b/modules/security/src/main/UserSpy.scala index faafaa2c70..dab400145c 100644 --- a/modules/security/src/main/UserSpy.scala +++ b/modules/security/src/main/UserSpy.scala @@ -2,7 +2,6 @@ package lila.security import org.joda.time.DateTime import reactivemongo.api.ReadPreference -import scala.collection.breakOut import lila.common.{ IpAddress, EmailAddress } import lila.db.dsl._ @@ -34,12 +33,18 @@ case class UserSpy( def otherUserIds = otherUsers.map(_.user.id) } -final class UserSpyApi(firewall: Firewall, geoIP: GeoIP, coll: Coll) { +final class UserSpyApi( + firewall: Firewall, + store: Store, + userRepo: UserRepo, + geoIP: GeoIP, + coll: Coll +) { import UserSpy._ def apply(user: User): Fu[UserSpy] = for { - infos <- Store.chronoInfoByUser(user.id) + infos <- store.chronoInfoByUser(user.id) ips = distinctRecent(infos.map(_.datedIp)) prints = distinctRecent(infos.flatMap(_.datedFp)) sharingIp <- exploreSimilar("ip")(user) @@ -54,10 +59,10 @@ final class UserSpyApi(firewall: Firewall, geoIP: GeoIP, coll: Coll) { usersSharingFingerprint = sharingFingerprint ) - private[security] def userHasPrint(u: User): Fu[Boolean] = coll.exists( - $doc("user" -> u.id, "fp" $exists true), - readPreference = ReadPreference.secondaryPreferred - ) + private[security] def userHasPrint(u: User): Fu[Boolean] = + coll.secondaryPreferred.exists( + $doc("user" -> u.id, "fp" $exists true) + ) private def exploreSimilar(field: String)(user: User): Fu[Set[User]] = nextValues(field)(user.id) flatMap { nValues => @@ -69,34 +74,32 @@ final class UserSpyApi(firewall: Firewall, geoIP: GeoIP, coll: Coll) { $doc("user" -> userId), $doc(field -> true) ).list[Bdoc]() map { - _.flatMap(_.getAs[Value](field))(breakOut) + _.view.flatMap(_.getAsOpt[Value](field)).to(Set) } private def nextUsers(field: String)(values: Set[Value], user: User): Fu[Set[User]] = values.nonEmpty ?? { - coll.distinctWithReadPreference[String, Set]( + coll.secondaryPreferred.distinctEasy[String, Set]( "user", $doc( field $in values, "user" $ne user.id - ).some, - ReadPreference.secondaryPreferred + ) ) flatMap { userIds => - userIds.nonEmpty ?? (UserRepo byIds userIds) map (_.toSet) + userIds.nonEmpty ?? (userRepo byIds userIds) map (_.toSet) } } def getUserIdsWithSameIpAndPrint(userId: User.ID): Fu[Set[User.ID]] = for { ips <- nextValues("ip")(userId) fps <- nextValues("fp")(userId) - users <- (ips.nonEmpty && fps.nonEmpty) ?? coll.distinctWithReadPreference[User.ID, Set]( + users <- (ips.nonEmpty && fps.nonEmpty) ?? coll.secondaryPreferred.distinctEasy[User.ID, Set]( "user", $doc( "ip" $in ips, "fp" $in fps, "user" $ne userId - ).some, - ReadPreference.secondaryPreferred + ) ) } yield users } @@ -113,7 +116,7 @@ object UserSpy { all.foldLeft(Map.empty[V, DateTime]) { case (acc, Dated(v, _)) if acc.contains(v) => acc case (acc, Dated(v, date)) => acc + (v -> date) - }.map { case (v, date) => Dated(v, date) }(breakOut) + }.view.map { case (v, date) => Dated(v, date) }.to(List) type Value = String @@ -123,9 +126,9 @@ object UserSpy { def emailValueOf(u: User) = emails.get(u.id).map(_.value) } - def withMeSortedWithEmails(me: User, others: Set[OtherUser]): Fu[WithMeSortedWithEmails] = { + def withMeSortedWithEmails(userRepo: UserRepo, me: User, others: Set[OtherUser]): Fu[WithMeSortedWithEmails] = { val othersList = others.toList - lila.user.UserRepo.emailMap(me.id :: othersList.map(_.user.id)) map { emailMap => + userRepo.emailMap(me.id :: othersList.map(_.user.id)) map { emailMap => WithMeSortedWithEmails( (OtherUser(me, true, true) :: othersList).sortBy(-_.user.createdAt.getMillis), emailMap diff --git a/modules/user/src/main/Env.scala b/modules/user/src/main/Env.scala index 3d816f57b8..1a1923749f 100644 --- a/modules/user/src/main/Env.scala +++ b/modules/user/src/main/Env.scala @@ -9,7 +9,7 @@ import scala.concurrent.duration._ import lila.db.dsl.Coll import lila.common.config._ -import lila.common.{ CollName, Secret, MaxPerPage, LightUser } +import lila.common.{ MaxPerPage, LightUser } case class UserConfig( @ConfigName("paginator.max_per_page") paginatorMaxPerPage: MaxPerPage, diff --git a/modules/user/src/main/PasswordHasher.scala b/modules/user/src/main/PasswordHasher.scala index 7aad2cecfc..dae58d5f29 100644 --- a/modules/user/src/main/PasswordHasher.scala +++ b/modules/user/src/main/PasswordHasher.scala @@ -6,7 +6,7 @@ import javax.crypto.Cipher import javax.crypto.spec.{ IvParameterSpec, SecretKeySpec } import com.roundeights.hasher.Implicits._ -import lila.common.Secret +import lila.common.config.Secret /** * Encryption for bcrypt hashes. diff --git a/modules/user/src/main/UserRepo.scala b/modules/user/src/main/UserRepo.scala index 28671a5545..b961ba7262 100644 --- a/modules/user/src/main/UserRepo.scala +++ b/modules/user/src/main/UserRepo.scala @@ -5,7 +5,7 @@ import reactivemongo.api._ import reactivemongo.api.bson._ import reactivemongo.api.commands.GetLastError -import lila.common.{ CollName, ApiVersion, EmailAddress, NormalizedEmailAddress } +import lila.common.{ ApiVersion, EmailAddress, NormalizedEmailAddress } import lila.db.BSON.BSONJodaDateTimeHandler import lila.db.dsl._ import lila.rating.{ Perf, PerfType }