Merge branch 'master' of github.com:ornicar/lila into v2

* 'master' of github.com:ornicar/lila:
  user mod search: watchlist, regex, levenshtein (wip)
  allow email domains without a DNS A record - closes #4917
  move blacklisted email domains to
  disposable email domains
  remove .coach-intro
  lichess.redirect only handles local urls - closes lichess-org/api#13
  Remove incorrect caption until #4034 merges

app/controllers/Mod.scala

@@ -234,8 +234,13 @@ object Mod extends LilaController {
     }
   }
 
-  def search = Secure(_.UserSearch) { implicit ctx => me =>
-    searchTerm((~get("q")).trim)
+  def search = SecureBody(_.UserSearch) { implicit ctx => me =>
+    implicit def req = ctx.body
+    val f = lila.mod.UserSearch.form
+    f.bindFromRequest.fold(
+      err => BadRequest(html.mod.search(err, Nil)).fuccess,
+      query => Env.mod.search(query) map { html.mod.search(f.fill(query), _) }
+    )
   }
 
   protected[controllers] def searchTerm(query: String)(implicit ctx: Context) =
@@ -282,16 +287,17 @@ object Mod extends LilaController {
         val query = rawQuery.trim.split(' ').toList
         val email = query.headOption.map(EmailAddress.apply) flatMap Env.security.emailAddressValidator.validate
         val username = query lift 1
-        def tryWith(setEmail: EmailAddress, q: String): Fu[Option[Result]] = Env.mod.search(q) flatMap {
-          case List(user) => (!user.everLoggedIn).?? {
-            lila.mon.user.register.modConfirmEmail()
-            modApi.setEmail(me.id, user.id, setEmail)
-          } >>
-            UserRepo.email(user.id) map { email =>
-              Ok(html.mod.emailConfirm("", user.some, email)).some
-            }
-          case _ => fuccess(none)
-        }
+        def tryWith(setEmail: EmailAddress, q: String): Fu[Option[Result]] =
+          Env.mod.search(lila.mod.UserSearch.exact(q)) flatMap {
+            case List(UserModel.WithEmails(user, _)) => (!user.everLoggedIn).?? {
+              lila.mon.user.register.modConfirmEmail()
+              modApi.setEmail(me.id, user.id, setEmail)
+            } >>
+              UserRepo.email(user.id) map { email =>
+                Ok(html.mod.emailConfirm("", user.some, email)).some
+              }
+            case _ => fuccess(none)
+          }
         email.?? { em =>
           tryWith(em.acceptable, em.acceptable.value) orElse {
             username ?? { tryWith(em.acceptable, _) }

app/views/mod/search.scala.html

@@ -1,4 +1,4 @@
-@(query: String, users: List[User])(implicit ctx: Context)
+@(form: Form[_], users: List[User.WithEmails])(implicit ctx: Context)
 
 @base.layout(
 title = "Search users",
@@ -8,40 +8,10 @@ responsive = true) {
   @mod.menu("search").toHtml
   <div id="mod-search" class="page-menu__content box box-pad">
     <h1>Search users</h1>
-    <form class="search" action="@routes.Mod.search" method="GET">
-      <input name="q" placeholder="Search by IP, email, or username" value="@query" autofocus />
-    </form>
-    @if(users.nonEmpty) {
-    <table class="slist">
-      <thead>
-        <tr>
-          <th>User</th>
-          <th>Games</th>
-          <th>Marks</th>
-          <th>IPban</th>
-          <th>Closed</th>
-          <th>Created</th>
-          <th>Active</th>
-        </tr>
-      </thead>
-      <tbody>
-        @users.map { u =>
-        <tr>
-          <td>@userLink(u, withBestRating = true, params = "?mod")</td>
-          <td>@u.count.game.localize</td>
-          <td>
-            @if(u.engine){ENGINE}
-            @if(u.booster){BOOSTER}
-            @if(u.troll){SHADOWBAN}
-          </td>
-          <td>@if(u.ipBan){IPBAN}</td>
-          <td>@if(u.disabled){CLOSED}</td>
-          <td>@momentFromNow(u.createdAt)</td>
-          <td>@u.seenAt.map(momentFromNow(_))</td>
-        </tr>
-        }
-    </tbody></table>
-    }
-  </div>
-</main>
-}.toHtml
+  <form class="search" action="@routes.Mod.search" method="GET">
+    <input name="q" placeholder="Search by IP, email, or username" value="@form("q").value" />
+    @base.form.select(form("as"), lila.mod.UserSearch.asChoices)
+  </form>
+  @userTable(users)
+</div>
+}

app/views/mod/userTable.scala.html

@@ -0,0 +1,38 @@
+@(users: List[User.WithEmails])(implicit ctx: Context)
+@if(users.nonEmpty) {
+<table class="slist">
+  <thead>
+    <tr>
+      <th>User</th>
+      <th>Games</th>
+      <th>Marks</th>
+      <th>IPban</th>
+      <th>Closed</th>
+      <th>Created</th>
+      <th>Active</th>
+    </tr>
+  </thead>
+  <tbody>
+    @users.map {
+    case lila.user.User.WithEmails(u, emails) => {
+    <tr>
+      <td>
+        @userLink(u, withBestRating = true, params = "?mod")
+        <em>@emails.list.mkString(", ")</em>
+      </td>
+    </td>
+    <td>@u.count.game.localize</td>
+    <td>
+      @if(u.engine){ENGINE}
+      @if(u.booster){BOOSTER}
+      @if(u.troll){SHADOWBAN}
+    </td>
+    <td>@if(u.ipBan){IPBAN}</td>
+    <td>@if(u.disabled){CLOSED}</td>
+    <td>@momentFromNow(u.createdAt)</td>
+    <td>@u.seenAt.map(momentFromNow(_))</td>
+  </tr>
+  }
+  }
+</tbody></table>
+}

modules/db/src/main/dsl.scala

@@ -130,6 +130,10 @@ trait dsl extends LowPriorityDsl {
     $doc("$unset" -> $doc((Seq(field) ++ fields).map(k => BSONElement(k, BSONString("")))))
   }
 
+  def $setBoolOrUnset(field: String, value: Boolean): BSONDocument = {
+    if (value) $set(field -> true) else $unset(field)
+  }
+
   def $min(item: Producer[BSONElement]): BSONDocument = {
     $doc("$min" -> $doc(item))
   }

modules/mod/src/main/Env.scala

@@ -91,10 +91,13 @@ final class Env(
     historyColl = db(CollectionGamingHistory)
   )
 
-  lazy val search = new UserSearch(
-    securityApi = securityApi,
-    emailValidator = emailValidator
-  )
+  lazy val search = lila.user.UserRepo.withColl { userColl =>
+    new UserSearch(
+      securityApi = securityApi,
+      emailValidator = emailValidator,
+      userColl = userColl
+    )
+  }
 
   lazy val inquiryApi = new InquiryApi(reportApi, noteApi, logApi)
 

modules/mod/src/main/UserSearch.scala

@@ -1,18 +1,40 @@
 package lila.mod
 
+import play.api.data._
+import play.api.data.Forms._
+import play.api.data.validation.Constraints
+import reactivemongo.api.ReadPreference
+
 import lila.common.{ EmailAddress, IpAddress }
+import lila.db.dsl._
 import lila.user.{ User, UserRepo }
+import User.{ BSONFields => F }
 
 final class UserSearch(
     securityApi: lila.security.SecurityApi,
-    emailValidator: lila.security.EmailAddressValidator
+    emailValidator: lila.security.EmailAddressValidator,
+    userColl: Coll
 ) {
 
-  def apply(query: String): Fu[List[User]] =
-    if (query.isEmpty) fuccess(Nil)
-    else EmailAddress.from(query).map(searchEmail) orElse
-      IpAddress.from(query).map(searchIp) getOrElse
-      (searchUsername(query) zip searchFingerHash(query) map Function.tupled(_ ++ _)) // list concatenation, in case a fingerhash is also someone's username
+  def apply(query: UserSearch.Query): Fu[List[User.WithEmails]] = (~query.as match {
+    case "regex" => userColl.find($doc(
+      F.watchList -> true,
+      $or(
+        F.id $regex query.q,
+        F.email $regex query.q,
+        F.prevEmail $regex query.q
+      )
+    )).hint($doc(F.watchList -> 1))
+      .cursor[User](ReadPreference.secondaryPreferred)
+      .list(100)
+    case "levenshtein" => fuccess(Nil)
+    case _ => // "exact"
+      EmailAddress.from(query.q).map(searchEmail) orElse
+        IpAddress.from(query.q).map(searchIp) getOrElse
+        (searchUsername(query.q) zip searchFingerHash(query.q) map Function.tupled(_ ++ _)) // list concatenation, in case a fingerhash is also someone's username
+  }) flatMap { users =>
+    UserRepo withEmails users.map(_.id)
+  }
 
   private def searchIp(ip: IpAddress) =
     securityApi recentUserIdsByIp ip map (_.reverse) flatMap UserRepo.usersFromSecondary
@@ -31,3 +53,22 @@ final class UserSearch(
     }
   }
 }
+
+object UserSearch {
+
+  val asChoices = List(
+    "exact" -> "Exact match over all users",
+    "regex" -> "Regex match over bad users",
+    "levenshtein" -> "Levenshtein over bad users (expensive)"
+  )
+  val asValues = asChoices.map(_._1)
+
+  case class Query(q: String, as: Option[String])
+
+  def exact(q: String) = Query(q, none)
+
+  val form = Form(mapping(
+    "q" -> nonEmptyText,
+    "as" -> optional(nonEmptyText.verifying(asValues contains _))
+  )(Query.apply)(Query.unapply))
+}

modules/security/src/main/DisposableEmailDomain.scala

@@ -56,11 +56,7 @@ private object DisposableEmailDomain {
   def whitelisted(domain: Domain) = whitelist contains domain.value
 
   private val staticBlacklist = Set(
-    "lichess.org",
-    "gamil.com",
-    "mybx.site", "mywrld.top", "wemel.top", "matra.top", "dripbank.com", "xxi2.com",
-    "forevernew.in", "sss.pp.ua", "ttempm.com", "emailnext.com",
-    "dea-love.net"
+    "lichess.org", "gamil.com"
   )
 
   private val whitelist = Set(

modules/security/src/main/EmailAddressValidator.scala

@@ -60,12 +60,8 @@ final class EmailAddressValidator(
   private def hasAcceptableDns(e: EmailAddress): Fu[Boolean] =
     if (isAcceptable(e)) e.domain ?? { domain =>
       if (DisposableEmailDomain whitelisted domain) fuccess(true)
-      else {
-        dnsApi.a(domain) >>| {
-          domain.withoutSubdomain ?? dnsApi.a
-        }
-      } >>& dnsApi.mx(domain).map { domains =>
-        domains.nonEmpty && domains.forall { !disposable(_) }
+      else dnsApi.mx(domain).map { domains =>
+        domains.nonEmpty && !domains.exists { disposable(_) }
       }
     }
     else fuccess(false)

modules/user/src/main/User.scala

@@ -80,6 +80,8 @@ case class User(
 
   def lameOrTroll = lame || troll
 
+  def watchList = booster || engine || troll || reportban || rankban || ipBan
+
   def lightPerf(key: String) = perfs(key) map { perf =>
     User.LightPerf(light, key, perf.intRating, perf.progress)
   }
@@ -161,7 +163,9 @@ object User {
 
   case class Active(user: User)
 
-  case class Emails(current: Option[EmailAddress], previous: Option[NormalizedEmailAddress])
+  case class Emails(current: Option[EmailAddress], previous: Option[NormalizedEmailAddress]) {
+    def list = current.toList ::: previous.toList
+  }
   case class WithEmails(user: User, emails: Emails)
 
   case class ClearPassword(value: String) extends AnyVal {
@@ -231,6 +235,7 @@ object User {
     val bpass = "bpass"
     val sha512 = "sha512"
     val totpSecret = "totp"
+    val watchList = "watchList"
   }
 
   import lila.db.BSON

modules/user/src/main/UserRepo.scala

@@ -36,7 +36,11 @@ object UserRepo {
   def byIdsSecondary(ids: Iterable[ID]): Fu[List[User]] = coll.byIds[User](ids, ReadPreference.secondaryPreferred)
 
   def byEmail(email: NormalizedEmailAddress): Fu[Option[User]] = coll.uno[User]($doc(F.email -> email))
-  def byPrevEmail(email: NormalizedEmailAddress): Fu[List[User]] = coll.find($doc(F.prevEmail -> email)).list[User]()
+  def byPrevEmail(
+    email: NormalizedEmailAddress,
+    readPreference: ReadPreference = ReadPreference.secondaryPreferred
+  ): Fu[List[User]] =
+    coll.list[User]($doc(F.prevEmail -> email), readPreference)
 
   def idByEmail(email: NormalizedEmailAddress): Fu[Option[String]] =
     coll.primitiveOne[String]($doc(F.email -> email), "_id")
@@ -277,26 +281,31 @@ object UserRepo {
         }
     }
 
-  def toggleEngine(id: ID): Funit =
+  private def setWatchList(id: ID): Funit =
     coll.fetchUpdate[User]($id(id)) { u =>
-      $set("engine" -> !u.engine)
+      $setBoolOrUnset(F.watchList, u.watchList)
     }
 
-  def setEngine(id: ID, v: Boolean): Funit = coll.updateField($id(id), "engine", v).void
+  def toggleEngine(id: ID): Funit =
+    coll.fetchUpdate[User]($id(id)) { u =>
+      $set(F.engine -> !u.engine)
+    } >> setWatchList(id)
 
-  def setBooster(id: ID, v: Boolean): Funit = coll.updateField($id(id), "booster", v).void
+  def setEngine(id: ID, v: Boolean): Funit = coll.updateField($id(id), "engine", v) >> setWatchList(id)
 
-  def setReportban(id: ID, v: Boolean): Funit = coll.updateField($id(id), "reportban", v).void
+  def setBooster(id: ID, v: Boolean): Funit = coll.updateField($id(id), "booster", v) >> setWatchList(id)
 
-  def setRankban(id: ID, v: Boolean): Funit = coll.updateField($id(id), "rankban", v).void
+  def setReportban(id: ID, v: Boolean): Funit = coll.updateField($id(id), "reportban", v) >> setWatchList(id)
 
-  def setIpBan(id: ID, v: Boolean) = coll.updateField($id(id), "ipBan", v).void
+  def setRankban(id: ID, v: Boolean): Funit = coll.updateField($id(id), "rankban", v) >> setWatchList(id)
+
+  def setIpBan(id: ID, v: Boolean) = coll.updateField($id(id), "ipBan", v) >> setWatchList(id)
 
   def setKid(user: User, v: Boolean) = coll.updateField($id(user.id), "kid", v)
 
   def isKid(id: ID) = coll.exists($id(id) ++ $doc("kid" -> true))
 
-  def updateTroll(user: User) = coll.updateField($id(user.id), "troll", user.troll)
+  def updateTroll(user: User) = coll.updateField($id(user.id), "troll", user.troll) >> setWatchList(user.id)
 
   def isEngine(id: ID): Fu[Boolean] = coll.exists($id(id) ++ engineSelect(true))
 
@@ -380,6 +389,20 @@ object UserRepo {
       }
     }
 
+  def withEmails(names: List[String]): Fu[List[User.WithEmails]] =
+    coll.find($inIds(names map normalize))
+      .list[Bdoc](none, ReadPreference.secondaryPreferred).map {
+        _ map { doc =>
+          User.WithEmails(
+            userBSONHandler read doc,
+            User.Emails(
+              current = doc.getAs[EmailAddress](F.verbatimEmail) orElse doc.getAs[EmailAddress](F.email),
+              previous = doc.getAs[NormalizedEmailAddress](F.prevEmail)
+            )
+          )
+        }
+      }
+
   def hasEmail(id: ID): Fu[Boolean] = email(id).map(_.isDefined)
 
   def setBot(user: User): Funit =

public/javascripts/login.js

@@ -23,7 +23,7 @@ function load($f) {
           $f.find('.submit').attr('disabled', false);
           if (res === 'InvalidTotpToken') $f.find('.two-factor .error').show();
         }
-        else lichess.redirect(res.startsWith('ok:') ? res.substr(3) : '/');
+        else location.href = res.startsWith('ok:') ? res.substr(3) : '/';
       },
       error: function(err) {
         $f.replaceWith($(err.responseText).find(selector));

ui/perfStat/src/glicko.js

@@ -26,7 +26,7 @@ function progress(p) {
 module.exports = function(d) {
   return [
     m('h2', [
-      'Glicko2 rating: ',
+      'Rating: ',
       m('strong', {
         title: 'Yes, ratings have decimal accuracy.'
       }, d.perf.glicko.rating),