more on fingerprinting

app/controllers/User.scala

@@ -179,7 +179,7 @@ object User extends LilaController {
       (Env.security userSpy user.id) zip
         (Env.mod.assessApi.getPlayerAggregateAssessmentWithGames(user.id)) flatMap {
           case (spy, playerAggregateAssessment) =>
-            (Env.playban.api bans spy.otherUsers.map(_.id)) map { bans =>
+            (Env.playban.api bans spy.usersSharingIp.map(_.id)) map { bans =>
               html.user.mod(user, spy, playerAggregateAssessment, bans)
             }
         }

app/views/user/mod.scala.html

@@ -212,12 +212,13 @@
   </div>
   }
   @if(spy.otherUsers.size < 1) {
-  <strong>No user found with this IP</strong>
+  <strong>No similar user found</strong>
   } else {
   <table class="others slist">
     <thead>
       <tr>
-        <th>@spy.otherUsers.size user(s) sharing IPs</th>
+        <th>@spy.otherUsers.size similar user(s)</th>
+        <th>Same</th>
         <th>Games</th>
         <th>Marks</th>
         <th>IPban</th>
@@ -226,9 +227,11 @@
       </tr>
     </thead>
     <tbody>
-      @spy.otherUsers.map { o =>
+      @spy.otherUsers.map {
+      case lila.security.UserSpy.OtherUser(o, byIp, byFp) => {
       <tr @if(o == u){class="same"}>
         <td>@userLink(o, withBestRating = true, params = "?mod")</td>
+        <td>@List(byIp option "IP", byFp option "Browser").flatten.mkString(", ")</td>
         <td>@o.count.game</td>
         <td>
           @if(o.engine){ENGINE}
@@ -245,6 +248,7 @@
         <td>@momentFromNow(o.createdAt)</td>
       </tr>
       }
+      }
     </tbody>
   </table>
   }

modules/common/src/main/String.scala

@@ -18,8 +18,4 @@ object String {
 
     def apply(url: String) = regex.replaceAllIn(url, netDomain)
   }
-
-  def hex2bytes(hex: String): Array[Byte] = hex.sliding(2, 2).toArray.map(Integer.parseInt(_, 16).toByte)
-
-  def bytes2hex(bytes: Array[Byte]): String = bytes.map("%02x".format(_)).mkString
 }

modules/security/src/main/Api.scala

@@ -1,5 +1,7 @@
 package lila.security
 
+import lila.db.ByteArray
+import lila.db.ByteArray.ByteArrayBSONHandler
 import ornicar.scalalib.Random
 import play.api.data._
 import play.api.data.Forms._
@@ -65,6 +67,25 @@ private[security] final class Api(firewall: Firewall, tor: Tor) {
             _.flatMap(_.getAs[String]("user"))
           }
       }
+
+  def userIdsSharingFingerprint(userId: String): Fu[List[String]] =
+    tube.storeColl.find(
+      BSONDocument("user" -> userId, "fp" -> BSONDocument("$exists" -> true)),
+      BSONDocument("fp" -> true)
+    ).cursor[BSONDocument]().collect[List]().map {
+        _.flatMap(_.getAs[ByteArray]("fp"))
+      }.flatMap {
+        case Nil => fuccess(Nil)
+        case fps => tube.storeColl.find(
+          BSONDocument(
+            "fp" -> BSONDocument("$in" -> fps.distinct),
+            "user" -> BSONDocument("$ne" -> userId)
+          ),
+          BSONDocument("user" -> true)
+        ).cursor[BSONDocument]().collect[List]().map {
+            _.flatMap(_.getAs[String]("user"))
+          }
+      }
 }
 
 object Api {

modules/security/src/main/Store.scala

@@ -6,9 +6,9 @@ import org.joda.time.DateTime
 import play.api.mvc.RequestHeader
 import reactivemongo.bson.BSONDocument
 
-import lila.common.String.{ hex2bytes, bytes2hex }
 import lila.db.api._
 import lila.db.BSON.BSONJodaDateTimeHandler
+import lila.db.ByteArray
 import lila.user.{ User, UserRepo }
 import tube.storeColl
 
@@ -49,14 +49,19 @@ object Store {
     BSONDocument("$set" -> BSONDocument("up" -> false)),
     multi = true).void
 
-  def setFingerprint(id: String, fingerprint: String) = storeColl.update(
-    BSONDocument("_id" -> id),
-    BSONDocument("$set" -> BSONDocument("fp" -> hex2bytes(fingerprint)))).void
+  def setFingerprint(id: String, fingerprint: String) =
+    ByteArray.fromHexStr(fingerprint).future flatMap { bytes =>
+      storeColl.update(
+        BSONDocument("_id" -> id),
+        BSONDocument("$set" -> BSONDocument("fp" -> bytes))).void
+    }
 
-  case class Info(ip: String, ua: String, tor: Option[Boolean], fp: Option[String]) {
+  case class Info(ip: String, ua: String, tor: Option[Boolean], fp: Option[ByteArray]) {
     def isTorExitNode = ~tor
+    def fingerprint = fp.map(_.toString)
   }
   import reactivemongo.bson.Macros
+  import ByteArray.ByteArrayBSONHandler
   private implicit val InfoBSONHandler = Macros.handler[Info]
 
   def findInfoByUser(userId: String): Fu[List[Info]] =

modules/security/src/main/UserSpy.scala

@@ -13,17 +13,34 @@ import tube.storeColl
 case class UserSpy(
     ips: List[UserSpy.IPData],
     uas: List[String],
-    otherUsers: List[User]) {
+    usersSharingIp: List[User],
+    usersSharingFingerprint: List[User]) {
+
+  import UserSpy.OtherUser
 
   def ipStrings = ips map (_.ip)
 
   def ipsByLocations: List[(Location, List[UserSpy.IPData])] =
     ips.sortBy(_.ip).groupBy(_.location).toList.sortBy(_._1.comparable)
+
+  lazy val otherUsers: List[OtherUser] = {
+    usersSharingIp.map { u =>
+      OtherUser(u, true, usersSharingFingerprint contains u)
+    } ::: usersSharingFingerprint.filterNot(usersSharingIp.contains).map {
+      OtherUser(_, false, true)
+    }
+  }.sortBy(-_.user.createdAt.getMillis)
+
+  println(this)
 }
 
 object UserSpy {
 
+  case class OtherUser(user: User, byIp: Boolean, byFingerprint: Boolean)
+
   type IP = String
+  type Fingerprint = String
+  type Value = String
 
   case class IPData(ip: IP, blocked: Boolean, location: Location, tor: Boolean)
 
@@ -41,39 +58,35 @@ object UserSpy {
         case (ip, _)   => geoIP orUnknown ip
       }
     }
-    users ← explore(Set(user), Set.empty, Set(user))
+    sharingIp ← exploreSimilar("ip")(user)
+    sharingFingerprint ← exploreSimilar("fp")(user)
   } yield UserSpy(
     ips = ips zip blockedIps zip locations zip tors map {
       case (((ip, blocked), location), tor) => IPData(ip, blocked, location, tor)
     },
     uas = infos.map(_.ua).distinct,
-    otherUsers = (users + user).toList.sortBy(-_.createdAt.getMillis))
+    usersSharingIp = (sharingIp + user).toList.sortBy(-_.createdAt.getMillis),
+    usersSharingFingerprint = (sharingFingerprint + user).toList.sortBy(-_.createdAt.getMillis))
 
-  private def explore(users: Set[User], ips: Set[IP], _users: Set[User]): Fu[Set[User]] = {
-    nextIps(users, ips) flatMap { nIps =>
-      nextUsers(nIps, users) map { _ ++: users ++: _users }
-    }
-  }
-
-  private def nextIps(users: Set[User], ips: Set[IP]): Fu[Set[IP]] =
-    users.nonEmpty ?? {
-      storeColl.find(
-        BSONDocument(
-          "user" -> BSONDocument("$in" -> users.map(_.id)),
-          "ip" -> BSONDocument("$nin" -> ips)
-        ),
-        BSONDocument("ip" -> true)
-      ).cursor[BSONDocument]().collect[List]() map {
-          _.flatMap(_.getAs[IP]("ip")).toSet
-        }
+  private def exploreSimilar(field: String)(user: User): Fu[Set[User]] =
+    nextValues(field)(user).thenPp flatMap { nValues =>
+      nextUsers(field)(nValues, user).thenPp map { _ + user }
     }
 
-  private def nextUsers(ips: Set[IP], users: Set[User]): Fu[Set[User]] =
-    ips.nonEmpty ?? {
+  private def nextValues(field: String)(user: User): Fu[Set[Value]] =
+    storeColl.find(
+      BSONDocument("user" -> user.id),
+      BSONDocument(field -> true)
+    ).cursor[BSONDocument]().collect[List]() map {
+        _.flatMap(_.getAs[Value](field)).toSet
+      }
+
+  private def nextUsers(field: String)(values: Set[Value], user: User): Fu[Set[User]] =
+    values.nonEmpty ?? {
       storeColl.find(
         BSONDocument(
-          "ip" -> BSONDocument("$in" -> ips),
-          "user" -> BSONDocument("$nin" -> users.map(_.id))
+          field -> BSONDocument("$in" -> values),
+          "user" -> BSONDocument("$ne" -> user.id)
         ),
         BSONDocument("user" -> true)
       ).cursor[BSONDocument]().collect[List]() map {

public/javascripts/big.js

@@ -1049,7 +1049,9 @@ lichess.storage = {
     });
 
     if (window.Fingerprint2) setTimeout(function() {
-      new Fingerprint2().get(function(res) {
+      new Fingerprint2({
+        excludeJsFonts: true
+      }).get(function(res) {
         $.post('/set-fingerprint/' + res);
       });
     }, 500);