better clean up text in chats and studies

modules/chat/src/main/ChatApi.scala

@@ -6,7 +6,7 @@ import scala.concurrent.duration._
 
 import lila.common.Bus
 import lila.common.config.NetDomain
-import lila.common.String.noShouting
+import lila.common.String.{ fullCleanUp, noShouting }
 import lila.db.dsl._
 import lila.hub.actorApi.shutup.{ PublicSource, RecordPrivateChat, RecordPublicChat }
 import lila.memo.CacheApi._
@@ -242,7 +242,7 @@ final class ChatApi(
     private[ChatApi] def makeLine(chatId: Chat.Id, userId: String, t1: String): Fu[Option[UserLine]] =
       userRepo.speaker(userId) zip chatTimeout.isActive(chatId, userId) dmap {
         case (Some(user), false) if user.enabled =>
-          Writer cut t1 flatMap { t2 =>
+          Writer.preprocessUserInput(t1, user.username.some) flatMap { t2 =>
             val allow =
               if (user.isBot) !lila.common.String.hasLinks(t2)
               else flood.allowMessage(userId, t2)
@@ -251,7 +251,7 @@ final class ChatApi(
                 user.username,
                 user.title,
                 user.isPatron,
-                Writer.removeSelfMention(Writer preprocessUserInput t2, user.username),
+                t2,
                 troll = user.isTroll,
                 deleted = false
               )
@@ -288,9 +288,9 @@ final class ChatApi(
       }
 
     private def makeLine(chatId: Chat.Id, color: Color, t1: String): Option[Line] =
-      Writer cut t1 flatMap { t2 =>
+      Writer.preprocessUserInput(t1, none) flatMap { t2 =>
         flood.allowMessage(s"$chatId/${color.letter}", t2) option
-          PlayerLine(color, Writer preprocessUserInput t2)
+          PlayerLine(color, t2)
       }
   }
 
@@ -323,15 +323,19 @@ final class ChatApi(
 
     import java.util.regex.{ Matcher, Pattern }
 
-    def preprocessUserInput(in: String) = multiline(spam.replace(noShouting(noPrivateUrl(in))))
+    def preprocessUserInput(in: String, username: Option[User.ID]): Option[String] = {
+      val out1 = multiline(
+        spam.replace(noShouting(noPrivateUrl(fullCleanUp(in))))
+      )
+      val out2 = username.fold(out1) { removeSelfMention(out1, _) }
+      out2.take(Line.textMaxSize).some.filter(_.nonEmpty)
+    }
 
-    def removeSelfMention(in: String, username: User.ID) =
+    private def removeSelfMention(in: String, username: User.ID) =
       if (in.contains('@'))
         ("""(?i)@(?<![\w@#/]@)""" + username + """(?![@\w-]|\.\w)""").r.replaceAllIn(in, username)
       else in
 
-    def cut(text: String) = Some(text.trim take Line.textMaxSize).filter(_.nonEmpty)
-
     private val gameUrlRegex   = (Pattern.quote(netDomain.value) + """\b/(\w{8})\w{4}\b""").r
     private val gameUrlReplace = Matcher.quoteReplacement(netDomain.value) + "/$1"
 

modules/common/src/main/Form.scala

@@ -72,6 +72,7 @@ object Form {
         .get(key)
         .map(_.trim)
         .map(String.normalize.apply)
+        .map(String.removeMultibyteSymbols)
         .toRight(Seq(FormError(key, "error.required", Nil)))
     def unbind(key: String, value: String) = Map(key -> String.normalize(value.trim))
   }

modules/study/src/main/Chapter.scala

@@ -1,11 +1,11 @@
 package lila.study
 
 import chess.format.pgn.{ Glyph, Tags }
+import chess.opening.{ FullOpening, FullOpeningDB }
 import chess.variant.Variant
 import chess.{ Centis, Color }
 import org.joda.time.DateTime
 
-import chess.opening.{ FullOpening, FullOpeningDB }
 import lila.tree.Node.{ Comment, Gamebook, Shapes }
 import lila.user.User
 
@@ -172,7 +172,7 @@ object Chapter {
   private val defaultNameRegex = """Chapter \d+""".r
   def isDefaultName(n: Name)   = n.value.isEmpty || defaultNameRegex.matches(n.value)
 
-  def fixName(n: Name) = Name(n.value.trim take 80)
+  def fixName(n: Name) = Name(lila.common.String.fullCleanUp(n.value) take 80)
 
   val idSize = 8
 

modules/study/src/main/ChapterMaker.scala

@@ -252,5 +252,7 @@ private[study] object ChapterMaker {
     def hasDescription = description.nonEmpty
   }
 
-  case class DescData(id: Chapter.Id, desc: String)
+  case class DescData(id: Chapter.Id, desc: String) {
+    lazy val clean = lila.common.String.fullCleanUp(desc)
+  }
 }

modules/study/src/main/Study.scala

@@ -92,7 +92,7 @@ object Study {
     def id = _id
   }
 
-  def toName(str: String) = Name(str.trim take 100)
+  def toName(str: String) = Name(lila.common.String.fullCleanUp(str) take 100)
 
   sealed trait Visibility {
     lazy val key = toString.toLowerCase

modules/study/src/main/StudyApi.scala

@@ -697,7 +697,7 @@ final class StudyApi(
         chapterRepo.byIdAndStudy(data.id, studyId) flatMap {
           _ ?? { chapter =>
             val newChapter = chapter.copy(
-              description = data.desc.nonEmpty option data.desc
+              description = data.clean.nonEmpty option data.clean
             )
             (chapter != newChapter) ?? {
               chapterRepo.update(newChapter) >>- {

modules/study/src/main/actorApi.scala

@@ -4,7 +4,7 @@ package actorApi
 case class StartStudy(studyId: Study.Id)
 case class SaveStudy(study: Study)
 case class SetTag(chapterId: Chapter.Id, name: String, value: String) {
-  def tag = chess.format.pgn.Tag(name, value take 140)
+  def tag = chess.format.pgn.Tag(name, lila.common.String.fullCleanUp(value) take 140)
 }
 case class ExplorerGame(ch: Chapter.Id, path: String, gameId: String, insert: Boolean) {
   def chapterId = ch

modules/tree/src/main/tree.scala

@@ -149,7 +149,8 @@ object Node {
     }
     def sanitize(text: String) =
       Text {
-        text.trim
+        lila.common.String
+          .fullCleanUp(text)
           .take(4000)
           .replaceAll("""\r\n""", "\n") // these 3 lines dedup white spaces and new lines
           .replaceAll("""(?m)(^ *| +(?= |$))""", "")