From 1d9b289e1d41fa029c68946d93598d57e9ca1fff Mon Sep 17 00:00:00 2001 From: Thibault Duplessis Date: Mon, 7 Jun 2021 13:41:00 +0200 Subject: [PATCH] mod progress WIP --- app/views/mod/progress.scala | 9 ++- app/views/report/list.scala | 2 +- bin/mongodb/mod-progress.js | 18 +++++ bin/mongodb/modlog-human.js | 3 - bin/mongodb/report-atom.js | 54 --------------- bin/mongodb/report-migrate-comm.js | 23 ------- bin/mongodb/report-room.js | 12 ---- bin/mongodb/report-sweep.js | 32 --------- modules/mod/src/main/Gamify.scala | 19 +++--- modules/mod/src/main/ModProgress.scala | 78 ++++++++++++++-------- modules/report/src/main/BSONHandlers.scala | 3 +- modules/report/src/main/Report.scala | 8 ++- modules/report/src/main/ReportApi.scala | 10 +-- 13 files changed, 101 insertions(+), 170 deletions(-) create mode 100644 bin/mongodb/mod-progress.js delete mode 100644 bin/mongodb/modlog-human.js delete mode 100644 bin/mongodb/report-atom.js delete mode 100644 bin/mongodb/report-migrate-comm.js delete mode 100644 bin/mongodb/report-room.js delete mode 100644 bin/mongodb/report-sweep.js diff --git a/app/views/mod/progress.scala b/app/views/mod/progress.scala index 1dca6988aa..85ec10e1c3 100644 --- a/app/views/mod/progress.scala +++ b/app/views/mod/progress.scala @@ -6,6 +6,7 @@ import lila.app.ui.ScalatagsTemplate._ import lila.mod.ModProgress._ import controllers.routes +import lila.report.Room object progress { @@ -23,8 +24,11 @@ object progress { thead( tr( th("Date"), + Room.all.map { r => + th("Report", br, r.name) + }, Action.all.map { a => - th(a.toString) + th("Action", br, a.toString) } ) ), @@ -32,6 +36,9 @@ object progress { p.data.map { case (date, row) => tr( th(showDate(date)), + Room.all.map { r => + td(~row.reports.get(r)) + }, Action.all.map { a => td(~row.actions.get(a)) } diff --git a/app/views/report/list.scala b/app/views/report/list.scala index c1fa066e6d..4b4ba7bd4d 100644 --- a/app/views/report/list.scala +++ b/app/views/report/list.scala @@ -62,7 +62,7 @@ object list { td( r.inquiry match { case None => - if (r.processedBy.isDefined) + if (r.done.isDefined) postForm(action := routes.Report.inquiry(r.id), cls := "reopen")( submitButton(dataIcon := "G", cls := "text button button-metal")("Reopen") ) diff --git a/bin/mongodb/mod-progress.js b/bin/mongodb/mod-progress.js new file mode 100644 index 0000000000..d32c7edfde --- /dev/null +++ b/bin/mongodb/mod-progress.js @@ -0,0 +1,18 @@ +db.modlog.update({ human: { $ne: true }, mod: { $ne: 'lichess' } }, { $set: { human: true } }, { multi: true }); +db.modlog.createIndex({ mod: 1, date: -1 }, { partialFilterExpression: { human: true } }); + +db.report2.find({ processedBy: { $exists: 1 } }).forEach(r => + db.report2.update( + { _id: r._id }, + { + $unset: { processedBy: 1 }, + $set: { + done: { + by: r.processedBy, + at: r.atoms[0].at, + }, + }, + } + ) +); +db.report2.createIndex({ 'done.at': -1 }, { partialFilterExpression: { open: false } }); diff --git a/bin/mongodb/modlog-human.js b/bin/mongodb/modlog-human.js deleted file mode 100644 index 5c5e9ae18d..0000000000 --- a/bin/mongodb/modlog-human.js +++ /dev/null @@ -1,3 +0,0 @@ -db.modlog.update({ mod: { $ne: 'lichess' } }, { $set: { human: true } }, { multi: true }); - -db.modlog.createIndex({ mod: 1, date: -1 }, { partialFilterExpression: { human: true } }); diff --git a/bin/mongodb/report-atom.js b/bin/mongodb/report-atom.js deleted file mode 100644 index c374b20671..0000000000 --- a/bin/mongodb/report-atom.js +++ /dev/null @@ -1,54 +0,0 @@ -db.report2.drop(); - -db.report - .aggregate( - [ - { - $group: { - _id: { - user: '$user', - reason: '$reason', - processedBy: '$processedBy', - }, - reports: { $push: '$$ROOT' }, - }, - }, - ], - { allowDiskUse: true } - ) - .toArray() - .forEach(group => { - var reports = group.reports; - var first = reports[0]; - - var atoms = []; - reports.forEach(r => { - var same = atoms.find(a => a.by === r.createdBy); - if (same) same.text += '\n\n' + r.text; - else - atoms.push({ - by: r.createdBy, - at: r.createdAt, - text: r.text, - score: 30, - }); - }); - - var report = { - _id: first._id, - user: first.user, - reason: first.reason, - room: first.room, - atoms: atoms, - score: atoms.reduce((acc, atom) => acc + atom.score, 0), - open: !first.processedBy, - }; - if (first.processedBy) report.processedBy = first.processedBy; - - db.report2.insert(report); - }); - -db.report2.createIndex({ room: 1, score: -1 }, { partialFilterExpression: { open: true }, name: 'best_open' }); -db.report2.createIndex({ 'inquiry.mod': 1 }, { partialFilterExpression: { 'inquiry.mod': { $exists: true } } }); -db.report2.createIndex({ user: 1 }); -db.report2.createIndex({ 'atoms.by': 1 }); diff --git a/bin/mongodb/report-migrate-comm.js b/bin/mongodb/report-migrate-comm.js deleted file mode 100644 index 9e446bfc69..0000000000 --- a/bin/mongodb/report-migrate-comm.js +++ /dev/null @@ -1,23 +0,0 @@ -db.report2.update({ room: 'coms' }, { $set: { room: 'comm' } }, { multi: 1 }); - -db.report2.distinct('user', { room: 'comm' }).forEach(user => { - const reports = db.report2.find({ user: user, room: 'comm' }).toArray(); - - const report = reports[0]; - const others = reports.slice(1); - report.reason = 'comm'; - - others.forEach(rep => { - report.atoms = report.atoms.concat(rep.atoms); - report.score = report.score + rep.score; - report.open = report.open && rep.open; - report.processedBy = report.processedBy || rep.processedBy; - }); - - if (!report.processedBy || report.open) delete report.processedBy; - report.atoms.sort((a, b) => a.at > b.at); - - db.report2.update({ _id: report._id }, { $set: report }); - - if (others.length) db.report2.remove({ _id: { $in: others.map(r => r._id) } }); -}); diff --git a/bin/mongodb/report-room.js b/bin/mongodb/report-room.js deleted file mode 100644 index c9af55de9e..0000000000 --- a/bin/mongodb/report-room.js +++ /dev/null @@ -1,12 +0,0 @@ -var lastWeek = new Date(Date.now() - 1000 * 60 * 60 * 24 * 7); - -db.report.find().forEach(r => { - var room = 'others'; - if (!r.processedBy && r.createdAt < lastWeek) room = 'xfiles'; - else if (r.reason === 'cheat') room = 'cheat'; - else if (r.reason === 'cheatprint') room = 'print'; - else if (r.reason === 'troll') room = 'coms'; - else if (r.reason === 'insult') room = 'coms'; - - db.report.update({ _id: r._id }, { $set: { room: room } }); -}); diff --git a/bin/mongodb/report-sweep.js b/bin/mongodb/report-sweep.js deleted file mode 100644 index 9aff343ac7..0000000000 --- a/bin/mongodb/report-sweep.js +++ /dev/null @@ -1,32 +0,0 @@ -db.report - .find({ - processedBy: { - $exists: false, - }, - }) - .sort({ - createdAt: -1, - }) - .skip(150) - .limit(1) - .toArray() - .forEach(function (report) { - db.report.update( - { - processedBy: { - $exists: false, - }, - createdAt: { - $gt: report.createdAt, - }, - }, - { - $set: { - processedBy: 'lichess-sweep', - }, - }, - { - multi: true, - } - ); - }); diff --git a/modules/mod/src/main/Gamify.scala b/modules/mod/src/main/Gamify.scala index 8a5e8e2493..0c780e7b7f 100644 --- a/modules/mod/src/main/Gamify.scala +++ b/modules/mod/src/main/Gamify.scala @@ -135,12 +135,12 @@ final class Gamify( import framework._ Match( $doc( - "atoms.0.at" -> dateRange(after, before), - "room" $in Room.all, // required to make use of the mongodb index room+atoms.0.at - "processedBy" -> $nin(hidden) + "done.at" -> dateRange(after, before), + "done.by" -> $nin(hidden), + "open" -> false ) ) -> List( - GroupField("processedBy")( + GroupField("done.by")( "nb" -> Sum( $doc( "$cond" -> $arr($doc("$eq" -> $arr("room", Room.Cheat.key)), 3, 1) @@ -150,11 +150,12 @@ final class Gamify( Sort(Descending("nb")) ) } - .map { - _.flatMap { obj => - import cats.implicits._ - (obj.string("_id"), obj.int("nb")) mapN ModCount.apply - } + .map { docs => + for { + doc <- docs + id <- doc.string("_id") + nb <- doc.int("nb") + } yield ModCount(id, nb) } } diff --git a/modules/mod/src/main/ModProgress.scala b/modules/mod/src/main/ModProgress.scala index c6dea44666..279928cccb 100644 --- a/modules/mod/src/main/ModProgress.scala +++ b/modules/mod/src/main/ModProgress.scala @@ -8,8 +8,10 @@ import scala.util.Try import lila.db.dsl._ import lila.user.User +import lila.report.Report +import lila.report.Room -final class ModProgress(repo: ModlogRepo)(implicit ec: ExecutionContext) { +final class ModProgress(repo: ModlogRepo, reportApi: lila.report.ReportApi)(implicit ec: ExecutionContext) { import ModProgress._ @@ -23,48 +25,72 @@ final class ModProgress(repo: ModlogRepo)(implicit ec: ExecutionContext) { readPreference = ReadPreference.secondaryPreferred ) { framework => import framework._ + val dateSince = period match { + case Period.Week => DateTime.now.minusWeeks(1) + case Period.Month => DateTime.now.minusMonths(1) + case Period.Year => DateTime.now.minusYears(1) + } + def dateToString(field: String): Bdoc = + $doc("$dateToString" -> $doc("format" -> "%Y-%m-%d", "date" -> s"$$$field")) Match( $doc( "human" -> true, - "date" $gt (period match { - case Period.Week => DateTime.now.minusWeeks(1) - case Period.Month => DateTime.now.minusMonths(1) - case Period.Year => DateTime.now.minusYears(1) - }) + "date" $gt dateSince ) ++ (who match { case Who.Me(userId) => $doc("mod" -> userId) case Who.Team => $empty }) ) -> List( - Group( - $arr( - $doc("$dateToString" -> $doc("format" -> "%Y-%m-%d", "date" -> "$date")), - "$action" + Group($arr(dateToString("date"), "$action"))("nb" -> SumAll), + PipelineOperator( + $doc( + "$unionWith" -> $doc( + "coll" -> reportApi.coll.name, + "pipeline" -> List( + Match( + $doc( + "open" -> false, + "done.by" $nin List(User.lichessId, "irwin"), + "done.at" $gt dateSince + ) + ), + Group($arr(dateToString("done.at"), "$room"))("nb" -> SumAll) + ) + ) ) - )("nb" -> SumAll) + ), + Sort(Descending("_id.0")) ) } .map { docs => for { - doc <- docs - id <- doc.getAsOpt[List[String]]("_id") - date <- id.headOption - actionKey <- id lift 1 - action <- Action.dbMap get actionKey - nb <- doc.int("nb") - } yield (date, action, nb) + doc <- docs + id <- doc.getAsOpt[List[String]]("_id") + date <- id.headOption + key <- id lift 1 + nb <- doc.int("nb") + } yield (date, key, nb) } .map { - _.foldLeft(Map.empty[String, Map[Action, Int]]) { case (acc, (date, action, nb)) => - acc.updated(date, acc.getOrElse(date, Map.empty).updated(action, nb)) + _.foldLeft(Map.empty[String, Row]) { case (acc, (date, key, nb)) => + acc.updated( + date, { + val row = acc.getOrElse(date, Row(Map.empty, Map.empty)) + Room.byKey + .get(key) + .map(row.add(_, nb)) + .orElse(Action.dbMap.get(key).map(row.add(_, nb))) + .getOrElse(row) + } + ) } } .map { data => Result( period, who, - data.toList.sortBy(_._1).reverse.flatMap { case (date, actions) => - Try(dateFormat parseDateTime date).toOption map { _ -> Row(actions) } + data.toList.sortBy(_._1).reverse.flatMap { case (date, row) => + Try(dateFormat parseDateTime date).toOption map { _ -> row } } ) } @@ -78,7 +104,10 @@ object ModProgress { data: List[(DateTime, Row)] ) - case class Row(actions: Map[Action, Int]) + case class Row(actions: Map[Action, Int], reports: Map[Room, Int]) { + def add(action: Action, nb: Int) = copy(actions = actions.updatedWith(action)(prev => Some(~prev + nb))) + def add(room: Room, nb: Int) = copy(reports = reports.updatedWith(room)(prev => Some(~prev + nb))) + } private val dateFormat = DateTimeFormat forPattern "yyyy-MM-dd" @@ -107,7 +136,6 @@ object ModProgress { case object MarkTroll extends Action case object MarkBoost extends Action case object CloseAccount extends Action - case object CloseTeam extends Action case object ChatTimeout extends Action case object Appeal extends Action case object SetEmail extends Action @@ -125,8 +153,6 @@ object ModProgress { "unalt" -> CloseAccount, "closeAccount" -> CloseAccount, "reopenAccount" -> CloseAccount, - "disableTeam" -> CloseTeam, - "enableTeam" -> CloseTeam, "chatTimeout" -> ChatTimeout, "appealPost" -> Appeal, "appealClose" -> Appeal, diff --git a/modules/report/src/main/BSONHandlers.scala b/modules/report/src/main/BSONHandlers.scala index 6b5f801e19..dfd2484422 100644 --- a/modules/report/src/main/BSONHandlers.scala +++ b/modules/report/src/main/BSONHandlers.scala @@ -8,8 +8,9 @@ object BSONHandlers { implicit val ReasonBSONHandler = isoHandler[Reason, String](Reason.reasonIso) implicit val RoomBSONHandler = isoHandler[Room, String](Room.roomIso) - import Report.{ Atom, Inquiry, Score } + import Report.{ Atom, Done, Inquiry, Score } implicit val InquiryBSONHandler = Macros.handler[Inquiry] + implicit val DoneBSONHandler = Macros.handler[Done] implicit val ReporterIdBSONHandler = stringIsoHandler[ReporterId](ReporterId.reporterIdIso) implicit val ScoreIdBSONHandler = doubleIsoHandler[Score](Report.scoreIso) implicit val AtomBSONHandler = Macros.handler[Atom] diff --git a/modules/report/src/main/Report.scala b/modules/report/src/main/Report.scala index 5653661648..b426ba1e87 100644 --- a/modules/report/src/main/Report.scala +++ b/modules/report/src/main/Report.scala @@ -14,7 +14,7 @@ case class Report( score: Report.Score, inquiry: Option[Report.Inquiry], open: Boolean, - processedBy: Option[User.ID] + done: Option[Report.Done] ) extends Reason.WithReason { import Report.{ Atom, Score } @@ -67,7 +67,7 @@ case class Report( def process(by: User) = copy( open = false, - processedBy = by.id.some + done = Report.Done(by.id, DateTime.now).some ) def userIds: List[User.ID] = user :: atoms.toList.map(_.by.value) @@ -106,6 +106,8 @@ object Report { def byLichess = by == ReporterId.lichess } + case class Done(by: User.ID, at: DateTime) + case class Inquiry(mod: User.ID, seenAt: DateTime) case class WithSuspect(report: Report, suspect: Suspect, isOnline: Boolean) { @@ -163,7 +165,7 @@ object Report { score = score, inquiry = none, open = true, - processedBy = none + done = none ) )(_ add c.atom) } diff --git a/modules/report/src/main/ReportApi.scala b/modules/report/src/main/ReportApi.scala index 703827feea..9e01db21cd 100644 --- a/modules/report/src/main/ReportApi.scala +++ b/modules/report/src/main/ReportApi.scala @@ -229,12 +229,12 @@ final class ReportApi( def reopenReports(suspect: Suspect): Funit = for { all <- recent(suspect, 10) - closed = all.filter(_.processedBy has ModId.lichess.value) + closed = all.filter(_.done.map(_.by) has ModId.lichess.value) _ <- coll.update .one( $inIds(closed.map(_.id)), - $set("open" -> true) ++ $unset("processedBy"), + $set("open" -> true) ++ $unset("done"), multi = true ) .void @@ -306,8 +306,8 @@ final class ReportApi( .one( selector, $set( - "open" -> false, - "processedBy" -> by.value + "open" -> false, + "done" -> Report.Done(by.value, DateTime.now) ) ++ $unset("inquiry"), multi = true ) @@ -598,7 +598,7 @@ final class ReportApi( coll.update .one( $id(report.id), - $unset("inquiry", "processedBy") ++ $set("open" -> true) + $unset("inquiry", "done") ++ $set("open" -> true) ) .void