deepcrayon
/
lila
1
0
Fork 0
Code Releases Activity

send more distinct queries to db secondaries

pull/6529/head
Thibault Duplessis 2020-04-30 10:38:58 -06:00
parent 4380596749
commit 491a0bde52
20 changed files with 83 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/bookmark/src/main/BookmarkApi.scala
View File

@ -29,7 +29,8 @@ final class BookmarkApi(
user ?? { u =>
val candidateIds = games collect { case g if g.bookmarks > 0 => g.id }
candidateIds.nonEmpty ??
coll.distinctEasy[Game.ID, Set]("g", userIdQuery(u.id) ++ $doc("g" $in candidateIds))
coll.secondaryPreferred
.distinctEasy[Game.ID, Set]("g", userIdQuery(u.id) ++ $doc("g" $in candidateIds))
}
def removeByGameId(gameId: Game.ID): Funit =

2
modules/clas/src/main/ClasApi.scala
View File

@ -234,7 +234,7 @@ final class ClasApi(
private val idsCache = cacheApi.unit[Set[User.ID]] {
_.refreshAfterWrite(601 seconds)
.buildAsyncFuture { _ =>
coll.distinctEasy[User.ID, Set]("userId", $empty)
coll.distinctEasy[User.ID, Set]("userId", $empty, ReadPreference.secondaryPreferred)
}
}

2
modules/coach/src/main/CoachApi.scala
View File

@ -87,7 +87,7 @@ final class CoachApi(
private val languagesCache = cacheApi.unit[Set[String]] {
_.refreshAfterWrite(1 hour)
.buildAsyncFuture { _ =>
coachColl.distinctEasy[String, Set]("languages", $empty)
coachColl.secondaryPreferred.distinctEasy[String, Set]("languages", $empty)
}
}
def allLanguages: Fu[Set[String]] = languagesCache.get({})

5
modules/db/src/main/CollExt.scala
View File

@ -237,13 +237,14 @@ trait CollExt { self: dsl with QueryBuilderExt =>
def distinctEasy[T, M[_] <: Iterable[_]](
key: String,
selector: coll.pack.Document
selector: coll.pack.Document,
readPreference: ReadPreference = ReadPreference.primary
)(
implicit
reader: coll.pack.NarrowValueReader[T],
cbf: Factory[T, M[T]]
): Fu[M[T]] =
coll.distinct(key, selector.some, ReadConcern.Local, None)
coll.withReadPreference(readPreference).distinct(key, selector.some, ReadConcern.Local, None)
def findAndUpdate[D: BSONDocumentReader](
selector: coll.pack.Document,

4
modules/forum/src/main/PostRepo.scala
View File

@ -100,10 +100,10 @@ final class PostRepo(val coll: Coll, filter: Filter = Safe)(
def sortQuery = $sort.createdAsc
def userIdsByTopicId(topicId: String): Fu[List[String]] =
coll.distinctEasy[User.ID, List]("userId", $doc("topicId" -> topicId))
coll.distinctEasy[User.ID, List]("userId", $doc("topicId" -> topicId), ReadPreference.secondaryPreferred)
def idsByTopicId(topicId: String): Fu[List[String]] =
coll.distinctEasy[String, List]("_id", $doc("topicId" -> topicId))
coll.distinctEasy[String, List]("_id", $doc("topicId" -> topicId), ReadPreference.secondaryPreferred)
def cursor =
coll.ext

17
modules/game/src/main/GameRepo.scala
View File

@ -190,7 +190,11 @@ final class GameRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
}
def playingRealtimeNoAi(user: User): Fu[List[Game.ID]] =
coll.distinctEasy[Game.ID, List](F.id, Query.nowPlaying(user.id) ++ Query.noAi ++ Query.clock(true))
coll.distinctEasy[Game.ID, List](
F.id,
Query.nowPlaying(user.id) ++ Query.noAi ++ Query.clock(true),
ReadPreference.secondaryPreferred
)
def lastPlayedPlayingId(userId: User.ID): Fu[Option[Game.ID]] =
coll
@ -235,19 +239,12 @@ final class GameRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
def setTv(id: ID) = coll.updateFieldUnchecked($id(id), F.tvAt, DateTime.now)
def setAnalysed(id: ID): Unit = {
coll.updateFieldUnchecked($id(id), F.analysed, true)
}
def setUnanalysed(id: ID): Unit = {
coll.updateFieldUnchecked($id(id), F.analysed, false)
}
def setAnalysed(id: ID): Unit = coll.updateFieldUnchecked($id(id), F.analysed, true)
def setUnanalysed(id: ID): Unit = coll.updateFieldUnchecked($id(id), F.analysed, false)
def isAnalysed(id: ID): Fu[Boolean] =
coll.exists($id(id) ++ Query.analysed(true))
def filterAnalysed(ids: Seq[ID]): Fu[Set[ID]] =
coll.distinctEasy[ID, Set]("_id", $inIds(ids) ++ $doc(F.analysed -> true))
def exists(id: ID) = coll.exists($id(id))
def tournamentId(id: ID): Fu[Option[String]] = coll.primitiveOne[String]($id(id), F.tournamentId)

6
modules/pref/src/main/PrefApi.scala
View File

@ -48,11 +48,9 @@ final class PrefApi(
}
def unfollowableIds(userIds: List[User.ID]): Fu[Set[User.ID]] =
coll.distinctEasy[User.ID, Set](
coll.secondaryPreferred.distinctEasy[User.ID, Set](
"_id",
($inIds(userIds) ++ $doc(
"follow" -> false
))
($inIds(userIds) ++ $doc("follow" -> false))
)
def followableIds(userIds: List[User.ID]): Fu[Set[User.ID]] =

4
modules/relation/src/main/RelationRepo.scala
View File

@ -34,13 +34,13 @@ final private class RelationRepo(coll: Coll)(implicit ec: scala.concurrent.Execu
rp: ReadPreference = ReadPreference.primary
): Fu[Set[ID]] =
coll
.withReadPreference(rp)
.distinctEasy[ID, Set](
"u1",
$doc(
"u2" -> userId,
"r" -> relation
)
),
rp
)
private def relating(userId: ID, relation: Relation): Fu[Set[ID]] =

7
modules/report/src/main/ReportApi.scala
View File

@ -351,12 +351,13 @@ final class ReportApi(
)
def recentReportersOf(sus: Suspect): Fu[List[ReporterId]] =
coll.secondaryPreferred.distinctEasy[ReporterId, List](
coll.distinctEasy[ReporterId, List](
"atoms.by",
$doc(
"user" -> sus.user.id,
"atoms.0.at" $gt DateTime.now.minusDays(3)
)
),
ReadPreference.secondaryPreferred
) dmap (_ filterNot ReporterId.lichess.==)
def openAndRecentWithFilter(nb: Int, room: Option[Room]): Fu[List[Report.WithSuspect]] =
@ -419,7 +420,7 @@ final class ReportApi(
def invalidate(selector: Bdoc): Funit =
coll
.distinctEasy[User.ID, List]("atoms.by", selector)
.distinctEasy[User.ID, List]("atoms.by", selector, ReadPreference.secondaryPreferred)
.map {
_ foreach cache.invalidate
}

19
modules/security/src/main/CheckMail.scala
View File

@ -1,8 +1,8 @@
package lila.security
import scala.concurrent.duration._
import play.api.libs.ws.WSClient
import reactivemongo.api.ReadPreference
import lila.common.Domain
import lila.db.dsl._
@ -28,14 +28,17 @@ final private class CheckMail(
true
}
// expensive
private[security] def fetchAllBlocked: Fu[List[String]] =
cache.coll.distinctEasy[String, List](
"_id",
$doc(
"_id" $regex s"^$prefix:",
"v" -> false
)
) map { ids =>
cache.coll
.distinctEasy[String, List](
"_id",
$doc(
"_id" $regex s"^$prefix:",
"v" -> false
),
ReadPreference.secondaryPreferred
) map { ids =>
val dropSize = prefix.size + 1
ids.map(_ drop dropSize)
}

3
modules/security/src/main/Firewall.scala
View File

@ -3,6 +3,7 @@ package lila.security
import org.joda.time.DateTime
import play.api.mvc.RequestHeader
import scala.concurrent.duration._
import reactivemongo.api.ReadPreference
import lila.common.IpAddress
import lila.db.BSON.BSONJodaDateTimeHandler
@ -46,7 +47,7 @@ final class Firewall(
coll.delete.one($inIds(ips.filter(validIp))).void >>- loadFromDb
private def loadFromDb: Funit =
coll.distinctEasy[String, Set]("_id", $empty).map { ips =>
coll.distinctEasy[String, Set]("_id", $empty, ReadPreference.secondaryPreferred).map { ips =>
current = ips
lila.mon.security.firewall.ip.update(ips.size)
}

14
modules/security/src/main/PrintBan.scala
View File

@ -12,6 +12,7 @@ final class PrintBan(coll: Coll)(implicit ec: scala.concurrent.ExecutionContext)
def blocks(hash: FingerHash): Boolean = current contains hash.value
def toggle(hash: FingerHash, block: Boolean): Funit = {
current = if (block) current + hash.value else current - hash.value
if (block)
coll.update
.one(
@ -20,12 +21,11 @@ final class PrintBan(coll: Coll)(implicit ec: scala.concurrent.ExecutionContext)
upsert = true
)
.void
else coll.delete.one($id(hash.value))
} >> loadFromDb
else coll.delete.one($id(hash.value)).void
}
private def loadFromDb: Funit =
coll.distinctEasy[String, Set]("_id", $empty).map { hashes =>
current = hashes
lila.mon.security.firewall.prints.update(hashes.size)
}
coll.secondaryPreferred.distinctEasy[String, Set]("_id", $empty).map { hashes =>
current = hashes
lila.mon.security.firewall.prints.update(hashes.size)
}
}

8
modules/security/src/main/SecurityApi.scala
View File

@ -7,6 +7,7 @@ import play.api.data.Forms._
import play.api.data.validation.{ Constraint, Valid => FormValid, Invalid, ValidationError }
import play.api.mvc.RequestHeader
import reactivemongo.api.bson._
import reactivemongo.api.ReadPreference
import scala.annotation.nowarn
import scala.concurrent.duration._
@ -168,10 +169,10 @@ final class SecurityApi(
}
def ipUas(ip: IpAddress): Fu[List[String]] =
store.coll.distinctEasy[String, List]("ua", $doc("ip" -> ip.value))
store.coll.distinctEasy[String, List]("ua", $doc("ip" -> ip.value), ReadPreference.secondaryPreferred)
def printUas(fh: FingerHash): Fu[List[String]] =
store.coll.distinctEasy[String, List]("ua", $doc("fp" -> fh.value))
store.coll.distinctEasy[String, List]("ua", $doc("fp" -> fh.value), ReadPreference.secondaryPreferred)
private def recentUserIdsByField(field: String)(value: String): Fu[List[User.ID]] =
store.coll.distinctEasy[User.ID, List](
@ -179,7 +180,8 @@ final class SecurityApi(
$doc(
field -> value,
"date" $gt DateTime.now.minusYears(1)
)
),
ReadPreference.secondaryPreferred
)
}

2
modules/study/src/main/StudyRepo.scala
View File

@ -162,7 +162,7 @@ final class StudyRepo(private[study] val coll: Coll)(implicit ec: scala.concurre
// heavy AF. Only use for GDPR.
private[study] def allIdsByOwner(userId: User.ID): Fu[List[Study.Id]] =
coll.distinctEasy[Study.Id, List]("_id", selectOwnerId(userId))
coll.distinctEasy[Study.Id, List]("_id", selectOwnerId(userId), ReadPreference.secondaryPreferred)
def recentByContributor(userId: User.ID, nb: Int) =
coll

2
modules/team/src/main/MemberRepo.scala
View File

@ -14,7 +14,7 @@ final class MemberRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCo
// expensive with thousands of members!
def userIdsByTeam(teamId: ID): Fu[Set[ID]] =
coll.distinctEasy[String, Set]("user", $doc("team" -> teamId))
coll.secondaryPreferred.distinctEasy[String, Set]("user", $doc("team" -> teamId))
def teamIdsByUser(userId: User.ID): Fu[Set[ID]] =
coll.distinctEasy[ID, Set]("team", $doc("user" -> userId))

2
modules/team/src/main/TeamApi.scala
View File

@ -235,7 +235,7 @@ final class TeamApi(
teamRepo.leads(teamId, userId)
def filterExistingIds(ids: Set[String]): Fu[Set[Team.ID]] =
teamRepo.coll.distinctEasy[Team.ID, Set]("_id", $doc("_id" $in ids))
teamRepo.coll.distinctEasy[Team.ID, Set]("_id", $doc("_id" $in ids), ReadPreference.secondaryPreferred)
def autocomplete(term: String, max: Int): Fu[List[Team]] =
teamRepo.coll.ext

8
modules/team/src/main/TeamRepo.scala
View File

@ -19,8 +19,12 @@ final class TeamRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
def enabled(id: Team.ID) = coll.one[Team]($id(id) ++ enabledSelect)
def enabledTeamIdsByLeader(userId: User.ID): Fu[List[String]] =
coll.distinctEasy[String, List]("_id", $doc("leaders" -> userId) ++ enabledSelect)
def enabledTeamIdsByLeader(userId: User.ID): Fu[List[Team.ID]] =
coll.distinctEasy[Team.ID, List](
"_id",
$doc("leaders" -> userId) ++ enabledSelect,
ReadPreference.secondaryPreferred
)
def byIdsSortPopular(ids: Seq[Team.ID]): Fu[List[Team]] =
coll.ext

11
modules/timeline/src/main/UnsubApi.scala
View File

@ -3,26 +3,27 @@ package lila.timeline
import reactivemongo.api.bson._
import lila.db.dsl._
import lila.user.User
final class UnsubApi(coll: Coll)(implicit ec: scala.concurrent.ExecutionContext) {
private def makeId(channel: String, userId: String) = s"$userId@$channel"
private def makeId(channel: String, userId: User.ID) = s"$userId@$channel"
private def select(channel: String, userId: String) = $id(makeId(channel, userId))
private def select(channel: String, userId: User.ID) = $id(makeId(channel, userId))
def set(channel: String, userId: String, v: Boolean): Funit = {
def set(channel: String, userId: User.ID, v: Boolean): Funit = {
if (v) coll.insert.one(select(channel, userId)).void
else coll.delete.one(select(channel, userId)).void
} recover {
case _: Exception => ()
}
def get(channel: String, userId: String): Fu[Boolean] =
def get(channel: String, userId: User.ID): Fu[Boolean] =
coll.countSel(select(channel, userId)) dmap (0 !=)
private def canUnsub(channel: String) = channel startsWith "forum:"
def filterUnsub(channel: String, userIds: List[String]): Fu[List[String]] =
def filterUnsub(channel: String, userIds: List[User.ID]): Fu[List[String]] =
canUnsub(channel) ?? coll.distinctEasy[String, List](
"_id",
$inIds(userIds.map { makeId(channel, _) })

31
modules/user/src/main/UserRepo.scala
View File

@ -110,7 +110,7 @@ final class UserRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
coll.primitiveOne[User.ID]($id(id), F.username)
def usernamesByIds(ids: List[ID]) =
coll.distinctEasy[String, List](F.username, $inIds(ids))
coll.distinctEasy[String, List](F.username, $inIds(ids), ReadPreference.secondaryPreferred)
def createdAtById(id: ID) =
coll.primitiveOne[DateTime]($id(id), F.createdAt)
@ -491,9 +491,10 @@ final class UserRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
if (v) $doc(F.title -> Title.BOT)
else $doc(F.title -> $ne(Title.BOT))
private[user] def botIds = coll.secondaryPreferred.distinctEasy[String, Set](
private[user] def botIds = coll.distinctEasy[String, Set](
"_id",
botSelect(true) ++ enabledSelect
botSelect(true) ++ enabledSelect,
ReadPreference.secondaryPreferred
)
def getTitle(id: ID): Fu[Option[Title]] = coll.primitiveOne[Title]($id(id), F.title)
@ -554,20 +555,19 @@ final class UserRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
def langOf(id: ID): Fu[Option[String]] = coll.primitiveOne[String]($id(id), "lang")
// def idsSumToints(ids: Iterable[String]): Fu[Int] =
// ids.nonEmpty ?? coll.aggregateOne(
// Match($inIds(ids)),
// List(Group(BSONNull)(F.toints -> SumField(F.toints))),
// ReadPreference.secondaryPreferred
// ).map {
// _ flatMap { _.getAs[Int](F.toints) }
// }.map(~_)
def filterByEnabledPatrons(userIds: List[User.ID]): Fu[Set[User.ID]] =
coll.distinctEasy[String, Set](F.id, $inIds(userIds) ++ enabledSelect ++ patronSelect)
coll.distinctEasy[String, Set](
F.id,
$inIds(userIds) ++ enabledSelect ++ patronSelect,
ReadPreference.secondaryPreferred
)
def filterByRole(userIds: Seq[User.ID], role: String): Fu[Set[User.ID]] =
coll.distinctEasy[String, Set](F.id, $inIds(userIds) ++ enabledSelect ++ $doc(F.roles -> role))
coll.distinctEasy[String, Set](
F.id,
$inIds(userIds) ++ enabledSelect ++ $doc(F.roles -> role),
ReadPreference.secondaryPreferred
)
def userIdsWithRoles(roles: List[String]): Fu[Set[User.ID]] =
coll.distinctEasy[String, Set]("_id", $doc("roles" $in roles))
@ -624,7 +624,8 @@ final class UserRepo(val coll: Coll)(implicit ec: scala.concurrent.ExecutionCont
def filterClosedOrInactiveIds(since: DateTime)(ids: Iterable[ID]): Fu[List[ID]] =
coll.distinctEasy[ID, List](
F.id,
$inIds(ids) ++ $or(disabledSelect, F.seenAt $lt since)
$inIds(ids) ++ $or(disabledSelect, F.seenAt $lt since),
ReadPreference.secondaryPreferred
)
private def newUser(

5
modules/video/src/main/VideoApi.scala
View File

@ -89,7 +89,7 @@ final private[video] class VideoApi(
.void
def allIds: Fu[List[Video.ID]] =
videoColl.distinctEasy[String, List]("_id", $empty)
videoColl.distinctEasy[String, List]("_id", $empty, ReadPreference.secondaryPreferred)
def popular(user: Option[User], page: Int): Fu[Paginator[VideoView]] = Paginator(
adapter = new Adapter[Video](
@ -200,7 +200,8 @@ final private[video] class VideoApi(
View.BSONFields.videoId,
$inIds(videos.map { v =>
View.makeId(v.id, user.id)
})
}),
ReadPreference.secondaryPreferred
)
}