add cron jobs

bin/cron/mongodb-puzzle-denormalize-themes.js

@@ -0,0 +1,54 @@
+/* Denormalize puzzle themes based on round theme votes.
+ * Only looks for puzzles with the `dirty` flag, and removes it.
+ *
+ * mongo <IP>:<PORT>/<DB> mongodb-puzzle-denormalize-themes.js
+ * 
+ * Must run on the puzzle database.
+ * Should run every 5 minutes.
+ * Should complete within 10 seconds.
+ * OK to run many times in a row.
+ * OK to skip runs.
+ * OK to run concurrently.
+ */
+
+const playColl = db.puzzle2_puzzle;
+const roundColl = db.puzzle2_round;
+
+const staticThemes = new Set([
+  "oneMove", "short", "long", "veryLong",
+  "mateIn1", "mateIn2", "mateIn3", "mateIn4", "mateIn5",
+  "enPassant"
+]);
+
+playColl.find({ dirty: true }, { themes: true }).forEach(p => {
+
+  const oldThemes = p.themes || [];
+  const themeMap = {};
+
+  roundColl.aggregate([
+    {$match:{p:p._id,t:{$exists:true}}},
+    {$project:{_id:0,t:1,w:1}},
+    {$unwind:'$t'},
+    {$group:{_id:'$t',v:{$sum:'$w'}}}
+  ]).forEach(x => {
+    const signum = x._id[0] == '+' ? 1 : -1;
+    const theme = x._id.substring(1);
+    themeMap[theme] = x.v * signum + (themeMap[theme] || 0);
+  });
+
+  const newThemes = new Set();
+  Object.keys(themeMap).forEach(theme => {
+    if (themeMap[theme] > 0) newThemes.add(theme);
+  });
+
+  const update = {$unset:{dirty:true}};
+  if (
+    oldThemes.length !== newThemes.size ||
+    oldThemes.find(t => !newThemes.has(t))
+  ) {
+    const arr = Array.from(newThemes);
+    print(`Update ${p._id} themes: ${oldThemes.join(', ')} -> ${arr.join(', ')}`);
+    update['$set'] = {themes:arr};
+  }
+  playColl.update({_id:p._id},update);
+});

bin/cron/mongodb-puzzle-regen-paths.js

@@ -0,0 +1,117 @@
+/* Generates and saves a new generation of puzzle paths.
+ * Drops the previous generation.
+ *
+ * mongo <IP>:<PORT>/<DB> mongodb-puzzle-regen-paths.js
+ *
+ * Must run on the puzzle database.
+ * Should run every 60 minutes.
+ * Should complete within 3 minutes.
+ * OK to run many times in a row.
+ * OK to skip runs.
+ * NOT OK to run concurrently.
+ */
+
+const puzzleColl = db.puzzle2_puzzle;
+const pathColl = db.puzzle2_path;
+
+const generation = Date.now()
+
+const tiers = [
+  ['top', 33 / 100],
+  ['all', 100 / 100]
+];
+
+const acceptableVoteSelect = {
+  vote: {
+    $gt: -150
+  }
+};
+
+const themes = db.puzzle2_puzzle.distinct('themes',{});
+
+function makeTier(theme, tierName, thresholdRatio) {
+
+  const selector = {
+    ...acceptableVoteSelect,
+    ...(theme ? {themes: theme} : {})
+  };
+
+  const nbAcceptablePuzzles = puzzleColl.count(selector);
+
+  const nbPuzzles = Math.round(nbAcceptablePuzzles * thresholdRatio);
+
+  if (!nbPuzzles) return [];
+
+  const pathLength = Math.max(20, Math.min(100, Math.round(nbPuzzles / 100)));
+
+  const nbRatingBuckets = Math.max(3, Math.min(8, Math.round(nbPuzzles / pathLength / 15)));
+
+  // explodeArray([1..12], 3)
+  // [[1, 4, 7, 10], [2, 5, 8, 11], [3, 5, 9, 12]]
+  function explodeArray(arr, nb) {
+    const res = [];
+    for (i = 0; i < nb; i++) res[i] = [];
+    for (i in arr) res[i % nb].push(arr[i]);
+    return res;
+  }
+
+  const docs = [];
+  let bucketNumber = 0;
+
+  db.puzzle2_puzzle.aggregate([{
+    $match: selector
+  }, {
+    $sort: {
+      vote: -1
+    }
+  }, {
+    $limit: nbPuzzles
+  }, {
+    $bucketAuto: {
+      groupBy: '$glicko.r',
+      buckets: nbRatingBuckets,
+      output: {
+        puzzles: {
+          $addToSet: '$_id'
+        }
+      }
+    }
+  }], {
+    allowDiskUse: true,
+    comment: 'make-paths'
+  }).forEach(bucket => {
+    const ratingMin = Math.ceil(bucket._id.min);
+    const ratingMax = ++bucketNumber == nbRatingBuckets ? 9999 : Math.floor(bucket._id.max);
+    const nbPaths = Math.floor(bucket.puzzles.length / pathLength);
+    const puzzles = bucket.puzzles.slice(0, nbPaths * pathLength);
+    const paths = explodeArray(puzzles, nbPaths);
+    // print(`  ${ratingMin}->${ratingMax} paths: ${paths.length}`);
+    paths.forEach((ids, i) => {
+      docs.push({
+        _id: `${theme || 'any'}_${tierName}_${ratingMin}-${ratingMax}_${generation}_${i}`,
+        tier: tierName,
+        theme: theme,
+        min: ratingMin,
+        max: ratingMax,
+        ids,
+        length: ids.length,
+        gen: generation
+      });
+    });
+  });
+
+  print(`theme: ${theme}, tier: ${tierName}, threshold: ${thresholdRatio}, puzzles: ${nbPuzzles}, path length: ${pathLength}, rating buckets: ${nbRatingBuckets}, paths: ${docs.length}`);
+
+  return docs;
+}
+
+const docs = [];
+
+themes.concat([null]).forEach(theme =>
+// ['exposedKing'].forEach(theme =>
+  tiers.forEach(([name, threshold]) => makeTier(theme, name, threshold).forEach(p => docs.push(p)))
+);
+
+pathColl.insert(docs, {ordered: false});
+
+pathColl.remove({gen:{$ne:generation}});

bin/cron/mongodb-report-score-decay.js

@@ -1,12 +1,14 @@
-// Makes report scores decay with time.
-// This script is meant to run once per day.
-// It's ok to run it more than that once a day.
-// It's ok to run it less than that once a day.
-// It's ok to run it several times in a row.
-// The script is expected to complete within a couple seconds.
-//
-// Run it with:
-// mongo <IP>:27017/lichess mongodb-report-score-decay.js
+/* Makes report scores decay with time.
+ *
+ * mongo <IP>:<PORT>/<DB> mongodb-report-score-decay.js
+ *
+ * Must run on the main lila database.
+ * Should run once a day, preferably at night.
+ * Should complete within 1 minute.
+ * OK to run many times in a row.
+ * OK to skip runs.
+ * NOT OK to run concurrently.
+ */
 
 const atomMinScore = 5;