From 842515dedbd0c4759cb00bba55d2eb3a3f06327f Mon Sep 17 00:00:00 2001
From: Christian Fehmer <cfe@sexy-developer.com>
Date: Mon, 21 Apr 2025 23:40:46 +0200
Subject: [PATCH 1/9] refactor: backend database migrations (@fehmer)

---
 backend/__migration__/funboxConfig.ts |  47 ++++
 backend/__migration__/funboxResult.ts |  46 ++++
 backend/__migration__/index.ts        | 109 +++++++++
 backend/__migration__/testActivity.ts | 322 ++++++++++----------------
 backend/__migration__/types.ts        |  21 ++
 backend/src/dal/config.ts             |   2 +-
 6 files changed, 344 insertions(+), 203 deletions(-)
 create mode 100644 backend/__migration__/funboxConfig.ts
 create mode 100644 backend/__migration__/funboxResult.ts
 create mode 100644 backend/__migration__/index.ts
 create mode 100644 backend/__migration__/types.ts

diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts
new file mode 100644
index 000000000000..11aa0a9b3310
--- /dev/null
+++ b/backend/__migration__/funboxConfig.ts
@@ -0,0 +1,47 @@
+import { Collection, Db } from "mongodb";
+import { Migration } from "./types";
+import type { DBConfig } from "../src/dal/config";
+
+export class FunboxConfig implements Migration {
+  private configCollection!: Collection<DBConfig>;
+  private filter = { "config.funbox": { $exists: true, $type: "string" } };
+  private collectionName = "configs2"; //TODO rename
+
+  name: string = "FunboxConfig";
+
+  async setup(db: Db): Promise<void> {
+    this.configCollection = db.collection(this.collectionName);
+  }
+  async getRemainingCount(): Promise<number> {
+    return this.configCollection.countDocuments(this.filter);
+  }
+
+  async migrate({ batchSize }: { batchSize: number }): Promise<number> {
+    await this.configCollection
+      .aggregate([
+        { $match: this.filter },
+        { $limit: batchSize },
+        {
+          $addFields: {
+            "config.funbox": {
+              $cond: {
+                if: { $eq: ["$config.funbox", "none"] },
+                // eslint-disable-next-line no-thenable
+                then: undefined,
+                else: { $split: ["$config.funbox", "#"] },
+              },
+            },
+          },
+        },
+        {
+          $merge: {
+            into: this.collectionName,
+            on: "_id",
+            whenMatched: "merge",
+          },
+        },
+      ])
+      .toArray();
+    return batchSize; //TODO hmmm....
+  }
+}
diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts
new file mode 100644
index 000000000000..ab25172e3cde
--- /dev/null
+++ b/backend/__migration__/funboxResult.ts
@@ -0,0 +1,46 @@
+import { Collection, Db } from "mongodb";
+import { Migration } from "./types";
+import type { DBResult } from "../src/utils/result";
+
+export class funboxResult implements Migration {
+  private resultCollection!: Collection<DBResult>;
+  private filter = { funbox: { $exists: true, $not: { $type: "array" } } };
+  private collectionName = "results2"; //TODO rename
+  name: string = "FunboxResult";
+
+  async setup(db: Db): Promise<void> {
+    this.resultCollection = db.collection(this.collectionName);
+  }
+  async getRemainingCount(): Promise<number> {
+    return this.resultCollection.countDocuments(this.filter);
+  }
+
+  async migrate({ batchSize }: { batchSize: number }): Promise<number> {
+    await this.resultCollection
+      .aggregate([
+        { $match: this.filter },
+        { $limit: batchSize },
+        {
+          $addFields: {
+            funbox: {
+              $cond: {
+                if: { $eq: ["$funbox", "none"] },
+                // eslint-disable-next-line no-thenable
+                then: undefined,
+                else: { $split: ["$funbox", "#"] },
+              },
+            },
+          },
+        },
+        {
+          $merge: {
+            into: this.collectionName,
+            on: "_id",
+            whenMatched: "merge",
+          },
+        },
+      ])
+      .toArray();
+    return batchSize; //TODO hmmm....
+  }
+}
diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
new file mode 100644
index 000000000000..9f19b984c57a
--- /dev/null
+++ b/backend/__migration__/index.ts
@@ -0,0 +1,109 @@
+import "dotenv/config";
+import * as DB from "../src/init/db";
+import { Db } from "mongodb";
+import readlineSync from "readline-sync";
+import { funboxResult } from "./funboxResult";
+
+const batchSize = 50;
+let appRunning = true;
+let db: Db | undefined;
+const migration = new funboxResult();
+
+process.on("SIGINT", () => {
+  console.log("\nshutting down...");
+  appRunning = false;
+});
+
+if (require.main === module) {
+  void main();
+}
+
+async function main(): Promise<void> {
+  try {
+    console.log(
+      `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...`
+    );
+
+    if (
+      !readlineSync.keyInYN(`Ready to start migration  ${migration.name} ?`)
+    ) {
+      appRunning = false;
+    }
+
+    if (appRunning) {
+      await DB.connect();
+      console.log("Connected to database");
+      db = DB.getDb();
+      if (db === undefined) {
+        throw Error("db connection failed");
+      }
+
+      console.log(`Running migration ${migration.name}`);
+
+      await migrate();
+    }
+
+    console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`);
+  } catch (e) {
+    console.log("error occured:", { e });
+  } finally {
+    await DB.close();
+  }
+}
+
+export async function migrate(): Promise<void> {
+  await migration.setup(db as Db);
+
+  await migrateResults();
+}
+
+async function migrateResults(): Promise<void> {
+  const remainingCount = await migration.getRemainingCount();
+  if (remainingCount === 0) {
+    console.log("No documents to migrate.");
+    return;
+  } else {
+    console.log("Documents to migrate:", remainingCount);
+  }
+
+  console.log(
+    `Migrating ~${remainingCount} documents using batchSize=${batchSize}`
+  );
+
+  let count = 0;
+  const start = new Date().valueOf();
+
+  do {
+    const t0 = Date.now();
+
+    const migratedCount = await migration.migrate({ batchSize });
+
+    //progress tracker
+    count += migratedCount;
+    updateProgress(remainingCount, count, start, Date.now() - t0);
+  } while (remainingCount - count > 0 && appRunning);
+
+  if (appRunning) updateProgress(100, 100, start, 0);
+}
+
+function updateProgress(
+  all: number,
+  current: number,
+  start: number,
+  previousBatchSizeTime: number
+): void {
+  const percentage = (current / all) * 100;
+  const timeLeft = Math.round(
+    (((new Date().valueOf() - start) / percentage) * (100 - percentage)) / 1000
+  );
+
+  process.stdout.clearLine?.(0);
+  process.stdout.cursorTo?.(0);
+  process.stdout.write(
+    `Previous batch took ${Math.round(previousBatchSizeTime)}ms (~${
+      previousBatchSizeTime / batchSize
+    }ms per document) ${Math.round(
+      percentage
+    )}% done, estimated time left ${timeLeft} seconds.`
+  );
+}
diff --git a/backend/__migration__/testActivity.ts b/backend/__migration__/testActivity.ts
index 211d64d6362d..fb7b0850745e 100644
--- a/backend/__migration__/testActivity.ts
+++ b/backend/__migration__/testActivity.ts
@@ -1,185 +1,124 @@
-import "dotenv/config";
-import * as DB from "../src/init/db";
 import { Collection, Db } from "mongodb";
-
-import readlineSync from "readline-sync";
 import { DBUser } from "../src/dal/user";
 import { DBResult } from "../src/utils/result";
 
-const batchSize = 50;
-
-let appRunning = true;
-let db: Db | undefined;
-let userCollection: Collection<DBUser>;
-let resultCollection: Collection<DBResult>;
-
-const filter = { testActivity: { $exists: false } };
+import { Migration } from "./types";
 
-process.on("SIGINT", () => {
-  console.log("\nshutting down...");
-  appRunning = false;
-});
+export class TestActivityMigration implements Migration {
+  private userCollection!: Collection<DBUser>;
+  private resultCollection!: Collection<DBResult>;
+  private filter = { testActivity: { $exists: false } };
+  name: string = "TestActivity";
 
-if (require.main === module) {
-  void main();
-}
-
-async function main(): Promise<void> {
-  try {
-    console.log(
-      `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...`
-    );
+  async setup(db: Db): Promise<void> {
+    this.userCollection = db.collection("users");
+    this.resultCollection = db.collection("results");
 
-    //@ts-ignore
-    if (!readlineSync.keyInYN("Ready to start migration?")) {
-      appRunning = false;
-    }
-
-    if (appRunning) {
-      await DB.connect();
-      console.log("Connected to database");
-      db = DB.getDb();
-      if (db === undefined) {
-        throw Error("db connection failed");
-      }
-
-      await migrate();
-    }
-
-    console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`);
-  } catch (e) {
-    console.log("error occured:", { e });
-  } finally {
-    await DB.close();
+    console.log("Creating index on users collection...");
+    const t1 = Date.now();
+    await this.userCollection.createIndex({ uid: 1 }, { unique: true });
+    console.log("Index created in", Date.now() - t1, "ms");
   }
-}
 
-export async function migrate(): Promise<void> {
-  userCollection = DB.collection("users");
-  resultCollection = DB.collection("results");
-
-  console.log("Creating index on users collection...");
-  const t1 = Date.now();
-  await userCollection.createIndex({ uid: 1 }, { unique: true });
-  console.log("Index created in", Date.now() - t1, "ms");
-  await migrateResults();
-}
-
-async function migrateResults(): Promise<void> {
-  const allUsersCount = await userCollection.countDocuments(filter);
-  if (allUsersCount === 0) {
-    console.log("No users to migrate.");
-    return;
-  } else {
-    console.log("Users to migrate:", allUsersCount);
+  async getRemainingCount(): Promise<number> {
+    return this.userCollection.countDocuments(this.filter);
   }
 
-  console.log(`Migrating ~${allUsersCount} users using batchSize=${batchSize}`);
-
-  let count = 0;
-  const start = new Date().valueOf();
-  let uids: string[] = [];
-  do {
-    const t0 = Date.now();
+  async migrate({ batchSize }: { batchSize: number }): Promise<number> {
     console.log("Fetching users to migrate...");
     const t1 = Date.now();
-    uids = await getUsersToMigrate(batchSize);
+    const uids = await this.getUsersToMigrate(batchSize);
     console.log("Fetched", uids.length, "users in", Date.now() - t1, "ms");
     console.log("Users to migrate:", uids.join(","));
 
     //migrate
     const t2 = Date.now();
-    await migrateUsers(uids);
+    await this.migrateUsers(uids);
     console.log("Migrated", uids.length, "users in", Date.now() - t2, "ms");
     const t3 = Date.now();
-    await handleUsersWithNoResults(uids);
+    await this.handleUsersWithNoResults(uids);
     console.log("Handled users with no results in", Date.now() - t3, "ms");
 
-    //progress tracker
-    count += uids.length;
-    updateProgress(allUsersCount, count, start, Date.now() - t0);
-  } while (uids.length > 0 && appRunning);
-
-  if (appRunning) updateProgress(100, 100, start, 0);
-}
+    return uids.length;
+  }
 
-async function getUsersToMigrate(limit: number): Promise<string[]> {
-  return (
-    await userCollection
-      .find(filter, { limit })
-      .project({ uid: 1, _id: 0 })
-      .toArray()
-  ).map((it) => it["uid"]);
-}
+  async getUsersToMigrate(limit: number): Promise<string[]> {
+    return (
+      await this.userCollection
+        .find(this.filter, { limit })
+        .project({ uid: 1, _id: 0 })
+        .toArray()
+    ).map((it) => it["uid"]);
+  }
 
-async function migrateUsers(uids: string[]): Promise<void> {
-  await resultCollection
-    .aggregate(
-      [
-        {
-          $match: {
-            uid: { $in: uids },
-          },
-        },
-        {
-          $project: {
-            _id: 0,
-            timestamp: -1,
-            uid: 1,
-          },
-        },
-        {
-          $addFields: {
-            date: {
-              $toDate: "$timestamp",
+  async migrateUsers(uids: string[]): Promise<void> {
+    await this.resultCollection
+      .aggregate(
+        [
+          {
+            $match: {
+              uid: { $in: uids },
             },
           },
-        },
-        {
-          $replaceWith: {
-            uid: "$uid",
-            year: {
-              $year: "$date",
+          {
+            $project: {
+              _id: 0,
+              timestamp: -1,
+              uid: 1,
             },
-            day: {
-              $dayOfYear: "$date",
+          },
+          {
+            $addFields: {
+              date: {
+                $toDate: "$timestamp",
+              },
             },
           },
-        },
-        {
-          $group: {
-            _id: {
+          {
+            $replaceWith: {
               uid: "$uid",
-              year: "$year",
-              day: "$day",
-            },
-            count: {
-              $sum: 1,
+              year: {
+                $year: "$date",
+              },
+              day: {
+                $dayOfYear: "$date",
+              },
             },
           },
-        },
-        {
-          $group: {
-            _id: {
-              uid: "$_id.uid",
-              year: "$_id.year",
+          {
+            $group: {
+              _id: {
+                uid: "$uid",
+                year: "$year",
+                day: "$day",
+              },
+              count: {
+                $sum: 1,
+              },
             },
-            days: {
-              $addToSet: {
-                day: "$_id.day",
-                tests: "$count",
+          },
+          {
+            $group: {
+              _id: {
+                uid: "$_id.uid",
+                year: "$_id.year",
+              },
+              days: {
+                $addToSet: {
+                  day: "$_id.day",
+                  tests: "$count",
+                },
               },
             },
           },
-        },
-        {
-          $replaceWith: {
-            uid: "$_id.uid",
-            days: {
-              $function: {
-                lang: "js",
-                args: ["$days", "$_id.year"],
-                body: `function (days, year) {
+          {
+            $replaceWith: {
+              uid: "$_id.uid",
+              days: {
+                $function: {
+                  lang: "js",
+                  args: ["$days", "$_id.year"],
+                  body: `function (days, year) {
                                 var max = Math.max(
                                     ...days.map((it) => it.day)
                                 )-1;
@@ -191,69 +130,48 @@ async function migrateUsers(uids: string[]): Promise<void> {
                                 result[year] = arr;
                                 return result;
                             }`,
+                },
               },
             },
           },
-        },
-        {
-          $group: {
-            _id: "$uid",
-            testActivity: {
-              $mergeObjects: "$days",
+          {
+            $group: {
+              _id: "$uid",
+              testActivity: {
+                $mergeObjects: "$days",
+              },
             },
           },
-        },
-        {
-          $addFields: {
-            uid: "$_id",
+          {
+            $addFields: {
+              uid: "$_id",
+            },
           },
-        },
-        {
-          $project: {
-            _id: 0,
+          {
+            $project: {
+              _id: 0,
+            },
           },
-        },
-        {
-          $merge: {
-            into: "users",
-            on: "uid",
-            whenMatched: "merge",
-            whenNotMatched: "discard",
+          {
+            $merge: {
+              into: "users",
+              on: "uid",
+              whenMatched: "merge",
+              whenNotMatched: "discard",
+            },
           },
-        },
-      ],
-      { allowDiskUse: true }
-    )
-    .toArray();
-}
-
-async function handleUsersWithNoResults(uids: string[]): Promise<void> {
-  await userCollection.updateMany(
-    {
-      $and: [{ uid: { $in: uids } }, filter],
-    },
-    { $set: { testActivity: {} } }
-  );
-}
-
-function updateProgress(
-  all: number,
-  current: number,
-  start: number,
-  previousBatchSizeTime: number
-): void {
-  const percentage = (current / all) * 100;
-  const timeLeft = Math.round(
-    (((new Date().valueOf() - start) / percentage) * (100 - percentage)) / 1000
-  );
+        ],
+        { allowDiskUse: true }
+      )
+      .toArray();
+  }
 
-  process.stdout.clearLine?.(0);
-  process.stdout.cursorTo?.(0);
-  process.stdout.write(
-    `Previous batch took ${Math.round(previousBatchSizeTime)}ms (~${
-      previousBatchSizeTime / batchSize
-    }ms per user) ${Math.round(
-      percentage
-    )}% done, estimated time left ${timeLeft} seconds.`
-  );
+  async handleUsersWithNoResults(uids: string[]): Promise<void> {
+    await this.userCollection.updateMany(
+      {
+        $and: [{ uid: { $in: uids } }, this.filter],
+      },
+      { $set: { testActivity: {} } }
+    );
+  }
 }
diff --git a/backend/__migration__/types.ts b/backend/__migration__/types.ts
new file mode 100644
index 000000000000..51baaa583309
--- /dev/null
+++ b/backend/__migration__/types.ts
@@ -0,0 +1,21 @@
+import { Db } from "mongodb";
+
+export type Migration = {
+  name: string;
+  /**
+   * setup migration
+   * @param db mongo database
+   * @returns
+   */
+  setup: (db: Db) => Promise<void>;
+  /**
+   *
+   * @returns number of documents remaining for migration
+   */
+  getRemainingCount: () => Promise<number>;
+  /**
+   *
+   * @returns number of documents migrated
+   */
+  migrate: (options: { batchSize: number }) => Promise<number>;
+};
diff --git a/backend/src/dal/config.ts b/backend/src/dal/config.ts
index a11b32fe5bbf..b5e50e92b89e 100644
--- a/backend/src/dal/config.ts
+++ b/backend/src/dal/config.ts
@@ -23,7 +23,7 @@ const configLegacyProperties = [
   "enableAds",
 ];
 
-type DBConfig = {
+export type DBConfig = {
   _id: ObjectId;
   uid: string;
   config: PartialConfig;

From f6ce7661cdcb72c8f699316fa7f5ca269dc5beb0 Mon Sep 17 00:00:00 2001
From: Christian Fehmer <cfe@sexy-developer.com>
Date: Fri, 25 Apr 2025 12:48:24 +0200
Subject: [PATCH 2/9] wip

---
 backend/__migration__/funboxConfig.ts |  6 ++++--
 backend/__migration__/funboxResult.ts |  2 +-
 backend/__migration__/index.ts        | 17 ++++++++++-------
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts
index 11aa0a9b3310..72b96b6e1bf8 100644
--- a/backend/__migration__/funboxConfig.ts
+++ b/backend/__migration__/funboxConfig.ts
@@ -4,8 +4,10 @@ import type { DBConfig } from "../src/dal/config";
 
 export class FunboxConfig implements Migration {
   private configCollection!: Collection<DBConfig>;
-  private filter = { "config.funbox": { $exists: true, $type: "string" } };
-  private collectionName = "configs2"; //TODO rename
+  private filter = {
+    "config.funbox": { $exists: true, $not: { $type: "array" } },
+  };
+  private collectionName = "config";
 
   name: string = "FunboxConfig";
 
diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts
index ab25172e3cde..a5555d957e71 100644
--- a/backend/__migration__/funboxResult.ts
+++ b/backend/__migration__/funboxResult.ts
@@ -5,7 +5,7 @@ import type { DBResult } from "../src/utils/result";
 export class funboxResult implements Migration {
   private resultCollection!: Collection<DBResult>;
   private filter = { funbox: { $exists: true, $not: { $type: "array" } } };
-  private collectionName = "results2"; //TODO rename
+  private collectionName = "results";
   name: string = "FunboxResult";
 
   async setup(db: Db): Promise<void> {
diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index 9f19b984c57a..4ec74d1fab13 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -4,7 +4,7 @@ import { Db } from "mongodb";
 import readlineSync from "readline-sync";
 import { funboxResult } from "./funboxResult";
 
-const batchSize = 50;
+const batchSize = 250_000;
 let appRunning = true;
 let db: Db | undefined;
 const migration = new funboxResult();
@@ -54,10 +54,6 @@ async function main(): Promise<void> {
 export async function migrate(): Promise<void> {
   await migration.setup(db as Db);
 
-  await migrateResults();
-}
-
-async function migrateResults(): Promise<void> {
   const remainingCount = await migration.getRemainingCount();
   if (remainingCount === 0) {
     console.log("No documents to migrate.");
@@ -83,9 +79,16 @@ async function migrateResults(): Promise<void> {
     updateProgress(remainingCount, count, start, Date.now() - t0);
   } while (remainingCount - count > 0 && appRunning);
 
-  if (appRunning) updateProgress(100, 100, start, 0);
+  if (appRunning) {
+    updateProgress(100, 100, start, 0);
+    const left = await migration.getRemainingCount();
+    if (left !== 0) {
+      console.log(
+        `After migration there are ${left} unmigrated documents left. You might want to run the migration again.`
+      );
+    }
+  }
 }
-
 function updateProgress(
   all: number,
   current: number,

From 34e39174a6183a8c8ea5a7efc19e9a8465deb503 Mon Sep 17 00:00:00 2001
From: Christian Fehmer <cfe@sexy-developer.com>
Date: Mon, 28 Apr 2025 09:03:46 +0200
Subject: [PATCH 3/9] use index for results, migrate config.customLayoutfluid

---
 backend/__migration__/funboxConfig.ts | 39 ++++++++++++++++++---
 backend/__migration__/funboxResult.ts | 49 ++++++++++++++-------------
 backend/__migration__/index.ts        | 14 ++++++--
 3 files changed, 71 insertions(+), 31 deletions(-)

diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts
index 72b96b6e1bf8..01d7cf790f34 100644
--- a/backend/__migration__/funboxConfig.ts
+++ b/backend/__migration__/funboxConfig.ts
@@ -5,9 +5,12 @@ import type { DBConfig } from "../src/dal/config";
 export class FunboxConfig implements Migration {
   private configCollection!: Collection<DBConfig>;
   private filter = {
-    "config.funbox": { $exists: true, $not: { $type: "array" } },
+    $or: [
+      { "config.funbox": { $type: 2, $not: { $type: 4 } } },
+      { "config.customLayoutfluid": { $type: 2, $not: { $type: 4 } } },
+    ],
   };
-  private collectionName = "config";
+  private collectionName = "configs";
 
   name: string = "FunboxConfig";
 
@@ -23,14 +26,40 @@ export class FunboxConfig implements Migration {
       .aggregate([
         { $match: this.filter },
         { $limit: batchSize },
+        //don't use projection
         {
           $addFields: {
             "config.funbox": {
               $cond: {
-                if: { $eq: ["$config.funbox", "none"] },
+                if: {
+                  $and: [
+                    { $ne: ["$config.funbox", null] },
+                    { $ne: [{ $type: "$config.funbox" }, "array"] },
+                  ],
+                },
                 // eslint-disable-next-line no-thenable
-                then: undefined,
-                else: { $split: ["$config.funbox", "#"] },
+                then: {
+                  $cond: {
+                    if: { $eq: ["$config.funbox", "none"] },
+                    // eslint-disable-next-line no-thenable
+                    then: [],
+                    else: { $split: ["$config.funbox", "#"] },
+                  },
+                },
+                else: "$config.funbox",
+              },
+            },
+            "config.customLayoutfluid": {
+              $cond: {
+                if: {
+                  $and: [
+                    { $ne: ["$config.customLayoutfluid", null] },
+                    { $ne: [{ $type: "$config.customLayoutfluid" }, "array"] },
+                  ],
+                },
+                // eslint-disable-next-line no-thenable
+                then: { $split: ["$config.customLayoutfluid", "#"] },
+                else: "$config.customLayoutfluid",
               },
             },
           },
diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts
index a5555d957e71..34ad8da13cf5 100644
--- a/backend/__migration__/funboxResult.ts
+++ b/backend/__migration__/funboxResult.ts
@@ -2,9 +2,9 @@ import { Collection, Db } from "mongodb";
 import { Migration } from "./types";
 import type { DBResult } from "../src/utils/result";
 
-export class funboxResult implements Migration {
+export class FunboxResult implements Migration {
   private resultCollection!: Collection<DBResult>;
-  private filter = { funbox: { $exists: true, $not: { $type: "array" } } };
+  private filter = { funbox: { $type: 2, $not: { $type: 4 } } }; //string, not array of strings
   private collectionName = "results";
   name: string = "FunboxResult";
 
@@ -16,31 +16,34 @@ export class funboxResult implements Migration {
   }
 
   async migrate({ batchSize }: { batchSize: number }): Promise<number> {
-    await this.resultCollection
-      .aggregate([
-        { $match: this.filter },
-        { $limit: batchSize },
-        {
-          $addFields: {
-            funbox: {
-              $cond: {
-                if: { $eq: ["$funbox", "none"] },
-                // eslint-disable-next-line no-thenable
-                then: undefined,
-                else: { $split: ["$funbox", "#"] },
-              },
+    const pipeline = this.resultCollection.aggregate([
+      { $match: this.filter },
+      { $sort: { timestamp: 1 } },
+      { $limit: batchSize },
+      { $project: { _id: 1, timestamp: 1, funbox: 1 } },
+
+      {
+        $addFields: {
+          funbox: {
+            $cond: {
+              if: { $eq: ["$funbox", "none"] },
+              // eslint-disable-next-line no-thenable
+              then: undefined,
+              else: { $split: ["$funbox", "#"] },
             },
           },
         },
-        {
-          $merge: {
-            into: this.collectionName,
-            on: "_id",
-            whenMatched: "merge",
-          },
+      },
+      {
+        $merge: {
+          into: this.collectionName,
+          on: "_id",
+          whenMatched: "merge",
         },
-      ])
-      .toArray();
+      },
+    ]);
+    await pipeline.toArray();
+
     return batchSize; //TODO hmmm....
   }
 }
diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index 4ec74d1fab13..d087b2cb6f9c 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -2,12 +2,17 @@ import "dotenv/config";
 import * as DB from "../src/init/db";
 import { Db } from "mongodb";
 import readlineSync from "readline-sync";
-import { funboxResult } from "./funboxResult";
+import { FunboxResult } from "./funboxResult";
 
-const batchSize = 250_000;
+const batchSize = 100_000;
 let appRunning = true;
 let db: Db | undefined;
-const migration = new funboxResult();
+const migration = new FunboxResult();
+const delay = 1_000;
+
+const sleep = (durationMs): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, durationMs));
+};
 
 process.on("SIGINT", () => {
   console.log("\nshutting down...");
@@ -77,6 +82,9 @@ export async function migrate(): Promise<void> {
     //progress tracker
     count += migratedCount;
     updateProgress(remainingCount, count, start, Date.now() - t0);
+    if (delay) {
+      await sleep(delay);
+    }
   } while (remainingCount - count > 0 && appRunning);
 
   if (appRunning) {

From 8c1ef4594e6fbeb7d510d9fbff094930ed2c877d Mon Sep 17 00:00:00 2001
From: Miodec <jack@monkeytype.com>
Date: Tue, 29 Apr 2025 11:47:03 +0200
Subject: [PATCH 4/9] add migration picker, export default all migrations

---
 backend/__migration__/funboxConfig.ts |  4 +--
 backend/__migration__/funboxResult.ts |  2 +-
 backend/__migration__/index.ts        | 36 +++++++++++++++++++++++++--
 backend/__migration__/testActivity.ts |  2 +-
 4 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts
index 01d7cf790f34..0aaea232e3c1 100644
--- a/backend/__migration__/funboxConfig.ts
+++ b/backend/__migration__/funboxConfig.ts
@@ -1,8 +1,8 @@
-import { Collection, Db } from "mongodb";
+// import { Collection, Db } from "mongodb";
 import { Migration } from "./types";
 import type { DBConfig } from "../src/dal/config";
 
-export class FunboxConfig implements Migration {
+export default class FunboxConfig implements Migration {
   private configCollection!: Collection<DBConfig>;
   private filter = {
     $or: [
diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts
index 34ad8da13cf5..c28883aa7aab 100644
--- a/backend/__migration__/funboxResult.ts
+++ b/backend/__migration__/funboxResult.ts
@@ -2,7 +2,7 @@ import { Collection, Db } from "mongodb";
 import { Migration } from "./types";
 import type { DBResult } from "../src/utils/result";
 
-export class FunboxResult implements Migration {
+export default class FunboxResult implements Migration {
   private resultCollection!: Collection<DBResult>;
   private filter = { funbox: { $type: 2, $not: { $type: 4 } } }; //string, not array of strings
   private collectionName = "results";
diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index d087b2cb6f9c..dd8e12fd136e 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -2,12 +2,20 @@ import "dotenv/config";
 import * as DB from "../src/init/db";
 import { Db } from "mongodb";
 import readlineSync from "readline-sync";
-import { FunboxResult } from "./funboxResult";
+import funboxResult from "./funboxResult";
+import funboxConfig from "./funboxConfig";
+import testActivity from "./testActivity";
+import { Migration } from "./types";
 
 const batchSize = 100_000;
 let appRunning = true;
 let db: Db | undefined;
-const migration = new FunboxResult();
+const migrations = {
+  funboxConfig,
+  funboxResult,
+  testActivity,
+};
+
 const delay = 1_000;
 
 const sleep = (durationMs): Promise<void> => {
@@ -23,12 +31,36 @@ if (require.main === module) {
   void main();
 }
 
+function getMigrationToRun(): Migration {
+  //read all files in files directory, then use readlint sync keyInSelect to select a migration to run
+  const migrationNames = Object.keys(migrations);
+  const selectedMigration = readlineSync.keyInSelect(
+    migrationNames,
+    "Select migration to run"
+  );
+  if (selectedMigration === -1) {
+    console.log("No migration selected");
+    process.exit(0);
+  }
+  const migrationName = migrationNames[selectedMigration];
+  console.log("Selected migration:", migrationName);
+
+  const migration = migrations[migrationName];
+  if (migration === undefined) {
+    throw new Error("Migration not found");
+  }
+  console.log("Migration found:", migration.name);
+  return migration;
+}
+
 async function main(): Promise<void> {
   try {
     console.log(
       `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...`
     );
 
+    const migration = getMigrationToRun();
+
     if (
       !readlineSync.keyInYN(`Ready to start migration  ${migration.name} ?`)
     ) {
diff --git a/backend/__migration__/testActivity.ts b/backend/__migration__/testActivity.ts
index fb7b0850745e..6b61e34a89b5 100644
--- a/backend/__migration__/testActivity.ts
+++ b/backend/__migration__/testActivity.ts
@@ -4,7 +4,7 @@ import { DBResult } from "../src/utils/result";
 
 import { Migration } from "./types";
 
-export class TestActivityMigration implements Migration {
+export default class TestActivityMigration implements Migration {
   private userCollection!: Collection<DBUser>;
   private resultCollection!: Collection<DBResult>;
   private filter = { testActivity: { $exists: false } };

From dd51eccfae614485a2a04c38b1920fafc4fd0006 Mon Sep 17 00:00:00 2001
From: Miodec <jack@monkeytype.com>
Date: Tue, 29 Apr 2025 11:47:49 +0200
Subject: [PATCH 5/9] missing param

---
 backend/__migration__/index.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index dd8e12fd136e..63733f5abd84 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -77,7 +77,7 @@ async function main(): Promise<void> {
 
       console.log(`Running migration ${migration.name}`);
 
-      await migrate();
+      await migrate(migration);
     }
 
     console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`);
@@ -88,7 +88,7 @@ async function main(): Promise<void> {
   }
 }
 
-export async function migrate(): Promise<void> {
+export async function migrate(migration: Migration): Promise<void> {
   await migration.setup(db as Db);
 
   const remainingCount = await migration.getRemainingCount();

From 64be4284a8728ba31ab4082158626d52fdd3154b Mon Sep 17 00:00:00 2001
From: Miodec <jack@monkeytype.com>
Date: Tue, 29 Apr 2025 11:48:30 +0200
Subject: [PATCH 6/9] accidental comment

---
 backend/__migration__/funboxConfig.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts
index 0aaea232e3c1..35495c35ec43 100644
--- a/backend/__migration__/funboxConfig.ts
+++ b/backend/__migration__/funboxConfig.ts
@@ -1,4 +1,4 @@
-// import { Collection, Db } from "mongodb";
+import { Collection, Db } from "mongodb";
 import { Migration } from "./types";
 import type { DBConfig } from "../src/dal/config";
 

From 60bc373a44ae86262d89f75cc9b17e9b61a763f7 Mon Sep 17 00:00:00 2001
From: Miodec <jack@monkeytype.com>
Date: Tue, 29 Apr 2025 12:29:31 +0200
Subject: [PATCH 7/9] fix

---
 backend/__migration__/index.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index 63733f5abd84..dbb4edfaed7d 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -11,9 +11,9 @@ const batchSize = 100_000;
 let appRunning = true;
 let db: Db | undefined;
 const migrations = {
-  funboxConfig,
-  funboxResult,
-  testActivity,
+  funboxConfig: new funboxConfig(),
+  funboxResult: new funboxResult(),
+  testActivity: new testActivity(),
 };
 
 const delay = 1_000;

From 4dea653a9572096bf5053f7ed87ed27133c4180d Mon Sep 17 00:00:00 2001
From: Christian Fehmer <cfe@sexy-developer.com>
Date: Tue, 29 Apr 2025 13:02:44 +0200
Subject: [PATCH 8/9] test migrations

---
 backend/__migration__/funboxResult.ts         |  2 +-
 .../__migration__/funboxResults.spec.ts       | 77 +++++++++++++++++++
 .../__migration__/testActivity.spec.ts        | 17 +++-
 3 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 backend/__tests__/__migration__/funboxResults.spec.ts

diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts
index c28883aa7aab..d65ea2258143 100644
--- a/backend/__migration__/funboxResult.ts
+++ b/backend/__migration__/funboxResult.ts
@@ -28,7 +28,7 @@ export default class FunboxResult implements Migration {
             $cond: {
               if: { $eq: ["$funbox", "none"] },
               // eslint-disable-next-line no-thenable
-              then: undefined,
+              then: [],
               else: { $split: ["$funbox", "#"] },
             },
           },
diff --git a/backend/__tests__/__migration__/funboxResults.spec.ts b/backend/__tests__/__migration__/funboxResults.spec.ts
new file mode 100644
index 000000000000..c63f8b5c7b83
--- /dev/null
+++ b/backend/__tests__/__migration__/funboxResults.spec.ts
@@ -0,0 +1,77 @@
+import { Db } from "mongodb";
+import FunboxResultMigration from "../../__migration__/funboxResult";
+import { getDb } from "../../src/init/db";
+import { DBResult } from "../../src/utils/result";
+import * as ResultDal from "../../src/dal/result";
+import { createUser } from "../__testData__/users";
+
+describe("FunboxResults migration", () => {
+  const migration = new FunboxResultMigration();
+
+  beforeAll(async () => {
+    migration.setup(getDb() as Db);
+  });
+
+  it("migrates results with funbox", async () => {
+    //GIVEN
+    const { uid, resultId } = await createResult({
+      funbox: "58008#read_ahead",
+      acc: 95,
+    });
+
+    //WHEN
+    await migration.migrate({ batchSize: 1000 });
+
+    //THEN
+    const migratedResult = await ResultDal.getResult(uid, resultId);
+
+    expect(migratedResult).toEqual(
+      expect.objectContaining({
+        acc: 95,
+        funbox: ["58008", "read_ahead"],
+      })
+    );
+  });
+
+  it("migrates results with funbox none", async () => {
+    //GIVEN
+    const { uid, resultId } = await createResult({
+      funbox: "none",
+      acc: 95,
+    });
+
+    //WHEN
+    await migration.migrate({ batchSize: 1000 });
+
+    //THEN
+    const migratedResult = await ResultDal.getResult(uid, resultId);
+    console.log(migratedResult);
+
+    expect(migratedResult.funbox).toEqual([]);
+  });
+
+  it("migrates results without funbox", async () => {
+    //GIVEN
+    const { uid, resultId } = await createResult({
+      acc: 95,
+    });
+
+    //WHEN
+    await migration.migrate({ batchSize: 1000 });
+
+    //THEN
+    const migratedResult = await ResultDal.getResult(uid, resultId);
+    expect(migratedResult).not.toHaveProperty("funbox");
+  });
+});
+
+async function createResult(result: Partial<DBResult>): Promise<{
+  uid: string;
+  resultId: string;
+}> {
+  const uid = (await createUser()).uid;
+  const resultId = (
+    await ResultDal.addResult(uid, result as any)
+  ).insertedId.toHexString();
+  return { uid, resultId };
+}
diff --git a/backend/__tests__/__migration__/testActivity.spec.ts b/backend/__tests__/__migration__/testActivity.spec.ts
index 2b65382fbbdd..9b5bfb9aebeb 100644
--- a/backend/__tests__/__migration__/testActivity.spec.ts
+++ b/backend/__tests__/__migration__/testActivity.spec.ts
@@ -1,17 +1,26 @@
-import * as Migration from "../../__migration__/testActivity";
+import TestActivityMigration from "../../__migration__/testActivity";
 import * as UserTestData from "../__testData__/users";
 import * as UserDal from "../../src/dal/user";
 import * as ResultDal from "../../src/dal/result";
 import { DBResult } from "../../src/utils/result";
+import { getDb } from "../../src/init/db";
+import { Db } from "mongodb";
 
 describe("testActivity migration", () => {
+  const migration = new TestActivityMigration();
+
+  beforeAll(async () => {
+    migration.setup(getDb() as Db);
+  });
+
   it("migrates users without results", async () => {
     //given
+
     const user1 = await UserTestData.createUser();
     const user2 = await UserTestData.createUser();
 
     //when
-    await Migration.migrate();
+    await migration.migrate({ batchSize: 1 });
 
     //then
     const readUser1 = await UserDal.getUser(user1.uid, "");
@@ -42,7 +51,7 @@ describe("testActivity migration", () => {
     await createResult(uid, 1704243600000);
 
     //when
-    await Migration.migrate();
+    await migration.migrate({ batchSize: 1 });
 
     //then
     const readWithResults = await UserDal.getUser(withResults.uid, "");
@@ -52,7 +61,7 @@ describe("testActivity migration", () => {
     });
 
     const readWithoutResults = await UserDal.getUser(withoutResults.uid, "");
-    expect(readWithoutResults.testActivity).toEqual({});
+    expect(readWithoutResults.testActivity).toEqual(undefined);
   });
 });
 

From 72bbe06f2e308983983f6a72479370365e269a70 Mon Sep 17 00:00:00 2001
From: Christian Fehmer <cfe@sexy-developer.com>
Date: Tue, 29 Apr 2025 13:11:39 +0200
Subject: [PATCH 9/9] cleanup

---
 backend/__migration__/index.ts                      |  3 +--
 .../__tests__/__migration__/funboxResults.spec.ts   | 13 ++++++++-----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts
index dbb4edfaed7d..d2cd0e5e7e19 100644
--- a/backend/__migration__/index.ts
+++ b/backend/__migration__/index.ts
@@ -3,7 +3,6 @@ import * as DB from "../src/init/db";
 import { Db } from "mongodb";
 import readlineSync from "readline-sync";
 import funboxResult from "./funboxResult";
-import funboxConfig from "./funboxConfig";
 import testActivity from "./testActivity";
 import { Migration } from "./types";
 
@@ -11,7 +10,7 @@ const batchSize = 100_000;
 let appRunning = true;
 let db: Db | undefined;
 const migrations = {
-  funboxConfig: new funboxConfig(),
+  //funboxConfig: new funboxConfig(), // not ready yet
   funboxResult: new funboxResult(),
   testActivity: new testActivity(),
 };
diff --git a/backend/__tests__/__migration__/funboxResults.spec.ts b/backend/__tests__/__migration__/funboxResults.spec.ts
index c63f8b5c7b83..5d3e20577d59 100644
--- a/backend/__tests__/__migration__/funboxResults.spec.ts
+++ b/backend/__tests__/__migration__/funboxResults.spec.ts
@@ -15,7 +15,7 @@ describe("FunboxResults migration", () => {
   it("migrates results with funbox", async () => {
     //GIVEN
     const { uid, resultId } = await createResult({
-      funbox: "58008#read_ahead",
+      funbox: "58008#read_ahead" as any,
       acc: 95,
     });
 
@@ -36,7 +36,7 @@ describe("FunboxResults migration", () => {
   it("migrates results with funbox none", async () => {
     //GIVEN
     const { uid, resultId } = await createResult({
-      funbox: "none",
+      funbox: "none" as any,
       acc: 95,
     });
 
@@ -45,9 +45,12 @@ describe("FunboxResults migration", () => {
 
     //THEN
     const migratedResult = await ResultDal.getResult(uid, resultId);
-    console.log(migratedResult);
-
-    expect(migratedResult.funbox).toEqual([]);
+    expect(migratedResult).toEqual(
+      expect.objectContaining({
+        acc: 95,
+        funbox: [],
+      })
+    );
   });
 
   it("migrates results without funbox", async () => {