diff --git a/backend/__migration__/funboxConfig.ts b/backend/__migration__/funboxConfig.ts new file mode 100644 index 000000000000..35495c35ec43 --- /dev/null +++ b/backend/__migration__/funboxConfig.ts @@ -0,0 +1,78 @@ +import { Collection, Db } from "mongodb"; +import { Migration } from "./types"; +import type { DBConfig } from "../src/dal/config"; + +export default class FunboxConfig implements Migration { + private configCollection!: Collection; + private filter = { + $or: [ + { "config.funbox": { $type: 2, $not: { $type: 4 } } }, + { "config.customLayoutfluid": { $type: 2, $not: { $type: 4 } } }, + ], + }; + private collectionName = "configs"; + + name: string = "FunboxConfig"; + + async setup(db: Db): Promise { + this.configCollection = db.collection(this.collectionName); + } + async getRemainingCount(): Promise { + return this.configCollection.countDocuments(this.filter); + } + + async migrate({ batchSize }: { batchSize: number }): Promise { + await this.configCollection + .aggregate([ + { $match: this.filter }, + { $limit: batchSize }, + //don't use projection + { + $addFields: { + "config.funbox": { + $cond: { + if: { + $and: [ + { $ne: ["$config.funbox", null] }, + { $ne: [{ $type: "$config.funbox" }, "array"] }, + ], + }, + // eslint-disable-next-line no-thenable + then: { + $cond: { + if: { $eq: ["$config.funbox", "none"] }, + // eslint-disable-next-line no-thenable + then: [], + else: { $split: ["$config.funbox", "#"] }, + }, + }, + else: "$config.funbox", + }, + }, + "config.customLayoutfluid": { + $cond: { + if: { + $and: [ + { $ne: ["$config.customLayoutfluid", null] }, + { $ne: [{ $type: "$config.customLayoutfluid" }, "array"] }, + ], + }, + // eslint-disable-next-line no-thenable + then: { $split: ["$config.customLayoutfluid", "#"] }, + else: "$config.customLayoutfluid", + }, + }, + }, + }, + { + $merge: { + into: this.collectionName, + on: "_id", + whenMatched: "merge", + }, + }, + ]) + .toArray(); + return batchSize; //TODO hmmm.... + } +} diff --git a/backend/__migration__/funboxResult.ts b/backend/__migration__/funboxResult.ts new file mode 100644 index 000000000000..d65ea2258143 --- /dev/null +++ b/backend/__migration__/funboxResult.ts @@ -0,0 +1,49 @@ +import { Collection, Db } from "mongodb"; +import { Migration } from "./types"; +import type { DBResult } from "../src/utils/result"; + +export default class FunboxResult implements Migration { + private resultCollection!: Collection; + private filter = { funbox: { $type: 2, $not: { $type: 4 } } }; //string, not array of strings + private collectionName = "results"; + name: string = "FunboxResult"; + + async setup(db: Db): Promise { + this.resultCollection = db.collection(this.collectionName); + } + async getRemainingCount(): Promise { + return this.resultCollection.countDocuments(this.filter); + } + + async migrate({ batchSize }: { batchSize: number }): Promise { + const pipeline = this.resultCollection.aggregate([ + { $match: this.filter }, + { $sort: { timestamp: 1 } }, + { $limit: batchSize }, + { $project: { _id: 1, timestamp: 1, funbox: 1 } }, + + { + $addFields: { + funbox: { + $cond: { + if: { $eq: ["$funbox", "none"] }, + // eslint-disable-next-line no-thenable + then: [], + else: { $split: ["$funbox", "#"] }, + }, + }, + }, + }, + { + $merge: { + into: this.collectionName, + on: "_id", + whenMatched: "merge", + }, + }, + ]); + await pipeline.toArray(); + + return batchSize; //TODO hmmm.... + } +} diff --git a/backend/__migration__/index.ts b/backend/__migration__/index.ts new file mode 100644 index 000000000000..d2cd0e5e7e19 --- /dev/null +++ b/backend/__migration__/index.ts @@ -0,0 +1,151 @@ +import "dotenv/config"; +import * as DB from "../src/init/db"; +import { Db } from "mongodb"; +import readlineSync from "readline-sync"; +import funboxResult from "./funboxResult"; +import testActivity from "./testActivity"; +import { Migration } from "./types"; + +const batchSize = 100_000; +let appRunning = true; +let db: Db | undefined; +const migrations = { + //funboxConfig: new funboxConfig(), // not ready yet + funboxResult: new funboxResult(), + testActivity: new testActivity(), +}; + +const delay = 1_000; + +const sleep = (durationMs): Promise => { + return new Promise((resolve) => setTimeout(resolve, durationMs)); +}; + +process.on("SIGINT", () => { + console.log("\nshutting down..."); + appRunning = false; +}); + +if (require.main === module) { + void main(); +} + +function getMigrationToRun(): Migration { + //read all files in files directory, then use readlint sync keyInSelect to select a migration to run + const migrationNames = Object.keys(migrations); + const selectedMigration = readlineSync.keyInSelect( + migrationNames, + "Select migration to run" + ); + if (selectedMigration === -1) { + console.log("No migration selected"); + process.exit(0); + } + const migrationName = migrationNames[selectedMigration]; + console.log("Selected migration:", migrationName); + + const migration = migrations[migrationName]; + if (migration === undefined) { + throw new Error("Migration not found"); + } + console.log("Migration found:", migration.name); + return migration; +} + +async function main(): Promise { + try { + console.log( + `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...` + ); + + const migration = getMigrationToRun(); + + if ( + !readlineSync.keyInYN(`Ready to start migration ${migration.name} ?`) + ) { + appRunning = false; + } + + if (appRunning) { + await DB.connect(); + console.log("Connected to database"); + db = DB.getDb(); + if (db === undefined) { + throw Error("db connection failed"); + } + + console.log(`Running migration ${migration.name}`); + + await migrate(migration); + } + + console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`); + } catch (e) { + console.log("error occured:", { e }); + } finally { + await DB.close(); + } +} + +export async function migrate(migration: Migration): Promise { + await migration.setup(db as Db); + + const remainingCount = await migration.getRemainingCount(); + if (remainingCount === 0) { + console.log("No documents to migrate."); + return; + } else { + console.log("Documents to migrate:", remainingCount); + } + + console.log( + `Migrating ~${remainingCount} documents using batchSize=${batchSize}` + ); + + let count = 0; + const start = new Date().valueOf(); + + do { + const t0 = Date.now(); + + const migratedCount = await migration.migrate({ batchSize }); + + //progress tracker + count += migratedCount; + updateProgress(remainingCount, count, start, Date.now() - t0); + if (delay) { + await sleep(delay); + } + } while (remainingCount - count > 0 && appRunning); + + if (appRunning) { + updateProgress(100, 100, start, 0); + const left = await migration.getRemainingCount(); + if (left !== 0) { + console.log( + `After migration there are ${left} unmigrated documents left. You might want to run the migration again.` + ); + } + } +} +function updateProgress( + all: number, + current: number, + start: number, + previousBatchSizeTime: number +): void { + const percentage = (current / all) * 100; + const timeLeft = Math.round( + (((new Date().valueOf() - start) / percentage) * (100 - percentage)) / 1000 + ); + + process.stdout.clearLine?.(0); + process.stdout.cursorTo?.(0); + process.stdout.write( + `Previous batch took ${Math.round(previousBatchSizeTime)}ms (~${ + previousBatchSizeTime / batchSize + }ms per document) ${Math.round( + percentage + )}% done, estimated time left ${timeLeft} seconds.` + ); +} diff --git a/backend/__migration__/testActivity.ts b/backend/__migration__/testActivity.ts index 211d64d6362d..6b61e34a89b5 100644 --- a/backend/__migration__/testActivity.ts +++ b/backend/__migration__/testActivity.ts @@ -1,185 +1,124 @@ -import "dotenv/config"; -import * as DB from "../src/init/db"; import { Collection, Db } from "mongodb"; - -import readlineSync from "readline-sync"; import { DBUser } from "../src/dal/user"; import { DBResult } from "../src/utils/result"; -const batchSize = 50; - -let appRunning = true; -let db: Db | undefined; -let userCollection: Collection; -let resultCollection: Collection; - -const filter = { testActivity: { $exists: false } }; +import { Migration } from "./types"; -process.on("SIGINT", () => { - console.log("\nshutting down..."); - appRunning = false; -}); +export default class TestActivityMigration implements Migration { + private userCollection!: Collection; + private resultCollection!: Collection; + private filter = { testActivity: { $exists: false } }; + name: string = "TestActivity"; -if (require.main === module) { - void main(); -} - -async function main(): Promise { - try { - console.log( - `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...` - ); + async setup(db: Db): Promise { + this.userCollection = db.collection("users"); + this.resultCollection = db.collection("results"); - //@ts-ignore - if (!readlineSync.keyInYN("Ready to start migration?")) { - appRunning = false; - } - - if (appRunning) { - await DB.connect(); - console.log("Connected to database"); - db = DB.getDb(); - if (db === undefined) { - throw Error("db connection failed"); - } - - await migrate(); - } - - console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`); - } catch (e) { - console.log("error occured:", { e }); - } finally { - await DB.close(); + console.log("Creating index on users collection..."); + const t1 = Date.now(); + await this.userCollection.createIndex({ uid: 1 }, { unique: true }); + console.log("Index created in", Date.now() - t1, "ms"); } -} -export async function migrate(): Promise { - userCollection = DB.collection("users"); - resultCollection = DB.collection("results"); - - console.log("Creating index on users collection..."); - const t1 = Date.now(); - await userCollection.createIndex({ uid: 1 }, { unique: true }); - console.log("Index created in", Date.now() - t1, "ms"); - await migrateResults(); -} - -async function migrateResults(): Promise { - const allUsersCount = await userCollection.countDocuments(filter); - if (allUsersCount === 0) { - console.log("No users to migrate."); - return; - } else { - console.log("Users to migrate:", allUsersCount); + async getRemainingCount(): Promise { + return this.userCollection.countDocuments(this.filter); } - console.log(`Migrating ~${allUsersCount} users using batchSize=${batchSize}`); - - let count = 0; - const start = new Date().valueOf(); - let uids: string[] = []; - do { - const t0 = Date.now(); + async migrate({ batchSize }: { batchSize: number }): Promise { console.log("Fetching users to migrate..."); const t1 = Date.now(); - uids = await getUsersToMigrate(batchSize); + const uids = await this.getUsersToMigrate(batchSize); console.log("Fetched", uids.length, "users in", Date.now() - t1, "ms"); console.log("Users to migrate:", uids.join(",")); //migrate const t2 = Date.now(); - await migrateUsers(uids); + await this.migrateUsers(uids); console.log("Migrated", uids.length, "users in", Date.now() - t2, "ms"); const t3 = Date.now(); - await handleUsersWithNoResults(uids); + await this.handleUsersWithNoResults(uids); console.log("Handled users with no results in", Date.now() - t3, "ms"); - //progress tracker - count += uids.length; - updateProgress(allUsersCount, count, start, Date.now() - t0); - } while (uids.length > 0 && appRunning); - - if (appRunning) updateProgress(100, 100, start, 0); -} + return uids.length; + } -async function getUsersToMigrate(limit: number): Promise { - return ( - await userCollection - .find(filter, { limit }) - .project({ uid: 1, _id: 0 }) - .toArray() - ).map((it) => it["uid"]); -} + async getUsersToMigrate(limit: number): Promise { + return ( + await this.userCollection + .find(this.filter, { limit }) + .project({ uid: 1, _id: 0 }) + .toArray() + ).map((it) => it["uid"]); + } -async function migrateUsers(uids: string[]): Promise { - await resultCollection - .aggregate( - [ - { - $match: { - uid: { $in: uids }, - }, - }, - { - $project: { - _id: 0, - timestamp: -1, - uid: 1, - }, - }, - { - $addFields: { - date: { - $toDate: "$timestamp", + async migrateUsers(uids: string[]): Promise { + await this.resultCollection + .aggregate( + [ + { + $match: { + uid: { $in: uids }, }, }, - }, - { - $replaceWith: { - uid: "$uid", - year: { - $year: "$date", + { + $project: { + _id: 0, + timestamp: -1, + uid: 1, }, - day: { - $dayOfYear: "$date", + }, + { + $addFields: { + date: { + $toDate: "$timestamp", + }, }, }, - }, - { - $group: { - _id: { + { + $replaceWith: { uid: "$uid", - year: "$year", - day: "$day", - }, - count: { - $sum: 1, + year: { + $year: "$date", + }, + day: { + $dayOfYear: "$date", + }, }, }, - }, - { - $group: { - _id: { - uid: "$_id.uid", - year: "$_id.year", + { + $group: { + _id: { + uid: "$uid", + year: "$year", + day: "$day", + }, + count: { + $sum: 1, + }, }, - days: { - $addToSet: { - day: "$_id.day", - tests: "$count", + }, + { + $group: { + _id: { + uid: "$_id.uid", + year: "$_id.year", + }, + days: { + $addToSet: { + day: "$_id.day", + tests: "$count", + }, }, }, }, - }, - { - $replaceWith: { - uid: "$_id.uid", - days: { - $function: { - lang: "js", - args: ["$days", "$_id.year"], - body: `function (days, year) { + { + $replaceWith: { + uid: "$_id.uid", + days: { + $function: { + lang: "js", + args: ["$days", "$_id.year"], + body: `function (days, year) { var max = Math.max( ...days.map((it) => it.day) )-1; @@ -191,69 +130,48 @@ async function migrateUsers(uids: string[]): Promise { result[year] = arr; return result; }`, + }, }, }, }, - }, - { - $group: { - _id: "$uid", - testActivity: { - $mergeObjects: "$days", + { + $group: { + _id: "$uid", + testActivity: { + $mergeObjects: "$days", + }, }, }, - }, - { - $addFields: { - uid: "$_id", + { + $addFields: { + uid: "$_id", + }, }, - }, - { - $project: { - _id: 0, + { + $project: { + _id: 0, + }, }, - }, - { - $merge: { - into: "users", - on: "uid", - whenMatched: "merge", - whenNotMatched: "discard", + { + $merge: { + into: "users", + on: "uid", + whenMatched: "merge", + whenNotMatched: "discard", + }, }, - }, - ], - { allowDiskUse: true } - ) - .toArray(); -} - -async function handleUsersWithNoResults(uids: string[]): Promise { - await userCollection.updateMany( - { - $and: [{ uid: { $in: uids } }, filter], - }, - { $set: { testActivity: {} } } - ); -} - -function updateProgress( - all: number, - current: number, - start: number, - previousBatchSizeTime: number -): void { - const percentage = (current / all) * 100; - const timeLeft = Math.round( - (((new Date().valueOf() - start) / percentage) * (100 - percentage)) / 1000 - ); + ], + { allowDiskUse: true } + ) + .toArray(); + } - process.stdout.clearLine?.(0); - process.stdout.cursorTo?.(0); - process.stdout.write( - `Previous batch took ${Math.round(previousBatchSizeTime)}ms (~${ - previousBatchSizeTime / batchSize - }ms per user) ${Math.round( - percentage - )}% done, estimated time left ${timeLeft} seconds.` - ); + async handleUsersWithNoResults(uids: string[]): Promise { + await this.userCollection.updateMany( + { + $and: [{ uid: { $in: uids } }, this.filter], + }, + { $set: { testActivity: {} } } + ); + } } diff --git a/backend/__migration__/types.ts b/backend/__migration__/types.ts new file mode 100644 index 000000000000..51baaa583309 --- /dev/null +++ b/backend/__migration__/types.ts @@ -0,0 +1,21 @@ +import { Db } from "mongodb"; + +export type Migration = { + name: string; + /** + * setup migration + * @param db mongo database + * @returns + */ + setup: (db: Db) => Promise; + /** + * + * @returns number of documents remaining for migration + */ + getRemainingCount: () => Promise; + /** + * + * @returns number of documents migrated + */ + migrate: (options: { batchSize: number }) => Promise; +}; diff --git a/backend/__tests__/__migration__/funboxResults.spec.ts b/backend/__tests__/__migration__/funboxResults.spec.ts new file mode 100644 index 000000000000..5d3e20577d59 --- /dev/null +++ b/backend/__tests__/__migration__/funboxResults.spec.ts @@ -0,0 +1,80 @@ +import { Db } from "mongodb"; +import FunboxResultMigration from "../../__migration__/funboxResult"; +import { getDb } from "../../src/init/db"; +import { DBResult } from "../../src/utils/result"; +import * as ResultDal from "../../src/dal/result"; +import { createUser } from "../__testData__/users"; + +describe("FunboxResults migration", () => { + const migration = new FunboxResultMigration(); + + beforeAll(async () => { + migration.setup(getDb() as Db); + }); + + it("migrates results with funbox", async () => { + //GIVEN + const { uid, resultId } = await createResult({ + funbox: "58008#read_ahead" as any, + acc: 95, + }); + + //WHEN + await migration.migrate({ batchSize: 1000 }); + + //THEN + const migratedResult = await ResultDal.getResult(uid, resultId); + + expect(migratedResult).toEqual( + expect.objectContaining({ + acc: 95, + funbox: ["58008", "read_ahead"], + }) + ); + }); + + it("migrates results with funbox none", async () => { + //GIVEN + const { uid, resultId } = await createResult({ + funbox: "none" as any, + acc: 95, + }); + + //WHEN + await migration.migrate({ batchSize: 1000 }); + + //THEN + const migratedResult = await ResultDal.getResult(uid, resultId); + expect(migratedResult).toEqual( + expect.objectContaining({ + acc: 95, + funbox: [], + }) + ); + }); + + it("migrates results without funbox", async () => { + //GIVEN + const { uid, resultId } = await createResult({ + acc: 95, + }); + + //WHEN + await migration.migrate({ batchSize: 1000 }); + + //THEN + const migratedResult = await ResultDal.getResult(uid, resultId); + expect(migratedResult).not.toHaveProperty("funbox"); + }); +}); + +async function createResult(result: Partial): Promise<{ + uid: string; + resultId: string; +}> { + const uid = (await createUser()).uid; + const resultId = ( + await ResultDal.addResult(uid, result as any) + ).insertedId.toHexString(); + return { uid, resultId }; +} diff --git a/backend/__tests__/__migration__/testActivity.spec.ts b/backend/__tests__/__migration__/testActivity.spec.ts index 2b65382fbbdd..9b5bfb9aebeb 100644 --- a/backend/__tests__/__migration__/testActivity.spec.ts +++ b/backend/__tests__/__migration__/testActivity.spec.ts @@ -1,17 +1,26 @@ -import * as Migration from "../../__migration__/testActivity"; +import TestActivityMigration from "../../__migration__/testActivity"; import * as UserTestData from "../__testData__/users"; import * as UserDal from "../../src/dal/user"; import * as ResultDal from "../../src/dal/result"; import { DBResult } from "../../src/utils/result"; +import { getDb } from "../../src/init/db"; +import { Db } from "mongodb"; describe("testActivity migration", () => { + const migration = new TestActivityMigration(); + + beforeAll(async () => { + migration.setup(getDb() as Db); + }); + it("migrates users without results", async () => { //given + const user1 = await UserTestData.createUser(); const user2 = await UserTestData.createUser(); //when - await Migration.migrate(); + await migration.migrate({ batchSize: 1 }); //then const readUser1 = await UserDal.getUser(user1.uid, ""); @@ -42,7 +51,7 @@ describe("testActivity migration", () => { await createResult(uid, 1704243600000); //when - await Migration.migrate(); + await migration.migrate({ batchSize: 1 }); //then const readWithResults = await UserDal.getUser(withResults.uid, ""); @@ -52,7 +61,7 @@ describe("testActivity migration", () => { }); const readWithoutResults = await UserDal.getUser(withoutResults.uid, ""); - expect(readWithoutResults.testActivity).toEqual({}); + expect(readWithoutResults.testActivity).toEqual(undefined); }); }); diff --git a/backend/src/dal/config.ts b/backend/src/dal/config.ts index a11b32fe5bbf..b5e50e92b89e 100644 --- a/backend/src/dal/config.ts +++ b/backend/src/dal/config.ts @@ -23,7 +23,7 @@ const configLegacyProperties = [ "enableAds", ]; -type DBConfig = { +export type DBConfig = { _id: ObjectId; uid: string; config: PartialConfig;