simplifications

src/lib/server/db/apidata.db.ts

@@ -59,14 +59,7 @@ const upsertData = async (
 
         // Insert new entries in chunks
         console.time("insertion time");
-        const chunks = chunkArray(entries, chunkSize);
-        // .map(async (chunk) => {
-        //     await surreal.insert<BookingEntry>(tableName, chunk);
-        // });
-        // for (let i = 0; i < chunks.length; i += 2) {
-        //     await Promise.all(chunks.slice(i, i + 2));
-        // }
-        for (const chunk of chunks) {
+        for (const chunk of chunkArray(entries, chunkSize)) {
             await surreal.insert<BookingEntry>(tableName, chunk);
         }
         console.timeEnd("insertion time");

src/lib/server/db/apiuser.db.ts

@@ -5,8 +5,8 @@ import {
     type ApiPostUserWithParent,
     ApiUserTypes,
     DEFAULT_RANDOM_DISTRIBUTOR,
-    zApiPostUser,
 } from "$lib/utils/data.types";
+import { chunkArray } from "../array.chunk";
 import { parseToRID, surreal } from "../connectors/surreal.db";
 
 const getUserById = async (userId: string) => {
@@ -159,89 +159,67 @@ const doesExist = async (userId?: string) => {
     return false;
 };
 
-const insertMany = async (data: LooseApiUser[], postUsers: ApiPostUser[]) => {
-    console.log("insertMany :: ", data.length);
-    await surreal.insert<LooseApiUser>(
-        "apiuser",
-        data.map((e) => {
-            return {
-                ...e,
-                postData: !!postUsers.find((u) => u.userId === e.userId),
-                createdAt: new Date().toISOString(),
-                updatedAt: new Date().toISOString(),
-            };
-        }),
-    );
-};
-
-async function upsertMany(
+async function insertMany(
     data: LooseApiUser[],
-    wipeTable: boolean,
     deleteUserType:
         | typeof ApiUserTypes.DISTRIBUTOR
         | typeof ApiUserTypes.DEALER,
+    tries = 0,
 ) {
-    const postUsers = await getAllPostUsers();
-    console.log(postUsers);
-    if (wipeTable) {
-        console.log("[wipeTable] :: deleting all previous users");
+    if (tries >= 3) {
+        console.log("Max retries reached for upserting users");
+        return;
+    }
+
+    try {
+        const postUsers = await getAllPostUsers();
+        console.log(postUsers);
+        console.log("Wiping all previous users");
         await surreal.query("delete from apiuser where userType = $userType", {
             userType: deleteUserType,
         });
-    }
-    console.log("upsertMany :: ", data.length);
-    const toCreate = [] as LooseApiUser[];
-    const out = data.map(async (apiUser) => {
-        // INFO: if you do want to keep disabled users, remove this check
-        if (apiUser.disable === 1) {
-            return;
+
+        const entries = [];
+        for (const apiUser of data) {
+            if (apiUser.disable === 1) {
+                continue;
+            }
+            const uid = parseToRID(`apiuser:${apiUser.id}`);
+            const existingUser = await surreal.select<ApiUser>(uid);
+            const postData =
+                existingUser?.postData ??
+                !!postUsers.find((pu) => pu.userId === apiUser.userId) ??
+                false;
+            entries.push({
+                ...apiUser,
+                id: uid,
+                postData,
+                createdAt: existingUser?.createdAt ?? new Date().toISOString(),
+                updatedAt: new Date().toISOString(),
+            });
         }
-        const uid = parseToRID(`apiuser:${apiUser.id}`);
-        const u = await surreal.select<ApiUser>(uid);
-        if (!u || !u.id) {
-            toCreate.push(apiUser);
-            return;
+
+        console.log(`[+] Creating ${entries.length} users into apiuser table`);
+
+        let chunkSize = Math.floor(
+            Math.random() * (entries.length * 0.2 - entries.length * 0.05) +
+                entries.length * 0.05,
+        );
+        if (chunkSize > 1000) chunkSize = 1000;
+
+        console.log(`Chunk Size: ${chunkSize}`);
+
+        console.time("Insertion Time");
+        for (const chunk of chunkArray(entries, chunkSize)) {
+            await surreal.insert("apiuser", chunk);
         }
-        let postData =
-            u.postData ??
-            !!postUsers.find((pu) => pu.userId === u.userId) ??
-            false;
-        const qId = parseToRID(u.id);
-        await surreal.update<LooseApiUser>(qId, {
-            id: u.id,
-            userId: apiUser.userId,
-            userType: apiUser.userType,
-            disableBooking: apiUser.disableBooking,
-            sendVoucher: apiUser.sendVoucher,
-            voucherGenerated: apiUser.voucherGenerated,
-            parentAdmin: apiUser.parentAdmin,
-            parentDistributor: apiUser.parentDistributor,
-            userName: apiUser.userName,
-            userCity: apiUser.userCity,
-            password: apiUser.password,
-            accessDenied: apiUser.accessDenied,
-            phoneNumber: apiUser.phoneNumber,
-            emailAddress: apiUser.emailAddress,
-            disable: apiUser.disable,
-            commission: apiUser.commission,
-            commissionPangora: apiUser.commissionPangora,
-            allowTitles: apiUser.allowTitles,
-            specialDealer: apiUser.specialDealer,
-            allowBalance: apiUser.allowBalance,
-            balance: apiUser.balance,
-            profitlossShare: apiUser.profitlossShare,
-            shareProfitonly: apiUser.shareProfitonly,
-            allowRemoveold: apiUser.allowRemoveold,
-            removeDays: apiUser.removeDays,
-            language: apiUser.language,
-            postData,
-            createdAt: u.createdAt,
-            updatedAt: new Date().toISOString(),
-        });
-    });
-    await Promise.allSettled(out);
-    if (toCreate.length > 0) {
-        await insertMany(toCreate, postUsers);
+        console.timeEnd("Insertion Time");
+
+        console.log(`[+] Successfully processed ${entries.length} users`);
+    } catch (e) {
+        console.log("Failed to process users, attempting retry");
+        console.error(e);
+        return await insertMany(data, deleteUserType, tries + 1);
     }
 }
 
@@ -256,7 +234,7 @@ export const dbApiUser = {
     getRandomDistributor,
     getRandomDealer,
     doesExist,
-    upsertMany,
+    insertMany,
     setPostDataFlagForUsers,
     updatePostUsersBalances,
 };

src/routes/admin/api-controls/+page.server.ts

@@ -27,7 +27,7 @@ export const actions = {
                 errors: [{ message: done.message }],
             });
         }
-        await dbApiUser.upsertMany(done.data, true, ApiUserTypes.DISTRIBUTOR);
+        await dbApiUser.insertMany(done.data, ApiUserTypes.DISTRIBUTOR);
         return { success: true, errors: [] as ServerError };
     },
 
@@ -49,7 +49,7 @@ export const actions = {
         if (done.errors.length > 0) {
             return fail(400, { success: false, errors: done.errors });
         }
-        await dbApiUser.upsertMany(done.dealers, true, ApiUserTypes.DEALER);
+        await dbApiUser.insertMany(done.dealers, ApiUserTypes.DEALER);
         return { success: true, errors: [] as ServerError };
     },
 } satisfies Actions;