diff --git a/src/lib/server/db/apidata.db.ts b/src/lib/server/db/apidata.db.ts index a394719..26d2ba7 100755 --- a/src/lib/server/db/apidata.db.ts +++ b/src/lib/server/db/apidata.db.ts @@ -59,14 +59,7 @@ const upsertData = async ( // Insert new entries in chunks console.time("insertion time"); - const chunks = chunkArray(entries, chunkSize); - // .map(async (chunk) => { - // await surreal.insert(tableName, chunk); - // }); - // for (let i = 0; i < chunks.length; i += 2) { - // await Promise.all(chunks.slice(i, i + 2)); - // } - for (const chunk of chunks) { + for (const chunk of chunkArray(entries, chunkSize)) { await surreal.insert(tableName, chunk); } console.timeEnd("insertion time"); diff --git a/src/lib/server/db/apiuser.db.ts b/src/lib/server/db/apiuser.db.ts index 1e59da7..c13d508 100755 --- a/src/lib/server/db/apiuser.db.ts +++ b/src/lib/server/db/apiuser.db.ts @@ -5,8 +5,8 @@ import { type ApiPostUserWithParent, ApiUserTypes, DEFAULT_RANDOM_DISTRIBUTOR, - zApiPostUser, } from "$lib/utils/data.types"; +import { chunkArray } from "../array.chunk"; import { parseToRID, surreal } from "../connectors/surreal.db"; const getUserById = async (userId: string) => { @@ -159,89 +159,67 @@ const doesExist = async (userId?: string) => { return false; }; -const insertMany = async (data: LooseApiUser[], postUsers: ApiPostUser[]) => { - console.log("insertMany :: ", data.length); - await surreal.insert( - "apiuser", - data.map((e) => { - return { - ...e, - postData: !!postUsers.find((u) => u.userId === e.userId), - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }; - }), - ); -}; - -async function upsertMany( +async function insertMany( data: LooseApiUser[], - wipeTable: boolean, deleteUserType: | typeof ApiUserTypes.DISTRIBUTOR | typeof ApiUserTypes.DEALER, + tries = 0, ) { - const postUsers = await getAllPostUsers(); - console.log(postUsers); - if (wipeTable) { - console.log("[wipeTable] :: deleting all previous users"); + if (tries >= 3) { + console.log("Max retries reached for upserting users"); + return; + } + + try { + const postUsers = await getAllPostUsers(); + console.log(postUsers); + console.log("Wiping all previous users"); await surreal.query("delete from apiuser where userType = $userType", { userType: deleteUserType, }); - } - console.log("upsertMany :: ", data.length); - const toCreate = [] as LooseApiUser[]; - const out = data.map(async (apiUser) => { - // INFO: if you do want to keep disabled users, remove this check - if (apiUser.disable === 1) { - return; + + const entries = []; + for (const apiUser of data) { + if (apiUser.disable === 1) { + continue; + } + const uid = parseToRID(`apiuser:${apiUser.id}`); + const existingUser = await surreal.select(uid); + const postData = + existingUser?.postData ?? + !!postUsers.find((pu) => pu.userId === apiUser.userId) ?? + false; + entries.push({ + ...apiUser, + id: uid, + postData, + createdAt: existingUser?.createdAt ?? new Date().toISOString(), + updatedAt: new Date().toISOString(), + }); } - const uid = parseToRID(`apiuser:${apiUser.id}`); - const u = await surreal.select(uid); - if (!u || !u.id) { - toCreate.push(apiUser); - return; + + console.log(`[+] Creating ${entries.length} users into apiuser table`); + + let chunkSize = Math.floor( + Math.random() * (entries.length * 0.2 - entries.length * 0.05) + + entries.length * 0.05, + ); + if (chunkSize > 1000) chunkSize = 1000; + + console.log(`Chunk Size: ${chunkSize}`); + + console.time("Insertion Time"); + for (const chunk of chunkArray(entries, chunkSize)) { + await surreal.insert("apiuser", chunk); } - let postData = - u.postData ?? - !!postUsers.find((pu) => pu.userId === u.userId) ?? - false; - const qId = parseToRID(u.id); - await surreal.update(qId, { - id: u.id, - userId: apiUser.userId, - userType: apiUser.userType, - disableBooking: apiUser.disableBooking, - sendVoucher: apiUser.sendVoucher, - voucherGenerated: apiUser.voucherGenerated, - parentAdmin: apiUser.parentAdmin, - parentDistributor: apiUser.parentDistributor, - userName: apiUser.userName, - userCity: apiUser.userCity, - password: apiUser.password, - accessDenied: apiUser.accessDenied, - phoneNumber: apiUser.phoneNumber, - emailAddress: apiUser.emailAddress, - disable: apiUser.disable, - commission: apiUser.commission, - commissionPangora: apiUser.commissionPangora, - allowTitles: apiUser.allowTitles, - specialDealer: apiUser.specialDealer, - allowBalance: apiUser.allowBalance, - balance: apiUser.balance, - profitlossShare: apiUser.profitlossShare, - shareProfitonly: apiUser.shareProfitonly, - allowRemoveold: apiUser.allowRemoveold, - removeDays: apiUser.removeDays, - language: apiUser.language, - postData, - createdAt: u.createdAt, - updatedAt: new Date().toISOString(), - }); - }); - await Promise.allSettled(out); - if (toCreate.length > 0) { - await insertMany(toCreate, postUsers); + console.timeEnd("Insertion Time"); + + console.log(`[+] Successfully processed ${entries.length} users`); + } catch (e) { + console.log("Failed to process users, attempting retry"); + console.error(e); + return await insertMany(data, deleteUserType, tries + 1); } } @@ -256,7 +234,7 @@ export const dbApiUser = { getRandomDistributor, getRandomDealer, doesExist, - upsertMany, + insertMany, setPostDataFlagForUsers, updatePostUsersBalances, }; diff --git a/src/routes/admin/api-controls/+page.server.ts b/src/routes/admin/api-controls/+page.server.ts index 4119e11..b276028 100755 --- a/src/routes/admin/api-controls/+page.server.ts +++ b/src/routes/admin/api-controls/+page.server.ts @@ -27,7 +27,7 @@ export const actions = { errors: [{ message: done.message }], }); } - await dbApiUser.upsertMany(done.data, true, ApiUserTypes.DISTRIBUTOR); + await dbApiUser.insertMany(done.data, ApiUserTypes.DISTRIBUTOR); return { success: true, errors: [] as ServerError }; }, @@ -49,7 +49,7 @@ export const actions = { if (done.errors.length > 0) { return fail(400, { success: false, errors: done.errors }); } - await dbApiUser.upsertMany(done.dealers, true, ApiUserTypes.DEALER); + await dbApiUser.insertMany(done.dealers, ApiUserTypes.DEALER); return { success: true, errors: [] as ServerError }; }, } satisfies Actions;