import type { BookingEntry } from "$lib/utils/data.types"; import { chunkArray } from "../array.chunk"; import { surreal } from "../connectors/surreal.db"; const getTableName = (date: string) => { return `apidata${date.replaceAll("-", "")}`; }; const upsertData = async ( data: BookingEntry[], date: string, tries: number = 0, ): Promise => { if (tries >= 3) { console.log("Max tries exceeded for processing data"); return; } const tableName = getTableName(date); const drawId = data[0].drawId; console.log(`[...] Processing ${data.length} entries for ${tableName}`); try { // Delete existing entries for this date console.log( `[...] Deleting existing entries for ${date} in ${tableName}`, ); console.time("deletion time"); await surreal.query( `DELETE type::table($tableName) WHERE bookDate = $bookDate AND drawId = $drawId`, { tableName, bookDate: date, drawId }, ); console.timeEnd("deletion time"); // Prepare new entries const entries = data.map((entry) => ({ ...entry, id: `${tableName}:${entry.id}`, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), bookDate: entry.bookDate.split(" ")[0], requestId: entry.requestId ?? "", })); // Calculate chunk size (5 to 25% of total data length, max 10,000) let chunkSize = Math.floor( Math.random() * (data.length * 0.2 - data.length * 0.05) + data.length * 0.05, ); if (chunkSize > 4_000) { chunkSize = 4_000; } console.log(`Chunk Size: ${chunkSize}`); console.log( `[+] Inserting ${entries.length} entries into ${tableName}`, ); // Insert new entries in chunks console.time("insertion time"); const chunks = chunkArray(entries, chunkSize); // .map(async (chunk) => { // await surreal.insert(tableName, chunk); // }); // for (let i = 0; i < chunks.length; i += 2) { // await Promise.all(chunks.slice(i, i + 2)); // } for (const chunk of chunks) { await surreal.insert(tableName, chunk); } console.timeEnd("insertion time"); console.log( `[+] Successfully processed ${data.length} entries into ${tableName}`, ); } catch (err) { console.log("Failed to process data, attempting retry"); return await upsertData(data, date, tries + 1); } }; const upsertDataDED = async ( data: BookingEntry[], date: string, tries: number = 0, ): Promise => { const tableName = getTableName(date); console.log(`[...] Upserting ${data.length} entries into ${tableName}`); const alreadyPresentIds = new Set(); try { const [alreadyPresent] = await surreal.query<[string[]]>( `select value id from type::table($tableName) where bookDate = $bookDate`, { tableName, bookDate: date }, ); for (let eId of alreadyPresent ?? []) { alreadyPresentIds.add(eId); } } catch (err) { console.log("Failed to fetch, seeing if can try again"); if (tries >= 3) { console.log( "Max tries exceeded for initial fetch for upserting data", ); return; } return await upsertData(data, date, tries++); } const oldEntries = [] as any[]; const newEntries = [] as BookingEntry[]; for (let entry of data) { if (alreadyPresentIds.has(`${tableName}:${entry.id}`)) { oldEntries.push({ distributorId: entry.distributorId, dealerId: entry.dealerId, drawId: entry.drawId, bookDate: entry.bookDate.split(" ")[0], number: entry.number, first: entry.first, second: entry.second, changedBalance: entry.changedBalance, sheetName: entry.sheetName, sheetId: entry.sheetId, requestId: entry.requestId, updatedAt: new Date().toISOString(), }); continue; } newEntries.push({ ...entry, id: `${tableName}:${entry.id}`, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), bookDate: entry.bookDate.split(" ")[0], requestId: entry.requestId ?? "", }); } console.log( `[+] Inserting ${newEntries.length} new entries into ${tableName}`, ); // 5 to 25% of the total data length let chunkSize = Math.floor( Math.random() * (data.length * 0.2 - data.length * 0.05) + data.length * 0.05, ); if (chunkSize > 10_000) { chunkSize = 10_000; } console.log(`Chunk Size : ${chunkSize}`); console.log(`[+] Inserting new entries`); console.time("insertion time"); const chunks = chunkArray(newEntries, chunkSize).map(async (chunk) => { await surreal.insert(tableName, chunk); }); for (let i = 0; i < chunks.length; i += 2) { await Promise.all(chunks.slice(i, i + 2)); } console.timeEnd("insertion time"); console.log( `[+] Updating ${oldEntries.length} old entries into ${tableName}`, ); const chunks2 = chunkArray(oldEntries, chunkSize).map(async (chunk) => { await Promise.all( chunk.map(async (entry) => { // @ts-ignore await surreal.update(`${tableName}:${entry.id}`, { distributorId: entry.distributorId, dealerId: entry.dealerId, drawId: entry.drawId, bookDate: entry.bookDate.split(" ")[0], number: entry.number, first: entry.first, second: entry.second, changedBalance: entry.changedBalance, sheetName: entry.sheetName, sheetId: entry.sheetId, requestId: entry.requestId, updatedAt: new Date().toISOString(), }); }), ); }); console.time("update time"); for (let i = 0; i < chunks2.length; i += 10) { await Promise.all(chunks2.slice(i, i + 10)); } console.timeEnd("update time"); console.log( `[+] Successfully upserted ${data.length} entries into ${tableName}`, ); }; const getBookingEntriesForDealer = async ( date: string, drawId: string, userId: string, sorted?: boolean, ) => { const tableName = getTableName(date); let query = `select * from type::table($tableName) where bookDate = $date and dealerId = $userId and drawId = $drawId`; if (sorted) { query += " order by requestId desc"; } const [data] = await surreal.query<[BookingEntry[]]>(query, { tableName, date: `${date}`, userId: parseInt(userId), drawId: parseInt(drawId), }); console.log( `Found ${JSON.stringify( data, )} entries for ${userId}, filters are ${date}, ${drawId} for ${tableName}`, ); return data ?? []; }; const getBookingEntriesByDraw = async (date: string, drawId: string) => { const tableName = getTableName(date); const [data] = await surreal.query<[BookingEntry[]]>( `select * from type::table($tableName) where bookDate = $date and drawId = $drawId`, { tableName, date: date, drawId: parseInt( drawId.includes(":") ? drawId.split(":")[1] : drawId, ), }, ); return data ?? []; }; const deleteDataOlderThan2Weeks = async () => { const [out] = await surreal.query("info for db"); // @ts-ignore const tableNames = Object.keys(out.result.tables); const twoWeeksAgo = new Date(); twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14); for (const tableName of tableNames) { if (tableName.startsWith("apidata")) { const datePart = tableName.slice(7); const d = new Date( parseInt(datePart.slice(0, 4), 10), parseInt(datePart.slice(4, 6), 10) - 1, // Month is 0-based in JavaScript Date parseInt(datePart.slice(6, 8), 10), ); if (d < twoWeeksAgo) { console.log(`[...] Deleting ${tableName}`); await surreal.query("remove table if exists " + tableName); console.log(`[+] Deleted ${tableName}`); } } else if (tableName.startsWith("apipostdata_")) { const datePart = tableName.slice(12); const d = new Date( parseInt(datePart.slice(0, 4), 10), parseInt(datePart.slice(4, 6), 10) - 1, // Month is 0-based in JavaScript Date parseInt(datePart.slice(6, 8), 10), ); if (d < twoWeeksAgo) { console.log(`[...] Deleting ${tableName}`); await surreal.query("remove table if exists " + tableName); console.log(`[+] Deleted ${tableName}`); } } else { console.log(`Skipping ${tableName}`); } } }; export const dbApiData = { upsertData, getBookingEntriesForDealer, getBookingEntriesByDraw, deleteDataOlderThan2Weeks, };