diff --git a/src/lib/server/db/apidata.db.ts b/src/lib/server/db/apidata.db.ts index 73d929a..8943a9f 100755 --- a/src/lib/server/db/apidata.db.ts +++ b/src/lib/server/db/apidata.db.ts @@ -13,7 +13,7 @@ const upsertData = async ( ): Promise => { const tableName = getTableName(date); console.log(`[...] Upserting ${data.length} entries into ${tableName}`); - const alreadyPresentIds = new Set(); + const alreadyPresentIds = new Set(); try { const [alreadyPresent] = await surreal.query<[string[]]>( `select value id from type::table($tableName) where bookDate = $bookDate`, @@ -33,16 +33,7 @@ const upsertData = async ( const oldEntries = [] as any[]; const newEntries = [] as BookingEntry[]; for (let entry of data) { - if (!alreadyPresentIds.has(entry.id)) { - newEntries.push({ - ...entry, - id: `${tableName}:${entry.id}`, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - bookDate: entry.bookDate.split(" ")[0], - requestId: entry.requestId ?? "", - }); - } else { + if (alreadyPresentIds.has(`${tableName}:${entry.id}`)) { oldEntries.push({ distributorId: entry.distributorId, dealerId: entry.dealerId, @@ -57,7 +48,16 @@ const upsertData = async ( requestId: entry.requestId, updatedAt: new Date().toISOString(), }); + continue; } + newEntries.push({ + ...entry, + id: `${tableName}:${entry.id}`, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + bookDate: entry.bookDate.split(" ")[0], + requestId: entry.requestId ?? "", + }); } console.log( `[+] Inserting ${newEntries.length} new entries into ${tableName}`, @@ -65,7 +65,7 @@ const upsertData = async ( // 5 to 25% of the total data length let chunkSize = Math.floor( - Math.random() * (data.length * 0.25 - data.length * 0.05) + + Math.random() * (data.length * 0.2 - data.length * 0.05) + data.length * 0.05, ); if (chunkSize > 10_000) {