initial commit ??
This commit is contained in:
bootunloader
212
src/lib/server/db/apidata.db.ts
Executable file
212
src/lib/server/db/apidata.db.ts
Executable file
@@ -0,0 +1,212 @@
|
||||
import type { BookingEntry } from "$lib/utils/data.types";
|
||||
import { chunkArray } from "../array.chunk";
|
||||
import { surreal } from "../connectors/surreal.db";
|
||||
|
||||
const getTableName = (date: string) => {
|
||||
return `apidata${date.replaceAll("-", "")}`;
|
||||
};
|
||||
|
||||
const upsertData = async (
|
||||
data: BookingEntry[],
|
||||
date: string,
|
||||
tries: number = 0,
|
||||
): Promise<void> => {
|
||||
const tableName = getTableName(date);
|
||||
console.log(`[...] Upserting ${data.length} entries into ${tableName}`);
|
||||
const alreadyPresentIds = new Set();
|
||||
try {
|
||||
const [alreadyPresent] = await surreal.query<[string[]]>(
|
||||
`select value id from type::table($tableName) where bookDate = $bookDate`,
|
||||
{ tableName, bookDate: date },
|
||||
);
|
||||
for (let eId of alreadyPresent.result ?? []) {
|
||||
alreadyPresentIds.add(eId);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log("Failed to fetch, seeing if can try again");
|
||||
if (tries >= 3) {
|
||||
console.log("Max tries exceeded for initial fetch for upserting data");
|
||||
return;
|
||||
}
|
||||
return await upsertData(data, date, tries++);
|
||||
}
|
||||
const oldEntries = [] as any[];
|
||||
const newEntries = [] as BookingEntry[];
|
||||
for (let entry of data) {
|
||||
if (!alreadyPresentIds.has(entry.id)) {
|
||||
newEntries.push({
|
||||
...entry,
|
||||
id: `${tableName}:${entry.id}`,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
bookDate: entry.bookDate.split(" ")[0],
|
||||
requestId: entry.requestId ?? "",
|
||||
});
|
||||
} else {
|
||||
oldEntries.push({
|
||||
distributorId: entry.distributorId,
|
||||
dealerId: entry.dealerId,
|
||||
drawId: entry.drawId,
|
||||
bookDate: entry.bookDate.split(" ")[0],
|
||||
number: entry.number,
|
||||
first: entry.first,
|
||||
second: entry.second,
|
||||
changedBalance: entry.changedBalance,
|
||||
sheetName: entry.sheetName,
|
||||
sheetId: entry.sheetId,
|
||||
requestId: entry.requestId,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[+] Inserting ${newEntries.length} new entries into ${tableName}`,
|
||||
);
|
||||
|
||||
// 5 to 25% of the total data length
|
||||
let chunkSize = Math.floor(
|
||||
Math.random() * (data.length * 0.25 - data.length * 0.05) +
|
||||
data.length * 0.05,
|
||||
);
|
||||
if (chunkSize > 10_000) {
|
||||
chunkSize = 10_000;
|
||||
}
|
||||
|
||||
console.log(`Chunk Size : ${chunkSize}`);
|
||||
|
||||
console.log(`[+] Inserting new entries`);
|
||||
console.time("insertion time");
|
||||
const chunks = chunkArray(newEntries, chunkSize).map(async (chunk) => {
|
||||
await surreal.insert<BookingEntry>(tableName, chunk);
|
||||
});
|
||||
for (let i = 0; i < chunks.length; i += 2) {
|
||||
await Promise.all(chunks.slice(i, i + 2));
|
||||
}
|
||||
console.timeEnd("insertion time");
|
||||
|
||||
console.log(
|
||||
`[+] Updating ${oldEntries.length} old entries into ${tableName}`,
|
||||
);
|
||||
|
||||
const chunks2 = chunkArray(oldEntries, chunkSize).map(async (chunk) => {
|
||||
await Promise.all(
|
||||
chunk.map(async (entry) => {
|
||||
// @ts-ignore
|
||||
await surreal.update<BookingEntry>(`${tableName}:${entry.id}`, {
|
||||
distributorId: entry.distributorId,
|
||||
dealerId: entry.dealerId,
|
||||
drawId: entry.drawId,
|
||||
bookDate: entry.bookDate.split(" ")[0],
|
||||
number: entry.number,
|
||||
first: entry.first,
|
||||
second: entry.second,
|
||||
changedBalance: entry.changedBalance,
|
||||
sheetName: entry.sheetName,
|
||||
sheetId: entry.sheetId,
|
||||
requestId: entry.requestId,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
console.time("update time");
|
||||
for (let i = 0; i < chunks2.length; i += 10) {
|
||||
await Promise.all(chunks2.slice(i, i + 10));
|
||||
}
|
||||
console.timeEnd("update time");
|
||||
|
||||
console.log(
|
||||
`[+] Successfully upserted ${data.length} entries into ${tableName}`,
|
||||
);
|
||||
};
|
||||
|
||||
const getBookingEntriesForDealer = async (
|
||||
date: string,
|
||||
drawId: string,
|
||||
userId: string,
|
||||
sorted?: boolean,
|
||||
) => {
|
||||
const tableName = getTableName(date);
|
||||
let query = `select * from type::table($tableName) where bookDate = $date and dealerId = $userId and drawId = $drawId`;
|
||||
if (sorted) {
|
||||
query += " order by requestId desc";
|
||||
}
|
||||
const [data] = await surreal.query<[BookingEntry[]]>(query, {
|
||||
tableName,
|
||||
date: `${date}`,
|
||||
userId: parseInt(userId),
|
||||
drawId: parseInt(drawId),
|
||||
});
|
||||
console.log(
|
||||
`Found ${JSON.stringify(
|
||||
data,
|
||||
)} entries for ${userId}, filters are ${date}, ${drawId} for ${tableName}`,
|
||||
);
|
||||
if (data.status === "OK") {
|
||||
return data.result ?? [];
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const getBookingEntriesByDraw = async (date: string, drawId: string) => {
|
||||
const tableName = getTableName(date);
|
||||
const [data] = await surreal.query<[BookingEntry[]]>(
|
||||
`select * from type::table($tableName) where bookDate = $date and drawId = $drawId`,
|
||||
{
|
||||
tableName,
|
||||
date: date,
|
||||
drawId: parseInt(drawId.includes(":") ? drawId.split(":")[1] : drawId),
|
||||
},
|
||||
);
|
||||
if (data.status === "OK") {
|
||||
return data.result ?? [];
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const deleteDataOlderThan2Weeks = async () => {
|
||||
const [out] = await surreal.query("info for db");
|
||||
// @ts-ignore
|
||||
const tableNames = Object.keys(out.result.tables);
|
||||
|
||||
const twoWeeksAgo = new Date();
|
||||
twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14);
|
||||
|
||||
for (const tableName of tableNames) {
|
||||
if (tableName.startsWith("apidata")) {
|
||||
const datePart = tableName.slice(7);
|
||||
const d = new Date(
|
||||
parseInt(datePart.slice(0, 4), 10),
|
||||
parseInt(datePart.slice(4, 6), 10) - 1, // Month is 0-based in JavaScript Date
|
||||
parseInt(datePart.slice(6, 8), 10),
|
||||
);
|
||||
if (d < twoWeeksAgo) {
|
||||
console.log(`[...] Deleting ${tableName}`);
|
||||
await surreal.query("remove table if exists " + tableName);
|
||||
console.log(`[+] Deleted ${tableName}`);
|
||||
}
|
||||
} else if (tableName.startsWith("apipostdata_")) {
|
||||
const datePart = tableName.slice(12);
|
||||
const d = new Date(
|
||||
parseInt(datePart.slice(0, 4), 10),
|
||||
parseInt(datePart.slice(4, 6), 10) - 1, // Month is 0-based in JavaScript Date
|
||||
parseInt(datePart.slice(6, 8), 10),
|
||||
);
|
||||
if (d < twoWeeksAgo) {
|
||||
console.log(`[...] Deleting ${tableName}`);
|
||||
await surreal.query("remove table if exists " + tableName);
|
||||
console.log(`[+] Deleted ${tableName}`);
|
||||
}
|
||||
} else {
|
||||
console.log(`Skipping ${tableName}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const dbApiData = {
|
||||
upsertData,
|
||||
getBookingEntriesForDealer,
|
||||
getBookingEntriesByDraw,
|
||||
deleteDataOlderThan2Weeks,
|
||||
};
|
||||
Reference in New Issue
Block a user