admin/rdv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'parallelized' the data posting

Browse Source
This commit is contained in:
bootunloader
2024-11-20 14:36:04 +02:00
parent 33b4d6c254
commit 1e75b062f0
3 changed files with 120 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
src/lib/server/postdata/post.handler.ts
View File

@@ -19,7 +19,7 @@ export type APIRespnose<T> = {
};
export function buildMessageString(
i: number,
ptr: number,
rows: PostDataEntry[],
distributorId: number,
dealerId: number,
@@ -31,7 +31,7 @@ export function buildMessageString(
let total = 0;
let startReqId = new Date().getTime();
let x = 0;
for (let j = i; j < i + jumpSize; j++) {
for (let j = ptr; j < ptr + jumpSize; j++) {
if (j >= rows.length) {
break;
}
@@ -48,7 +48,7 @@ export function buildMessageString(
message += `${reqId},${distributorId},${dealerId},${drawId},${date},${no},${f},${s},${mTotal};`;
}
message = message.slice(0, -1);
return { message, total, jumped: i + jumpSize };
return { message, total, jumped: ptr + jumpSize };
}
export async function postDataToApi(payload: {
@@ -82,29 +82,29 @@ export async function postDataToApi(payload: {
}
try {
for (const userId in dataByUser) {
// Process all users concurrently
const userPromises = Object.entries(dataByUser).map(
async ([userId, userData]) => {
const session = payload.sessions[userId];
const usr = payload.users.find((u) => u.userId === userId);
if (!usr) {
console.log(`[!] User ${userId} not found for posting to api`);
return {
ok: false,
detail: "User not found to post data with",
errors: [{ message: "User not found for request" }] as ServerError,
};
throw new Error(`User ${userId} not found for posting to api`);
}
const distId = usr.parentDistributor ?? 0;
const dealerId = Number(session.userId.split(":")[1]);
const drawId = Number(payload.draw.id.split(":")[1]);
const date = new Date().toISOString().split("T")[0];
let i = 0;
while (i < dataByUser[userId].length) {
let ptr = 0;
const userResponseIds = [] as { requestId: number; bookId: string }[];
while (ptr < userData.length) {
let tries = 0;
while (tries < 3) {
let { message, total, jumped } = buildMessageString(
i,
dataByUser[userId],
ptr,
userData,
distId,
dealerId,
drawId,
@@ -121,8 +121,8 @@ export async function postDataToApi(payload: {
bookDtos: { bookId: string; requestId: number }[];
}>;
if (rj.code === 200 && res.status === 200) {
i = jumped;
responsesIds.push(
ptr = jumped;
userResponseIds.push(
...rj.data.bookDtos.map((b) => ({
requestId: b.requestId as number,
bookId: b.bookId as string,
@@ -136,13 +136,9 @@ export async function postDataToApi(payload: {
}
if (tries >= 3) {
console.log(
`[!] Failed to send data to api for user ${userId}, deleting all booked entries...`,
);
console.log(responsesIds);
if (responsesIds.length > 0) {
if (userResponseIds.length > 0) {
const out = await deleteAllBookedEntries({
data: responsesIds,
data: userResponseIds,
closeTime: payload.draw.closeTime,
dealerId,
drawId,
@@ -150,16 +146,36 @@ export async function postDataToApi(payload: {
});
console.log(await out.text());
}
throw new Error(`Failed to send data to api for user ${userId}`);
}
}
return userResponseIds;
},
);
// Wait for all user processes to complete
const results = await Promise.allSettled(userPromises);
// Process results
let hasErrors = false;
results.forEach((result) => {
if (result.status === "fulfilled") {
responsesIds.push(...result.value);
} else {
hasErrors = true;
console.log(`[!] Error processing user: ${result.reason}`);
}
});
if (hasErrors) {
return {
ok: false,
detail: "Failed to post data to API halfway through",
detail: "Failed to post data to API for some users",
errors: [
{ message: "Failed to post data to API halfway through" },
{ message: "Failed to post data to API for some users" },
] as ServerError,
};
}
}
}
console.log(`[+] Finished sending ${payload.data.length} requests`);
console.log(`[?] Failed responses: ${failedResponses}`);
@@ -253,83 +269,6 @@ async function mockSendBatchRequest(
);
}
async function sendRequest(
requestId: number,
session: APISession,
body: PostDataEntry,
dealerId: number,
distributorId: number,
draw: Draw,
) {
return Fetch(`${constants.SCRAP_API_URL}/v1/book/add`, {
agent: new HttpsProxyAgent(`http://${session.ip}`),
method: "POST",
headers: {
...constants.SCRAP_API_BASE_HEADERS,
"Content-Type": "application/json;charset=UTF-8",
Authorization: session.sessionToken,
"User-Agent": getRandomUserAgent(),
},
body: JSON.stringify({
retryIndex: 0,
requestId: requestId,
date: new Date().toISOString().split("T")[0],
drawId: Number(draw.id.split(":")[1]),
closeTime: draw.closeTime,
dealerId: dealerId,
distributorId: distributorId,
number: body.number,
first: body.first,
second: body.second,
changedBalance: body.first + body.second,
}),
});
}
async function mockSendRequest(
requestId: number,
session: APISession,
body: PostDataEntry,
dealerId: number,
distributorId: number,
draw: Draw,
) {
// between 5 to 15 ms
await sleep(Math.floor(Math.random() * 10 + 5));
// // simulate a failed response, 20% of the time
if (Math.random() < 0.05) {
// return a failed response
return new Response(
JSON.stringify({
code: 500,
success: false,
message: "Failed",
data: {},
time: new Date().toISOString(),
}),
{
status: 500,
headers: { "Content-Type": "application/json" },
statusText: "Failed",
},
);
}
return new Response(
JSON.stringify({
code: 200,
success: true,
message: "Success",
data: {},
time: new Date().toISOString(),
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
statusText: "OK",
},
);
}
async function deleteAllBookedEntries({
session,
data,

4
src/routes/admin/api-controls/+page.server.ts
View File

@@ -21,7 +21,7 @@ export const actions = {
}
const done = await getDistributors(sess.sessionToken);
console.log(`[+] ${done.data.length} distributors found`);
fs.writeFileSync("distributors.json", JSON.stringify(done.data, null, 2));
// fs.writeFileSync("distributors.json", JSON.stringify(done.data, null, 2));
if (!done.ok) {
return fail(400, {
success: false,
@@ -45,7 +45,7 @@ export const actions = {
);
const done = await getDealers(sess.sessionToken, distributor_ids);
console.log(`[+] ${done.dealers.length} dealers found`);
fs.writeFileSync("dealers.json", JSON.stringify(done.dealers, null, 2));
// fs.writeFileSync("dealers.json", JSON.stringify(done.dealers, null, 2));
if (done.errors.length > 0) {
return fail(400, { success: false, errors: done.errors });
}

30
vite.config.ts.timestamp-1732105007011-f8ba7379070a1.mjs Normal file
View File

@@ -0,0 +1,30 @@
// vite.config.ts
import { sveltekit } from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/@sveltejs+kit@1.25.0_svelte@4.2.1_vite@5.3.5_@types+node@20.6.4_/node_modules/@sveltejs/kit/src/exports/vite/index.js";
import AutoImport from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-auto-import@0.16.6_rollup@3.29.2/node_modules/unplugin-auto-import/dist/vite.js";
import IconsResolver from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-icons@0.16.6/node_modules/unplugin-icons/dist/resolver.mjs";
import Icons from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-icons@0.16.6/node_modules/unplugin-icons/dist/vite.mjs";
import { defineConfig } from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/vitest@0.33.0/node_modules/vitest/dist/config.js";
var vite_config_default = defineConfig({
plugins: [
sveltekit(),
Icons({ compiler: "svelte" }),
AutoImport({
resolvers: [
IconsResolver({
prefix: "Icon",
extension: "svelte"
})
],
dts: "src/auto-imports.d.ts",
imports: ["svelte"],
dirs: ["src"],
ignore: ["**/*.test.{js,ts}", "**/*.spec.{js,ts}"],
exclude: [/node_modules/, /@sveltejs\/kit/, /.git/]
})
],
test: { include: ["src/**/*.{test,spec}.{js,ts}"] }
});
export {
vite_config_default as default
};
//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsidml0ZS5jb25maWcudHMiXSwKICAic291cmNlc0NvbnRlbnQiOiBbImNvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9kaXJuYW1lID0gXCIvVXNlcnMvYWV0aGVyL0RvY3VtZW50cy9kYXRhL1Byb2plY3RzL290aGVyL3Jkdi9mZVwiO2NvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9maWxlbmFtZSA9IFwiL1VzZXJzL2FldGhlci9Eb2N1bWVudHMvZGF0YS9Qcm9qZWN0cy9vdGhlci9yZHYvZmUvdml0ZS5jb25maWcudHNcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfaW1wb3J0X21ldGFfdXJsID0gXCJmaWxlOi8vL1VzZXJzL2FldGhlci9Eb2N1bWVudHMvZGF0YS9Qcm9qZWN0cy9vdGhlci9yZHYvZmUvdml0ZS5jb25maWcudHNcIjtpbXBvcnQgeyBzdmVsdGVraXQgfSBmcm9tIFwiQHN2ZWx0ZWpzL2tpdC92aXRlXCI7XG5pbXBvcnQgQXV0b0ltcG9ydCBmcm9tIFwidW5wbHVnaW4tYXV0by1pbXBvcnQvdml0ZVwiO1xuaW1wb3J0IEljb25zUmVzb2x2ZXIgZnJvbSBcInVucGx1Z2luLWljb25zL3Jlc29sdmVyXCI7XG5pbXBvcnQgSWNvbnMgZnJvbSBcInVucGx1Z2luLWljb25zL3ZpdGVcIjtcbmltcG9ydCB7IGRlZmluZUNvbmZpZyB9IGZyb20gXCJ2aXRlc3QvY29uZmlnXCI7XG5cbmV4cG9ydCBkZWZhdWx0IGRlZmluZUNvbmZpZyh7XG4gIHBsdWdpbnM6IFtcbiAgICBzdmVsdGVraXQoKSxcbiAgICBJY29ucyh7IGNvbXBpbGVyOiBcInN2ZWx0ZVwiIH0pLFxuICAgIEF1dG9JbXBvcnQoe1xuICAgICAgcmVzb2x2ZXJzOiBbXG4gICAgICAgIEljb25zUmVzb2x2ZXIoe1xuICAgICAgICAgIHByZWZpeDogXCJJY29uXCIsXG4gICAgICAgICAgZXh0ZW5zaW9uOiBcInN2ZWx0ZVwiLFxuICAgICAgICB9KSxcbiAgICAgIF0sXG4gICAgICBkdHM6IFwic3JjL2F1dG8taW1wb3J0cy5kLnRzXCIsXG4gICAgICBpbXBvcnRzOiBbXCJzdmVsdGVcIl0sXG4gICAgICBkaXJzOiBbXCJzcmNcIl0sXG4gICAgICBpZ25vcmU6IFtcIioqLyoudGVzdC57anMsdHN9XCIsIFwiKiovKi5zcGVjLntqcyx0c31cIl0sXG4gICAgICBleGNsdWRlOiBbL25vZGVfbW9kdWxlcy8sIC9Ac3ZlbHRlanNcXC9raXQvLCAvLmdpdC9dLFxuICAgIH0pLFxuICBdLFxuICB0ZXN0OiB7IGluY2x1ZGU6IFtcInNyYy8qKi8qLnt0ZXN0LHNwZWN9Lntqcyx0c31cIl0gfSxcbn0pO1xuIl0sCiAgIm1hcHBpbmdzIjogIjtBQUF3VSxTQUFTLGlCQUFpQjtBQUNsVyxPQUFPLGdCQUFnQjtBQUN2QixPQUFPLG1CQUFtQjtBQUMxQixPQUFPLFdBQVc7QUFDbEIsU0FBUyxvQkFBb0I7QUFFN0IsSUFBTyxzQkFBUSxhQUFhO0FBQUEsRUFDMUIsU0FBUztBQUFBLElBQ1AsVUFBVTtBQUFBLElBQ1YsTUFBTSxFQUFFLFVBQVUsU0FBUyxDQUFDO0FBQUEsSUFDNUIsV0FBVztBQUFBLE1BQ1QsV0FBVztBQUFBLFFBQ1QsY0FBYztBQUFBLFVBQ1osUUFBUTtBQUFBLFVBQ1IsV0FBVztBQUFBLFFBQ2IsQ0FBQztBQUFBLE1BQ0g7QUFBQSxNQUNBLEtBQUs7QUFBQSxNQUNMLFNBQVMsQ0FBQyxRQUFRO0FBQUEsTUFDbEIsTUFBTSxDQUFDLEtBQUs7QUFBQSxNQUNaLFFBQVEsQ0FBQyxxQkFBcUIsbUJBQW1CO0FBQUEsTUFDakQsU0FBUyxDQUFDLGdCQUFnQixrQkFBa0IsTUFBTTtBQUFBLElBQ3BELENBQUM7QUFBQSxFQUNIO0FBQUEsRUFDQSxNQUFNLEVBQUUsU0FBUyxDQUFDLDhCQUE4QixFQUFFO0FBQ3BELENBQUM7IiwKICAibmFtZXMiOiBbXQp9Cg==