'parallelized' the data posting

src/lib/server/postdata/post.handler.ts

@@ -19,7 +19,7 @@ export type APIRespnose<T> = {
 };
 
 export function buildMessageString(
-  i: number,
+  ptr: number,
   rows: PostDataEntry[],
   distributorId: number,
   dealerId: number,
@@ -31,7 +31,7 @@ export function buildMessageString(
   let total = 0;
   let startReqId = new Date().getTime();
   let x = 0;
-  for (let j = i; j < i + jumpSize; j++) {
+  for (let j = ptr; j < ptr + jumpSize; j++) {
     if (j >= rows.length) {
       break;
     }
@@ -48,7 +48,7 @@ export function buildMessageString(
     message += `${reqId},${distributorId},${dealerId},${drawId},${date},${no},${f},${s},${mTotal};`;
   }
   message = message.slice(0, -1);
-  return { message, total, jumped: i + jumpSize };
+  return { message, total, jumped: ptr + jumpSize };
 }
 
 export async function postDataToApi(payload: {
@@ -82,83 +82,99 @@ export async function postDataToApi(payload: {
   }
 
   try {
-    for (const userId in dataByUser) {
+    // Process all users concurrently
-      const session = payload.sessions[userId];
+    const userPromises = Object.entries(dataByUser).map(
-      const usr = payload.users.find((u) => u.userId === userId);
+      async ([userId, userData]) => {
-      if (!usr) {
+        const session = payload.sessions[userId];
-        console.log(`[!] User ${userId} not found for posting to api`);
+        const usr = payload.users.find((u) => u.userId === userId);
-        return {
+        if (!usr) {
-          ok: false,
+          throw new Error(`User ${userId} not found for posting to api`);
-          detail: "User not found to post data with",
-          errors: [{ message: "User not found for request" }] as ServerError,
-        };
-      }
-      const distId = usr.parentDistributor ?? 0;
-      const dealerId = Number(session.userId.split(":")[1]);
-      const drawId = Number(payload.draw.id.split(":")[1]);
-      const date = new Date().toISOString().split("T")[0];
-
-      let i = 0;
-      while (i < dataByUser[userId].length) {
-        let tries = 0;
-        while (tries < 3) {
-          let { message, total, jumped } = buildMessageString(
-            i,
-            dataByUser[userId],
-            distId,
-            dealerId,
-            drawId,
-            date,
-          );
-          const res = await sendBatchRequest(
-            session,
-            dealerId,
-            payload.draw,
-            total,
-            message,
-          );
-          const rj = (await res.json()) as APIRespnose<{
-            bookDtos: { bookId: string; requestId: number }[];
-          }>;
-          if (rj.code === 200 && res.status === 200) {
-            i = jumped;
-            responsesIds.push(
-              ...rj.data.bookDtos.map((b) => ({
-                requestId: b.requestId as number,
-                bookId: b.bookId as string,
-              })),
-            );
-            successResponses++;
-            break;
-          }
-          failedResponses++;
-          tries++;
         }
 
-        if (tries >= 3) {
+        const distId = usr.parentDistributor ?? 0;
-          console.log(
+        const dealerId = Number(session.userId.split(":")[1]);
-            `[!] Failed to send data to api for user ${userId}, deleting all booked entries...`,
+        const drawId = Number(payload.draw.id.split(":")[1]);
-          );
+        const date = new Date().toISOString().split("T")[0];
-          console.log(responsesIds);
+
-          if (responsesIds.length > 0) {
+        let ptr = 0;
-            const out = await deleteAllBookedEntries({
+        const userResponseIds = [] as { requestId: number; bookId: string }[];
-              data: responsesIds,
+
-              closeTime: payload.draw.closeTime,
+        while (ptr < userData.length) {
+          let tries = 0;
+          while (tries < 3) {
+            let { message, total, jumped } = buildMessageString(
+              ptr,
+              userData,
+              distId,
               dealerId,
               drawId,
+              date,
+            );
+            const res = await sendBatchRequest(
               session,
-            });
+              dealerId,
-            console.log(await out.text());
+              payload.draw,
+              total,
+              message,
+            );
+            const rj = (await res.json()) as APIRespnose<{
+              bookDtos: { bookId: string; requestId: number }[];
+            }>;
+            if (rj.code === 200 && res.status === 200) {
+              ptr = jumped;
+              userResponseIds.push(
+                ...rj.data.bookDtos.map((b) => ({
+                  requestId: b.requestId as number,
+                  bookId: b.bookId as string,
+                })),
+              );
+              successResponses++;
+              break;
+            }
+            failedResponses++;
+            tries++;
+          }
+
+          if (tries >= 3) {
+            if (userResponseIds.length > 0) {
+              const out = await deleteAllBookedEntries({
+                data: userResponseIds,
+                closeTime: payload.draw.closeTime,
+                dealerId,
+                drawId,
+                session,
+              });
+              console.log(await out.text());
+            }
+            throw new Error(`Failed to send data to api for user ${userId}`);
           }
-          return {
-            ok: false,
-            detail: "Failed to post data to API halfway through",
-            errors: [
-              { message: "Failed to post data to API halfway through" },
-            ] as ServerError,
-          };
         }
+        return userResponseIds;
+      },
+    );
+
+    // Wait for all user processes to complete
+    const results = await Promise.allSettled(userPromises);
+
+    // Process results
+    let hasErrors = false;
+    results.forEach((result) => {
+      if (result.status === "fulfilled") {
+        responsesIds.push(...result.value);
+      } else {
+        hasErrors = true;
+        console.log(`[!] Error processing user: ${result.reason}`);
       }
+    });
+
+    if (hasErrors) {
+      return {
+        ok: false,
+        detail: "Failed to post data to API for some users",
+        errors: [
+          { message: "Failed to post data to API for some users" },
+        ] as ServerError,
+      };
     }
 
     console.log(`[+] Finished sending ${payload.data.length} requests`);
@@ -253,83 +269,6 @@ async function mockSendBatchRequest(
   );
 }
 
-async function sendRequest(
-  requestId: number,
-  session: APISession,
-  body: PostDataEntry,
-  dealerId: number,
-  distributorId: number,
-  draw: Draw,
-) {
-  return Fetch(`${constants.SCRAP_API_URL}/v1/book/add`, {
-    agent: new HttpsProxyAgent(`http://${session.ip}`),
-    method: "POST",
-    headers: {
-      ...constants.SCRAP_API_BASE_HEADERS,
-      "Content-Type": "application/json;charset=UTF-8",
-      Authorization: session.sessionToken,
-      "User-Agent": getRandomUserAgent(),
-    },
-    body: JSON.stringify({
-      retryIndex: 0,
-      requestId: requestId,
-      date: new Date().toISOString().split("T")[0],
-      drawId: Number(draw.id.split(":")[1]),
-      closeTime: draw.closeTime,
-      dealerId: dealerId,
-      distributorId: distributorId,
-      number: body.number,
-      first: body.first,
-      second: body.second,
-      changedBalance: body.first + body.second,
-    }),
-  });
-}
-
-async function mockSendRequest(
-  requestId: number,
-  session: APISession,
-  body: PostDataEntry,
-  dealerId: number,
-  distributorId: number,
-  draw: Draw,
-) {
-  // between 5 to 15 ms
-  await sleep(Math.floor(Math.random() * 10 + 5));
-  // // simulate a failed response, 20% of the time
-  if (Math.random() < 0.05) {
-    // return a failed response
-    return new Response(
-      JSON.stringify({
-        code: 500,
-        success: false,
-        message: "Failed",
-        data: {},
-        time: new Date().toISOString(),
-      }),
-      {
-        status: 500,
-        headers: { "Content-Type": "application/json" },
-        statusText: "Failed",
-      },
-    );
-  }
-  return new Response(
-    JSON.stringify({
-      code: 200,
-      success: true,
-      message: "Success",
-      data: {},
-      time: new Date().toISOString(),
-    }),
-    {
-      status: 200,
-      headers: { "Content-Type": "application/json" },
-      statusText: "OK",
-    },
-  );
-}
-
 async function deleteAllBookedEntries({
   session,
   data,

src/routes/admin/api-controls/+page.server.ts

@@ -21,7 +21,7 @@ export const actions = {
     }
     const done = await getDistributors(sess.sessionToken);
     console.log(`[+] ${done.data.length} distributors found`);
-    fs.writeFileSync("distributors.json", JSON.stringify(done.data, null, 2));
+    // fs.writeFileSync("distributors.json", JSON.stringify(done.data, null, 2));
     if (!done.ok) {
       return fail(400, {
         success: false,
@@ -45,7 +45,7 @@ export const actions = {
     );
     const done = await getDealers(sess.sessionToken, distributor_ids);
     console.log(`[+] ${done.dealers.length} dealers found`);
-    fs.writeFileSync("dealers.json", JSON.stringify(done.dealers, null, 2));
+    // fs.writeFileSync("dealers.json", JSON.stringify(done.dealers, null, 2));
     if (done.errors.length > 0) {
       return fail(400, { success: false, errors: done.errors });
     }

vite.config.ts.timestamp-1732105007011-f8ba7379070a1.mjs

@@ -0,0 +1,30 @@
+// vite.config.ts
+import { sveltekit } from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/@sveltejs+kit@1.25.0_svelte@4.2.1_vite@5.3.5_@types+node@20.6.4_/node_modules/@sveltejs/kit/src/exports/vite/index.js";
+import AutoImport from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-auto-import@0.16.6_rollup@3.29.2/node_modules/unplugin-auto-import/dist/vite.js";
+import IconsResolver from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-icons@0.16.6/node_modules/unplugin-icons/dist/resolver.mjs";
+import Icons from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/unplugin-icons@0.16.6/node_modules/unplugin-icons/dist/vite.mjs";
+import { defineConfig } from "file:///Users/aether/Documents/data/Projects/other/rdv/fe/node_modules/.pnpm/vitest@0.33.0/node_modules/vitest/dist/config.js";
+var vite_config_default = defineConfig({
+  plugins: [
+    sveltekit(),
+    Icons({ compiler: "svelte" }),
+    AutoImport({
+      resolvers: [
+        IconsResolver({
+          prefix: "Icon",
+          extension: "svelte"
+        })
+      ],
+      dts: "src/auto-imports.d.ts",
+      imports: ["svelte"],
+      dirs: ["src"],
+      ignore: ["**/*.test.{js,ts}", "**/*.spec.{js,ts}"],
+      exclude: [/node_modules/, /@sveltejs\/kit/, /.git/]
+    })
+  ],
+  test: { include: ["src/**/*.{test,spec}.{js,ts}"] }
+});
+export {
+  vite_config_default as default
+};
+//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsidml0ZS5jb25maWcudHMiXSwKICAic291cmNlc0NvbnRlbnQiOiBbImNvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9kaXJuYW1lID0gXCIvVXNlcnMvYWV0aGVyL0RvY3VtZW50cy9kYXRhL1Byb2plY3RzL290aGVyL3Jkdi9mZVwiO2NvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9maWxlbmFtZSA9IFwiL1VzZXJzL2FldGhlci9Eb2N1bWVudHMvZGF0YS9Qcm9qZWN0cy9vdGhlci9yZHYvZmUvdml0ZS5jb25maWcudHNcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfaW1wb3J0X21ldGFfdXJsID0gXCJmaWxlOi8vL1VzZXJzL2FldGhlci9Eb2N1bWVudHMvZGF0YS9Qcm9qZWN0cy9vdGhlci9yZHYvZmUvdml0ZS5jb25maWcudHNcIjtpbXBvcnQgeyBzdmVsdGVraXQgfSBmcm9tIFwiQHN2ZWx0ZWpzL2tpdC92aXRlXCI7XG5pbXBvcnQgQXV0b0ltcG9ydCBmcm9tIFwidW5wbHVnaW4tYXV0by1pbXBvcnQvdml0ZVwiO1xuaW1wb3J0IEljb25zUmVzb2x2ZXIgZnJvbSBcInVucGx1Z2luLWljb25zL3Jlc29sdmVyXCI7XG5pbXBvcnQgSWNvbnMgZnJvbSBcInVucGx1Z2luLWljb25zL3ZpdGVcIjtcbmltcG9ydCB7IGRlZmluZUNvbmZpZyB9IGZyb20gXCJ2aXRlc3QvY29uZmlnXCI7XG5cbmV4cG9ydCBkZWZhdWx0IGRlZmluZUNvbmZpZyh7XG4gIHBsdWdpbnM6IFtcbiAgICBzdmVsdGVraXQoKSxcbiAgICBJY29ucyh7IGNvbXBpbGVyOiBcInN2ZWx0ZVwiIH0pLFxuICAgIEF1dG9JbXBvcnQoe1xuICAgICAgcmVzb2x2ZXJzOiBbXG4gICAgICAgIEljb25zUmVzb2x2ZXIoe1xuICAgICAgICAgIHByZWZpeDogXCJJY29uXCIsXG4gICAgICAgICAgZXh0ZW5zaW9uOiBcInN2ZWx0ZVwiLFxuICAgICAgICB9KSxcbiAgICAgIF0sXG4gICAgICBkdHM6IFwic3JjL2F1dG8taW1wb3J0cy5kLnRzXCIsXG4gICAgICBpbXBvcnRzOiBbXCJzdmVsdGVcIl0sXG4gICAgICBkaXJzOiBbXCJzcmNcIl0sXG4gICAgICBpZ25vcmU6IFtcIioqLyoudGVzdC57anMsdHN9XCIsIFwiKiovKi5zcGVjLntqcyx0c31cIl0sXG4gICAgICBleGNsdWRlOiBbL25vZGVfbW9kdWxlcy8sIC9Ac3ZlbHRlanNcXC9raXQvLCAvLmdpdC9dLFxuICAgIH0pLFxuICBdLFxuICB0ZXN0OiB7IGluY2x1ZGU6IFtcInNyYy8qKi8qLnt0ZXN0LHNwZWN9Lntqcyx0c31cIl0gfSxcbn0pO1xuIl0sCiAgIm1hcHBpbmdzIjogIjtBQUF3VSxTQUFTLGlCQUFpQjtBQUNsVyxPQUFPLGdCQUFnQjtBQUN2QixPQUFPLG1CQUFtQjtBQUMxQixPQUFPLFdBQVc7QUFDbEIsU0FBUyxvQkFBb0I7QUFFN0IsSUFBTyxzQkFBUSxhQUFhO0FBQUEsRUFDMUIsU0FBUztBQUFBLElBQ1AsVUFBVTtBQUFBLElBQ1YsTUFBTSxFQUFFLFVBQVUsU0FBUyxDQUFDO0FBQUEsSUFDNUIsV0FBVztBQUFBLE1BQ1QsV0FBVztBQUFBLFFBQ1QsY0FBYztBQUFBLFVBQ1osUUFBUTtBQUFBLFVBQ1IsV0FBVztBQUFBLFFBQ2IsQ0FBQztBQUFBLE1BQ0g7QUFBQSxNQUNBLEtBQUs7QUFBQSxNQUNMLFNBQVMsQ0FBQyxRQUFRO0FBQUEsTUFDbEIsTUFBTSxDQUFDLEtBQUs7QUFBQSxNQUNaLFFBQVEsQ0FBQyxxQkFBcUIsbUJBQW1CO0FBQUEsTUFDakQsU0FBUyxDQUFDLGdCQUFnQixrQkFBa0IsTUFBTTtBQUFBLElBQ3BELENBQUM7QUFBQSxFQUNIO0FBQUEsRUFDQSxNQUFNLEVBQUUsU0FBUyxDQUFDLDhCQUE4QixFQUFFO0FBQ3BELENBQUM7IiwKICAibmFtZXMiOiBbXQp9Cg==