SHIT WORKS

data_raw_getData_2026-01-03_8.json

@@ -1,5 +0,0 @@
-{
-  "error": "Error with your request, please try again (you will not be charged for this request).You should: 1) check that your URL is correctly encoded 2) try with render_js=True (5 credits per request) 3) try with premium_proxy=True see documentation: https://www.scrapingbee.com/documentation#premium_proxy (10-25 credits per request) 4) try with stealth_proxy=True see documentation: https://www.scrapingbee.com/documentation#stealth_proxy (75 credits per request)Do not hesitate to check our troubleshooting guide:https://www.scrapingbee.com/help",
-  "reason": "Server responded with 500",
-  "help": "('Received response with content-encoding: br, but failed to decode it.', error("brotli: decoder process called with data when 'can_accept_more_data()' is False"))"
-}

src/lib/server/external/api.scraping.helpers.ts

@@ -6,8 +6,8 @@ import { baseDistributorId, constants } from "$lib/utils/constants";
 import type { BookingEntry, Draw, LooseApiUser } from "$lib/utils/data.types";
 import { rng } from "$lib/utils/rng";
 import fs from "fs";
-import { HttpProxyAgent } from "http-proxy-agent";
 import fetch from "node-fetch";
+import { tryCatch } from "../result";
 
 // function dumpDistributors(distributors: LooseApiUser[]) {
 //   fs.writeFileSync("distributors.json", JSON.stringify(distributors, null, 2));
@@ -150,8 +150,10 @@ export async function getUsersBalance(userId: number, jwt: string) {
     scrapingbeeUrl.searchParams.set("forward_headers", "true");
     scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
 
+    const encodedJwt = encodeURIComponent(jwt);
+
     const res = await fetch(scrapingbeeUrl.toString(), {
-      headers: { "Spb-Authorization": jwt, ...forwardHeaders },
+      headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` },
     });
 
     const rj = (await res.json()) as {
@@ -161,6 +163,7 @@ export async function getUsersBalance(userId: number, jwt: string) {
       data: { allowedBalance: number; balance: number };
       time: string;
     };
+    logger.debug(`[getUsersBalance] Response for ${userId}: ${JSON.stringify(rj)}`);
     if (res.status !== 200 || rj.code !== 200 || !rj.success) {
       logger.warn(`[getUsersBalance] Error getting balance for ${userId}: ${rj.message}`);
       return false;
@@ -372,27 +375,31 @@ export const getData = async (
   drawId: number,
   chosenDate: string,
 ) => {
-  const scraperApiKey = env.SCRAPERAPI_API_KEY ?? "";
+  if (!jwt || jwt.length < 1) {
+    logger.warn(`[getData] JWT is invalid : ${jwt}`);
+    return { ok: false, message: "JWT is invalid", data: [] };
+  }
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
   const targetUrl = `${constants.SCRAP_API_URL}/v1/book/list2`;
 
   logger.info(
     `[getData] Fetching draw data from API for ${chosenDate} ${drawId} for ${userIds.length} users`,
   );
 
-  const proxyUsername = "scraperapi.keep_headers=true";
+  const jwtUrlEncoded = encodeURIComponent(jwt);
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");
 
-  // Configure HTTP proxy agent
+  logger.debug(`[getData] Using proxy to fetch data from : ${finalUrl.toString()}`);
-  const proxyAgent = new HttpProxyAgent(
-    `http://${proxyUsername}:${scraperApiKey}@proxy-server.scraperapi.com:8001`,
-  );
 
-  logger.debug(`[getData] Using proxy to fetch data from : ${targetUrl}`);
+  const res = await fetch(finalUrl.toString(), {
-
-  const res = await fetch(targetUrl, {
     method: "POST",
     headers: {
-      "Content-Type": "application/json",
+      "sd-Content-Type": "application/json",
-      Authorization: jwt,
+      "sd-Authorization": jwt,
+      "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
     },
     body: JSON.stringify({
       userType: 3,
@@ -404,7 +411,6 @@ export const getData = async (
       containImported: false,
       keyword: "",
     }),
-    agent: proxyAgent,
   });
 
   type J = {
@@ -414,21 +420,21 @@ export const getData = async (
     data: { book: BookingEntry; user: any }[];
   };
 
-  res.headers.forEach((value: string, key: string) => {
+  logger.info(`[getData] headers : ${JSON.stringify(res.headers)}`);
-    logger.debug(`[getData] response headers - ${key}: ${value}`);
+  if (res.status !== 200 || !res.ok) {
-  });
+    logger.warn(`[getData] Error: ${JSON.stringify(res.status)}`);
-
+    const resTxt = await res.text();
-  if (res.headers.get("content-type") !== "application/json") {
+    logger.debug(`[getData] Response text: ${resTxt}`);
-    logger.warn(`[getData] Error: Content type is not application/json`);
+    return { ok: false, message: `Status code ${res.status}`, data: [] };
-    const rt = await res.text();
-    logger.debug(`[getData] Response: ${rt}`);
-    return {
-      ok: false,
-      message: `Content type is not application/json`,
-      data: [],
-    };
   }
-  let decoded = (await res.json()) as { data: J };
+
+  const result = await tryCatch(res.json());
+  if (result.error) {
+    logger.error(`[getData] Error parsing response : ${JSON.stringify(result.error)}`);
+    return { ok: false, message: result.error.message, data: [] };
+  }
+
+  let decoded = result.data as { data: J };
 
   dumpDataRaw(decoded, `getData_${chosenDate}_${drawId}`);
 

src/lib/server/postdata/postdata.gen.controller.ts

@@ -1,5 +1,7 @@
 import { dbApiPostData } from "$lib/server/db/apipostdata.db";
 import { dbApiUser } from "$lib/server/db/apiuser.db";
+import { dbPresetData } from "$lib/server/db/presetdata.db";
+import { getUsersBalance } from "$lib/server/external/api.scraping.helpers";
 import { getReducedFinalSheet } from "$lib/server/finalsheet.helpers";
 import {
   adjustRatesIfDuplicatesFound,
@@ -13,16 +15,15 @@ import { getDefaultTotals, getULID } from "$lib/utils";
 import type {
   ApiPostUser,
   ApiPostUserWithParent,
-    PresetDataEntry,
   PostDataEntry,
   PostDataFilters,
   PostDataHistoryFilters,
+  PresetDataEntry,
   ReducedFinalSheetData,
   ReducedFinalSheetRow,
   ServerError,
 } from "$lib/utils/data.types";
-import { dbPresetData } from "$lib/server/db/presetdata.db";
+import { logger } from "../logger";
-import { getUsersBalance } from "$lib/server/external/api.scraping.helpers";
 
 function filterMatching(
   data: ReducedFinalSheetRow[],
@@ -44,11 +45,11 @@ function filterMatching(
 
 export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
   const sessions = await getAllSessions();
-    console.log(sessions);
+  logger.debug(sessions);
-    console.log(`Fetching balances`);
+  logger.info(`Fetching balances`);
   const balances = [] as { id: string; balance: number }[];
   for (const user of users) {
-        console.log(`Finding ${user.id} in sessions`);
+    logger.info(`Finding ${user.id} in sessions`);
     const session = sessions.find((e) => e.value.userId.includes(user.id));
     const jwt = session?.value.sessionToken;
     if (!jwt) {
@@ -58,7 +59,7 @@ export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
       };
     }
     const uid = session.value.userId.toString();
-        console.log(`Found ${uid} in session`);
+    logger.info(`Found ${uid} in session`);
     const out = await getUsersBalance(+uid.split(":")[1], jwt);
     if (!out) {
       return {
@@ -68,8 +69,8 @@ export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
     }
     balances.push({ id: uid, balance: out });
   }
-    console.log(`Updating balances`);
+  logger.info(`Updating balances`);
-    console.log(balances);
+  logger.debug(balances);
   await dbApiUser.updatePostUsersBalances(balances);
   return {
     ok: true,
@@ -105,9 +106,7 @@ export async function fetchDataForPosting(
       detail: `Draw for the passed draw ID not found`,
       data: [],
       users: [],
-            errors: [
+      errors: [{ message: `Draw for the passed draw ID not found` }] as ServerError,
-                { message: `Draw for the passed draw ID not found` },
-            ] as ServerError,
     };
   }
   const data = await getReducedFinalSheet(fsData);
@@ -123,9 +122,7 @@ export async function fetchDataForPosting(
 
   console.log("[+] Filtering the fs data to get the numbers");
   const filteredF = filterMatching(fsData.data, minPrize, maxPrize, "first");
-    console.log(
+  console.log(`Filtered data: ${filteredF.abNums.size}; ${filteredF.abcNums.size}`);
-        `Filtered data: ${filteredF.abNums.size}; ${filteredF.abcNums.size}`,
-    );
 
   // ------------------------------------------
 
@@ -138,30 +135,19 @@ export async function fetchDataForPosting(
 
   if (draw.filterDuplicatesWhilePosting === true) {
     console.log(`[+] Removing numbers that have repeating digits`);
-        console.log(
+    console.log(`[=] Original : AB: ${abNums.length}, ABC: ${abcNums.length}`);
-            `[=] Original : AB: ${abNums.length}, ABC: ${abcNums.length}`,
-        );
     abNums = removeNumbersWithRepeatingDigits(abNums);
     abcNums = removeNumbersWithRepeatingDigits(abcNums);
   }
   console.log(`[=] AB: ${abNums.length}, ABC: ${abcNums.length}`);
 
   console.log(`Fetching preset data`);
-    const presetData = await dbPresetData.getDataGroupedBySheetByDraw(
+  const presetData = await dbPresetData.getDataGroupedBySheetByDraw(date, +draw.id.split(":")[1]);
-        date,
-        +draw.id.split(":")[1],
-    );
   console.log(`${presetData.all.length} preset entries found`);
 
   for (let tries = 0; tries < 3; tries++) {
     console.log(`[✍️] Try ${tries + 1} of generating the result`);
-        const out = await generatePostDataArrayFromBaseInfo(
+    const out = await generatePostDataArrayFromBaseInfo(input, users, abNums, abcNums, presetData);
-            input,
-            users,
-            abNums,
-            abcNums,
-            presetData,
-        );
     if (out.ok) {
       return out;
     }
@@ -242,18 +228,12 @@ export async function generatePostDataArrayFromBaseInfo(
 
   // ------------------------------------------
 
-    console.log(
+  console.log(`[+] Appending up to ${presetData.all.length} entries that are not ab, abc`);
-        `[+] Appending up to ${presetData.all.length} entries that are not ab, abc`,
-    );
   for (const entry of presetData.all) {
-        if (
+    if (alreadyPresent.has(entry.number) || (entry.first < 5 && entry.second < 5)) {
-            alreadyPresent.has(entry.number) ||
-            (entry.first < 5 && entry.second < 5)
-        ) {
       continue;
     }
-        const randomUserId =
+    const randomUserId = users[Math.floor(Math.random() * users.length)].userId;
-            users[Math.floor(Math.random() * users.length)].userId;
     if (balanceCounts[randomUserId] === undefined) {
       balanceCounts[randomUserId] = 0;
     }
@@ -264,29 +244,21 @@ export async function generatePostDataArrayFromBaseInfo(
   // ------------------------------------------
 
   const usersTotalbalance = users.reduce((a, b) => a + (b.balance ?? 0), 0);
-    let totalAmtForPostingData = Object.values(balanceCounts).reduce(
+  let totalAmtForPostingData = Object.values(balanceCounts).reduce((acc, curr) => acc + curr, 0);
-        (acc, curr) => acc + curr,
-        0,
-    );
   if (usersTotalbalance < totalAmtForPostingData) {
     return {
       ok: false,
       detail: `Not enough balance to book overall with ${usersTotalbalance} < ${totalAmtForPostingData}`,
       data: [],
       users: [],
-            errors: [
+      errors: [{ message: `Not enough balance to book overall` }] as ServerError,
-                { message: `Not enough balance to book overall` },
-            ] as ServerError,
     };
   }
 
   function isDistributionUnbalanced() {
     let out = false;
     for (const key in balanceCounts) {
-            if (
+      if (balanceCounts[key] > (users.find((u) => u.userId === key)?.balance ?? 0)) {
-                balanceCounts[key] >
-                (users.find((u) => u.userId === key)?.balance ?? 0)
-            ) {
         out = true;
         break;
       }
@@ -295,17 +267,16 @@ export async function generatePostDataArrayFromBaseInfo(
   }
 
   for (let tries = 0; tries < 5; tries++) {
-        console.log(
+    console.log(`Balance counts start : ${JSON.stringify(balanceCounts, null, 2)}`);
-            `Balance counts start : ${JSON.stringify(balanceCounts, null, 2)}`,
-        );
 
     rebalancePostDataListByBalanceOfUsers(balanceCounts, users, result);
 
     console.log(`Balance counts final : ${JSON.stringify(balanceCounts)}`);
 
-        let totalAmtForPostingDataAfterRebalance = Object.values(
+    let totalAmtForPostingDataAfterRebalance = Object.values(balanceCounts).reduce(
-            balanceCounts,
+      (acc, curr) => acc + curr,
-        ).reduce((acc, curr) => acc + curr, 0);
+      0,
+    );
 
     console.log(
       `Total amount for posting data after rebalance: ${totalAmtForPostingDataAfterRebalance}`,
@@ -355,9 +326,7 @@ export function rebalancePostDataListByBalanceOfUsers(
   users: ApiPostUser[],
   result: PostDataEntry[],
 ) {
-    console.log(
+  console.log(`[+] Checking if the users have enough balance to book their assigned data`);
-        `[+] Checking if the users have enough balance to book their assigned data`,
-    );
 
   for (const user of users) {
     const usersBalance = user.balance ?? 0;
@@ -367,14 +336,10 @@ export function rebalancePostDataListByBalanceOfUsers(
       continue;
     }
     if (usersBalance >= dueForUser) {
-            console.log(
+      console.log(`[✅] ${user.userName} can book the data of ${usersBalance} > ${dueForUser} `);
-                `[✅] ${user.userName} can book the data of ${usersBalance} > ${dueForUser} `,
-            );
       continue;
     }
-        console.log(
+    console.log(`[!❎!] ${user.userName} can't book it all ${usersBalance} < ${dueForUser}`);
-            `[!❎!] ${user.userName} can't book it all ${usersBalance} < ${dueForUser}`,
-        );
 
     const difference = dueForUser - usersBalance;
     let differenceLeft = Number(difference); // make a copy
@@ -386,17 +351,12 @@ export function rebalancePostDataListByBalanceOfUsers(
         }
       })
       .map((r) => r.id);
-        console.log(
+    console.log(`Have to move ${entriesToMove.length} entries to other users`);
-            `Have to move ${entriesToMove.length} entries to other users`,
-        );
 
     // find a user who has enough balance
 
     const userWithEnoughBalance = users.find((u) => {
-            return (
+      return (u.balance ?? 0) - balanceCounts[u.userId] >= difference && u.userId !== user.userId;
-                (u.balance ?? 0) - balanceCounts[u.userId] >= difference &&
-                u.userId !== user.userId
-            );
     });
     if (!userWithEnoughBalance) {
       return {
@@ -419,8 +379,7 @@ export function rebalancePostDataListByBalanceOfUsers(
         continue;
       }
       entry.userId = userWithEnoughBalance.userId;
-            balanceCounts[userWithEnoughBalance.userId] +=
+      balanceCounts[userWithEnoughBalance.userId] += entry.first + entry.second;
-                entry.first + entry.second;
       balanceCounts[user.userId] -= entry.first + entry.second;
       amountMoved += entry.first + entry.second;
       if (amountMoved >= difference) {
@@ -428,24 +387,18 @@ export function rebalancePostDataListByBalanceOfUsers(
         break;
       }
     }
-        console.log(
+    console.log(`[+] Moved ${entriesToMove.length} entries to ${userWithEnoughBalance.userName}`);
-            `[+] Moved ${entriesToMove.length} entries to ${userWithEnoughBalance.userName}`,
-        );
   }
 }
 
 export async function fetchPostDataHistory(input: PostDataHistoryFilters) {
   const { draw, date } = input;
-    console.log(
+  console.log(`Fetching post data from HISTORY for draw: ${date} - ${draw.id}`);
-        `Fetching post data from HISTORY for draw: ${date} - ${draw.id}`,
-    );
   const found = await dbApiPostData.getPostDataByDraw(date, draw.id);
   if (!found) {
     return { data: [], users: [], ok: false, detail: "Data not found" };
   }
-    console.log(
+  console.log(`Data found for the passed draw: ${date} - ${draw.id}, returning that`);
-        `Data found for the passed draw: ${date} - ${draw.id}, returning that`,
-    );
   const users = await dbApiUser.getAllPostUsers();
   const uniqueUserIds = [] as string[];
   for (const each of found) {