admin/rdv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SHIT WORKS

Browse Source
This commit is contained in:
bootunloader
2026-01-05 22:05:48 +02:00
parent 714b8f737b
commit 0111933d92
3 changed files with 398 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
data_raw_getData_2026-01-03_8.json
View File

@@ -1,5 +0,0 @@
{
"error": "Error with your request, please try again (you will not be charged for this request).You should: 1) check that your URL is correctly encoded 2) try with render_js=True (5 credits per request) 3) try with premium_proxy=True see documentation: https://www.scrapingbee.com/documentation#premium_proxy (10-25 credits per request) 4) try with stealth_proxy=True see documentation: https://www.scrapingbee.com/documentation#stealth_proxy (75 credits per request)Do not hesitate to check our troubleshooting guide:https://www.scrapingbee.com/help",
"reason": "Server responded with 500",
"help": "('Received response with content-encoding: br, but failed to decode it.', error("brotli: decoder process called with data when 'can_accept_more_data()' is False"))"
}

62
src/lib/server/external/api.scraping.helpers.ts vendored
View File

@@ -6,8 +6,8 @@ import { baseDistributorId, constants } from "$lib/utils/constants";
import type { BookingEntry, Draw, LooseApiUser } from "$lib/utils/data.types";
import { rng } from "$lib/utils/rng";
import fs from "fs";
import { HttpProxyAgent } from "http-proxy-agent";
import fetch from "node-fetch";
import { tryCatch } from "../result";
// function dumpDistributors(distributors: LooseApiUser[]) {
// fs.writeFileSync("distributors.json", JSON.stringify(distributors, null, 2));
@@ -150,8 +150,10 @@ export async function getUsersBalance(userId: number, jwt: string) {
scrapingbeeUrl.searchParams.set("forward_headers", "true");
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
const encodedJwt = encodeURIComponent(jwt);
const res = await fetch(scrapingbeeUrl.toString(), {
headers: { "Spb-Authorization": jwt, ...forwardHeaders },
headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` },
});
const rj = (await res.json()) as {
@@ -161,6 +163,7 @@ export async function getUsersBalance(userId: number, jwt: string) {
data: { allowedBalance: number; balance: number };
time: string;
};
logger.debug(`[getUsersBalance] Response for ${userId}: ${JSON.stringify(rj)}`);
if (res.status !== 200 || rj.code !== 200 || !rj.success) {
logger.warn(`[getUsersBalance] Error getting balance for ${userId}: ${rj.message}`);
return false;
@@ -372,27 +375,31 @@ export const getData = async (
drawId: number,
chosenDate: string,
) => {
const scraperApiKey = env.SCRAPERAPI_API_KEY ?? "";
if (!jwt || jwt.length < 1) {
logger.warn(`[getData] JWT is invalid : ${jwt}`);
return { ok: false, message: "JWT is invalid", data: [] };
}
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/list2`;
logger.info(
`[getData] Fetching draw data from API for ${chosenDate} ${drawId} for ${userIds.length} users`,
);
const proxyUsername = "scraperapi.keep_headers=true";
const jwtUrlEncoded = encodeURIComponent(jwt);
const finalUrl = new URL("http://api.scrape.do/");
finalUrl.searchParams.append("url", targetUrl);
finalUrl.searchParams.append("token", scrapeDoApiKey);
finalUrl.searchParams.append("extraHeaders", "true");
// Configure HTTP proxy agent
const proxyAgent = new HttpProxyAgent(
`http://${proxyUsername}:${scraperApiKey}@proxy-server.scraperapi.com:8001`,
);
logger.debug(`[getData] Using proxy to fetch data from : ${finalUrl.toString()}`);
logger.debug(`[getData] Using proxy to fetch data from : ${targetUrl}`);
const res = await fetch(targetUrl, {
const res = await fetch(finalUrl.toString(), {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: jwt,
"sd-Content-Type": "application/json",
"sd-Authorization": jwt,
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
},
body: JSON.stringify({
userType: 3,
@@ -404,7 +411,6 @@ export const getData = async (
containImported: false,
keyword: "",
}),
agent: proxyAgent,
});
type J = {
@@ -414,21 +420,21 @@ export const getData = async (
data: { book: BookingEntry; user: any }[];
};
res.headers.forEach((value: string, key: string) => {
logger.debug(`[getData] response headers - ${key}: ${value}`);
});
if (res.headers.get("content-type") !== "application/json") {
logger.warn(`[getData] Error: Content type is not application/json`);
const rt = await res.text();
logger.debug(`[getData] Response: ${rt}`);
return {
ok: false,
message: `Content type is not application/json`,
data: [],
};
logger.info(`[getData] headers : ${JSON.stringify(res.headers)}`);
if (res.status !== 200 || !res.ok) {
logger.warn(`[getData] Error: ${JSON.stringify(res.status)}`);
const resTxt = await res.text();
logger.debug(`[getData] Response text: ${resTxt}`);
return { ok: false, message: `Status code ${res.status}`, data: [] };
}
let decoded = (await res.json()) as { data: J };
const result = await tryCatch(res.json());
if (result.error) {
logger.error(`[getData] Error parsing response : ${JSON.stringify(result.error)}`);
return { ok: false, message: result.error.message, data: [] };
}
let decoded = result.data as { data: J };
dumpDataRaw(decoded, `getData_${chosenDate}_${drawId}`);

117
src/lib/server/postdata/postdata.gen.controller.ts
View File

@@ -1,5 +1,7 @@
import { dbApiPostData } from "$lib/server/db/apipostdata.db";
import { dbApiUser } from "$lib/server/db/apiuser.db";
import { dbPresetData } from "$lib/server/db/presetdata.db";
import { getUsersBalance } from "$lib/server/external/api.scraping.helpers";
import { getReducedFinalSheet } from "$lib/server/finalsheet.helpers";
import {
adjustRatesIfDuplicatesFound,
@@ -13,16 +15,15 @@ import { getDefaultTotals, getULID } from "$lib/utils";
import type {
ApiPostUser,
ApiPostUserWithParent,
PresetDataEntry,
PostDataEntry,
PostDataFilters,
PostDataHistoryFilters,
PresetDataEntry,
ReducedFinalSheetData,
ReducedFinalSheetRow,
ServerError,
} from "$lib/utils/data.types";
import { dbPresetData } from "$lib/server/db/presetdata.db";
import { getUsersBalance } from "$lib/server/external/api.scraping.helpers";
import { logger } from "../logger";
function filterMatching(
data: ReducedFinalSheetRow[],
@@ -44,11 +45,11 @@ function filterMatching(
export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
const sessions = await getAllSessions();
console.log(sessions);
console.log(`Fetching balances`);
logger.debug(sessions);
logger.info(`Fetching balances`);
const balances = [] as { id: string; balance: number }[];
for (const user of users) {
console.log(`Finding ${user.id} in sessions`);
logger.info(`Finding ${user.id} in sessions`);
const session = sessions.find((e) => e.value.userId.includes(user.id));
const jwt = session?.value.sessionToken;
if (!jwt) {
@@ -58,7 +59,7 @@ export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
};
}
const uid = session.value.userId.toString();
console.log(`Found ${uid} in session`);
logger.info(`Found ${uid} in session`);
const out = await getUsersBalance(+uid.split(":")[1], jwt);
if (!out) {
return {
@@ -68,8 +69,8 @@ export async function updateBalanceOfPostUsers(users: ApiPostUserWithParent[]) {
}
balances.push({ id: uid, balance: out });
}
console.log(`Updating balances`);
console.log(balances);
logger.info(`Updating balances`);
logger.debug(balances);
await dbApiUser.updatePostUsersBalances(balances);
return {
ok: true,
@@ -105,9 +106,7 @@ export async function fetchDataForPosting(
detail: `Draw for the passed draw ID not found`,
data: [],
users: [],
errors: [
{ message: `Draw for the passed draw ID not found` },
] as ServerError,
errors: [{ message: `Draw for the passed draw ID not found` }] as ServerError,
};
}
const data = await getReducedFinalSheet(fsData);
@@ -123,9 +122,7 @@ export async function fetchDataForPosting(
console.log("[+] Filtering the fs data to get the numbers");
const filteredF = filterMatching(fsData.data, minPrize, maxPrize, "first");
console.log(
`Filtered data: ${filteredF.abNums.size}; ${filteredF.abcNums.size}`,
);
console.log(`Filtered data: ${filteredF.abNums.size}; ${filteredF.abcNums.size}`);
// ------------------------------------------
@@ -138,30 +135,19 @@ export async function fetchDataForPosting(
if (draw.filterDuplicatesWhilePosting === true) {
console.log(`[+] Removing numbers that have repeating digits`);
console.log(
`[=] Original : AB: ${abNums.length}, ABC: ${abcNums.length}`,
);
console.log(`[=] Original : AB: ${abNums.length}, ABC: ${abcNums.length}`);
abNums = removeNumbersWithRepeatingDigits(abNums);
abcNums = removeNumbersWithRepeatingDigits(abcNums);
}
console.log(`[=] AB: ${abNums.length}, ABC: ${abcNums.length}`);
console.log(`Fetching preset data`);
const presetData = await dbPresetData.getDataGroupedBySheetByDraw(
date,
+draw.id.split(":")[1],
);
const presetData = await dbPresetData.getDataGroupedBySheetByDraw(date, +draw.id.split(":")[1]);
console.log(`${presetData.all.length} preset entries found`);
for (let tries = 0; tries < 3; tries++) {
console.log(`[✍️] Try ${tries + 1} of generating the result`);
const out = await generatePostDataArrayFromBaseInfo(
input,
users,
abNums,
abcNums,
presetData,
);
const out = await generatePostDataArrayFromBaseInfo(input, users, abNums, abcNums, presetData);
if (out.ok) {
return out;
}
@@ -242,18 +228,12 @@ export async function generatePostDataArrayFromBaseInfo(
// ------------------------------------------
console.log(
`[+] Appending up to ${presetData.all.length} entries that are not ab, abc`,
);
console.log(`[+] Appending up to ${presetData.all.length} entries that are not ab, abc`);
for (const entry of presetData.all) {
if (
alreadyPresent.has(entry.number) ||
(entry.first < 5 && entry.second < 5)
) {
if (alreadyPresent.has(entry.number) || (entry.first < 5 && entry.second < 5)) {
continue;
}
const randomUserId =
users[Math.floor(Math.random() * users.length)].userId;
const randomUserId = users[Math.floor(Math.random() * users.length)].userId;
if (balanceCounts[randomUserId] === undefined) {
balanceCounts[randomUserId] = 0;
}
@@ -264,29 +244,21 @@ export async function generatePostDataArrayFromBaseInfo(
// ------------------------------------------
const usersTotalbalance = users.reduce((a, b) => a + (b.balance ?? 0), 0);
let totalAmtForPostingData = Object.values(balanceCounts).reduce(
(acc, curr) => acc + curr,
0,
);
let totalAmtForPostingData = Object.values(balanceCounts).reduce((acc, curr) => acc + curr, 0);
if (usersTotalbalance < totalAmtForPostingData) {
return {
ok: false,
detail: `Not enough balance to book overall with ${usersTotalbalance} < ${totalAmtForPostingData}`,
data: [],
users: [],
errors: [
{ message: `Not enough balance to book overall` },
] as ServerError,
errors: [{ message: `Not enough balance to book overall` }] as ServerError,
};
}
function isDistributionUnbalanced() {
let out = false;
for (const key in balanceCounts) {
if (
balanceCounts[key] >
(users.find((u) => u.userId === key)?.balance ?? 0)
) {
if (balanceCounts[key] > (users.find((u) => u.userId === key)?.balance ?? 0)) {
out = true;
break;
}
@@ -295,17 +267,16 @@ export async function generatePostDataArrayFromBaseInfo(
}
for (let tries = 0; tries < 5; tries++) {
console.log(
`Balance counts start : ${JSON.stringify(balanceCounts, null, 2)}`,
);
console.log(`Balance counts start : ${JSON.stringify(balanceCounts, null, 2)}`);
rebalancePostDataListByBalanceOfUsers(balanceCounts, users, result);
console.log(`Balance counts final : ${JSON.stringify(balanceCounts)}`);
let totalAmtForPostingDataAfterRebalance = Object.values(
balanceCounts,
).reduce((acc, curr) => acc + curr, 0);
let totalAmtForPostingDataAfterRebalance = Object.values(balanceCounts).reduce(
(acc, curr) => acc + curr,
0,
);
console.log(
`Total amount for posting data after rebalance: ${totalAmtForPostingDataAfterRebalance}`,
@@ -355,9 +326,7 @@ export function rebalancePostDataListByBalanceOfUsers(
users: ApiPostUser[],
result: PostDataEntry[],
) {
console.log(
`[+] Checking if the users have enough balance to book their assigned data`,
);
console.log(`[+] Checking if the users have enough balance to book their assigned data`);
for (const user of users) {
const usersBalance = user.balance ?? 0;
@@ -367,14 +336,10 @@ export function rebalancePostDataListByBalanceOfUsers(
continue;
}
if (usersBalance >= dueForUser) {
console.log(
`[✅] ${user.userName} can book the data of ${usersBalance} > ${dueForUser} `,
);
console.log(`[✅] ${user.userName} can book the data of ${usersBalance} > ${dueForUser} `);
continue;
}
console.log(
`[!❎!] ${user.userName} can't book it all ${usersBalance} < ${dueForUser}`,
);
console.log(`[!❎!] ${user.userName} can't book it all ${usersBalance} < ${dueForUser}`);
const difference = dueForUser - usersBalance;
let differenceLeft = Number(difference); // make a copy
@@ -386,17 +351,12 @@ export function rebalancePostDataListByBalanceOfUsers(
}
})
.map((r) => r.id);
console.log(
`Have to move ${entriesToMove.length} entries to other users`,
);
console.log(`Have to move ${entriesToMove.length} entries to other users`);
// find a user who has enough balance
const userWithEnoughBalance = users.find((u) => {
return (
(u.balance ?? 0) - balanceCounts[u.userId] >= difference &&
u.userId !== user.userId
);
return (u.balance ?? 0) - balanceCounts[u.userId] >= difference && u.userId !== user.userId;
});
if (!userWithEnoughBalance) {
return {
@@ -419,8 +379,7 @@ export function rebalancePostDataListByBalanceOfUsers(
continue;
}
entry.userId = userWithEnoughBalance.userId;
balanceCounts[userWithEnoughBalance.userId] +=
entry.first + entry.second;
balanceCounts[userWithEnoughBalance.userId] += entry.first + entry.second;
balanceCounts[user.userId] -= entry.first + entry.second;
amountMoved += entry.first + entry.second;
if (amountMoved >= difference) {
@@ -428,24 +387,18 @@ export function rebalancePostDataListByBalanceOfUsers(
break;
}
}
console.log(
`[+] Moved ${entriesToMove.length} entries to ${userWithEnoughBalance.userName}`,
);
console.log(`[+] Moved ${entriesToMove.length} entries to ${userWithEnoughBalance.userName}`);
}
}
export async function fetchPostDataHistory(input: PostDataHistoryFilters) {
const { draw, date } = input;
console.log(
`Fetching post data from HISTORY for draw: ${date} - ${draw.id}`,
);
console.log(`Fetching post data from HISTORY for draw: ${date} - ${draw.id}`);
const found = await dbApiPostData.getPostDataByDraw(date, draw.id);
if (!found) {
return { data: [], users: [], ok: false, detail: "Data not found" };
}
console.log(
`Data found for the passed draw: ${date} - ${draw.id}, returning that`,
);
console.log(`Data found for the passed draw: ${date} - ${draw.id}, returning that`);
const users = await dbApiUser.getAllPostUsers();
const uniqueUserIds = [] as string[];
for (const each of found) {