admin/rdv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed scrapingbee cuse yuh

Browse Source
This commit is contained in:
bootunloader
2026-01-05 23:58:49 +02:00
parent 653fb266e5
commit 905405d0ae
3 changed files with 138 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
src/lib/server/external/api.scraping.helpers.ts vendored
View File

@@ -30,23 +30,26 @@ function dumpDataRaw(data: any, prefix: string) {
} }
export const testIfSessionIsValid = async (jwt: string) => { export const testIfSessionIsValid = async (jwt: string) => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[testIfSessionIsValid] ScrapingBee API key not configured"); logger.error("[testIfSessionIsValid] Scrape.do API key not configured");
return false; return false;
} }
try { try {
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`; const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(jwt);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); finalUrl.searchParams.append("extraHeaders", "true");
const res = await fetch(scrapingbeeUrl.toString(), { const res = await fetch(finalUrl.toString(), {
headers: { "Spb-Authorization": jwt }, headers: {
"sd-Authorization": jwt,
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
},
}); });
if (res.status !== 200 || !res.ok) { if (res.status !== 200 || !res.ok) {
@@ -75,38 +78,32 @@ export const getSessionToken = async (payload: {
code: string; code: string;
userType: number; userType: number;
}): Promise<{ ok: boolean; message: string }> => { }): Promise<{ ok: boolean; message: string }> => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[getSessionToken] ScrapingBee API key not configured"); logger.error("[getSessionToken] Scrape.do API key not configured");
return { ok: false, message: "ScrapingBee API key not configured" }; return { ok: false, message: "Scrape.do API key not configured" };
} }
const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`; const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`;
logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`); logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`);
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("extraHeaders", "true");
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
// Prefix headers with Spb- for ScrapingBee to forward them
const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
);
try { try {
const res = await fetch(scrapingbeeUrl.toString(), { const res = await fetch(finalUrl.toString(), {
method: "POST", method: "POST",
body: JSON.stringify(payload), body: JSON.stringify(payload),
headers: { "Spb-Content-Type": "application/json", ...forwardHeaders }, headers: { "sd-Content-Type": "application/json" },
}); });
if (!res.ok) { if (!res.ok) {
const errorText = await res.text().catch(() => "Unknown error"); const errorText = await res.text().catch(() => "Unknown error");
logger.error( logger.error(
`[getSessionToken] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`, `[getSessionToken] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`,
); );
return { return {
ok: false, ok: false,
@@ -130,30 +127,32 @@ export const getSessionToken = async (payload: {
}; };
export async function getUsersBalance(userId: number, jwt: string) { export async function getUsersBalance(userId: number, jwt: string) {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[getUsersBalance] ScrapingBee API key not configured"); logger.error("[getUsersBalance] Scrape.do API key not configured");
return false; return false;
} }
// Prefix headers with Spb- for ScrapingBee to forward them // Prefix headers with sd- for Scrape.do to forward them
const forwardHeaders = Object.fromEntries( const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]), Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
); );
try { try {
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`; const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(jwt);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); finalUrl.searchParams.append("extraHeaders", "true");
const encodedJwt = encodeURIComponent(jwt); const res = await fetch(finalUrl.toString(), {
headers: {
const res = await fetch(scrapingbeeUrl.toString(), { "sd-Authorization": jwt,
headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` }, "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
...forwardHeaders,
},
}); });
const rj = (await res.json()) as { const rj = (await res.json()) as {
@@ -176,12 +175,12 @@ export async function getUsersBalance(userId: number, jwt: string) {
} }
export const getDealers = async (jwt: string, distributor_ids: string[]) => { export const getDealers = async (jwt: string, distributor_ids: string[]) => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[getDealers] ScrapingBee API key not configured"); logger.error("[getDealers] Scrape.do API key not configured");
return { return {
dealers: [], dealers: [],
errors: [{ message: "ScrapingBee API key not configured" }], errors: [{ message: "Scrape.do API key not configured" }],
}; };
} }
@@ -194,23 +193,30 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
message: string; message: string;
}> = []; }> = [];
// Prefix headers with sd- for Scrape.do to forward them
const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
);
// Process each batch sequentially // Process each batch sequentially
for (const batch of batches) { for (const batch of batches) {
const batchRequests = batch.map(async (did, index) => { const batchRequests = batch.map(async (did, index) => {
await sleep(rng(100, 2000)); await sleep(rng(100, 2000));
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`; const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(jwt);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); finalUrl.searchParams.append("extraHeaders", "true");
const res = await fetch(scrapingbeeUrl.toString(), { const res = await fetch(finalUrl.toString(), {
method: "POST", method: "POST",
headers: { headers: {
"Spb-Authorization": jwt, "sd-Authorization": jwt,
"Spb-Content-Type": "application/json", "sd-Content-Type": "application/json",
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
...forwardHeaders,
}, },
body: JSON.stringify({ body: JSON.stringify({
page: 1, page: 1,
@@ -280,28 +286,35 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
}; };
export const getDistributors = async (jwt: string) => { export const getDistributors = async (jwt: string) => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[getDistributors] ScrapingBee API key not configured"); logger.error("[getDistributors] Scrape.do API key not configured");
return { return {
ok: false, ok: false,
message: "ScrapingBee API key not configured", message: "Scrape.do API key not configured",
data: [], data: [],
}; };
} }
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`; const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(jwt);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); finalUrl.searchParams.append("extraHeaders", "true");
const res = await fetch(scrapingbeeUrl.toString(), { // Prefix headers with sd- for Scrape.do to forward them
const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
);
const res = await fetch(finalUrl.toString(), {
method: "POST", method: "POST",
headers: { headers: {
"Spb-Authorization": jwt, "sd-Authorization": jwt,
"Spb-Content-Type": "application/json", "sd-Content-Type": "application/json",
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
...forwardHeaders,
}, },
body: JSON.stringify({ body: JSON.stringify({
page: 1, page: 1,
@@ -333,19 +346,32 @@ export const getDistributors = async (jwt: string) => {
}; };
export const getDraws = async (jwt: string) => { export const getDraws = async (jwt: string) => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapeDoApiKey) {
logger.error("[getDraws] Scrape.do API key not configured");
return { ok: false, message: "Scrape.do API key not configured", data: [] };
}
logger.info(`[getDraws] Fetching draws from the API`); logger.info(`[getDraws] Fetching draws from the API`);
const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`; const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(jwt);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); finalUrl.searchParams.append("extraHeaders", "true");
const res = await fetch(scrapingbeeUrl.toString(), { // Prefix headers with sd- for Scrape.do to forward them
headers: { "Spb-Authorization": jwt }, const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
);
const res = await fetch(finalUrl.toString(), {
headers: {
"sd-Authorization": jwt,
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
...forwardHeaders,
},
}); });
type J = { type J = {

60
src/lib/server/postdata/post.handler.ts
View File

@@ -205,31 +205,27 @@ async function sendBatchRequest(
changedBalance: number, changedBalance: number,
body: string, body: string,
) { ) {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[sendBatchRequest] ScrapingBee API key not configured"); logger.error("[sendBatchRequest] Scrape.do API key not configured");
throw new Error("ScrapingBee API key not configured"); throw new Error("Scrape.do API key not configured");
} }
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`; const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
finalUrl.searchParams.append("extraHeaders", "true");
// Prefix headers with Spb- for ScrapingBee to forward them
const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
);
logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`); logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`);
return fetch(scrapingbeeUrl.toString(), { return fetch(finalUrl.toString(), {
method: "POST", method: "POST",
headers: { headers: {
"Spb-Authorization": session.sessionToken, "sd-Authorization": session.sessionToken,
"Spb-Content-Type": "application/json", "sd-Content-Type": "application/json",
...forwardHeaders, "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
}, },
body: JSON.stringify({ body: JSON.stringify({
dealerId, dealerId,
@@ -310,31 +306,27 @@ async function deleteAllBookedEntries({
drawId: number; drawId: number;
closeTime: string; closeTime: string;
}) { }) {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapingbeeApiKey) { if (!scrapeDoApiKey) {
logger.error("[deleteAllBookedEntries] ScrapingBee API key not configured"); logger.error("[deleteAllBookedEntries] Scrape.do API key not configured");
throw new Error("ScrapingBee API key not configured"); throw new Error("Scrape.do API key not configured");
} }
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`; const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`;
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); const finalUrl = new URL("http://api.scrape.do/");
scrapingbeeUrl.searchParams.set("url", targetUrl); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("forward_headers", "true"); finalUrl.searchParams.append("token", scrapeDoApiKey);
finalUrl.searchParams.append("extraHeaders", "true");
// Prefix headers with Spb- for ScrapingBee to forward them
const forwardHeaders = Object.fromEntries(
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
);
logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`); logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`);
return fetch(scrapingbeeUrl.toString(), { return fetch(finalUrl.toString(), {
method: "POST", method: "POST",
headers: { headers: {
"Spb-Authorization": session.sessionToken, "sd-Authorization": session.sessionToken,
"Spb-Content-Type": "application/json", "sd-Content-Type": "application/json",
...forwardHeaders, "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
}, },
body: JSON.stringify({ body: JSON.stringify({
bookIds: data.map((e) => e.bookId), bookIds: data.map((e) => e.bookId),

24
src/lib/trpc/routers/apiauth.router.ts
View File

@@ -17,7 +17,14 @@ import { createTRPCRouter, protectedProcedure } from "../t";
export const apiAuthRouter = createTRPCRouter({ export const apiAuthRouter = createTRPCRouter({
getCaptcha: protectedProcedure.mutation(async () => { getCaptcha: protectedProcedure.mutation(async () => {
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
if (!scrapeDoApiKey) {
logger.error("[getCaptcha] Scrape.do API key not configured");
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Scrape.do API key not configured",
});
}
try { try {
const uuid = getUUID(); const uuid = getUUID();
@@ -25,22 +32,17 @@ export const apiAuthRouter = createTRPCRouter({
logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`); logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`);
// Build ScrapingBee API URL with params const finalUrl = new URL("http://api.scrape.do/");
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); finalUrl.searchParams.append("url", targetUrl);
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); finalUrl.searchParams.append("token", scrapeDoApiKey);
scrapingbeeUrl.searchParams.set("url", targetUrl);
scrapingbeeUrl.searchParams.set("render_js", "false");
scrapingbeeUrl.searchParams.set("block_resources", "false");
const res = await fetch(scrapingbeeUrl.toString()); const res = await fetch(finalUrl.toString());
if (!res.ok || res.status !== 200) { if (!res.ok || res.status !== 200) {
// Clone response before reading to avoid consuming body // Clone response before reading to avoid consuming body
const clonedRes = res.clone(); const clonedRes = res.clone();
const errorText = await clonedRes.text().catch(() => "Unknown error"); const errorText = await clonedRes.text().catch(() => "Unknown error");
logger.error( logger.error(`[getCaptcha] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`);
`[getCaptcha] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`,
);
throw new TRPCError({ throw new TRPCError({
code: "INTERNAL_SERVER_ERROR", code: "INTERNAL_SERVER_ERROR",
message: `Failed to fetch captcha image: ${res.status}`, message: `Failed to fetch captcha image: ${res.status}`,