From 905405d0aefbd325a07dd2a8fd0db6c41370e3b6 Mon Sep 17 00:00:00 2001 From: bootunloader Date: Mon, 5 Jan 2026 23:58:49 +0200 Subject: [PATCH] removed scrapingbee cuse yuh --- .../server/external/api.scraping.helpers.ts | 172 ++++++++++-------- src/lib/server/postdata/post.handler.ts | 60 +++--- src/lib/trpc/routers/apiauth.router.ts | 24 +-- 3 files changed, 138 insertions(+), 118 deletions(-) diff --git a/src/lib/server/external/api.scraping.helpers.ts b/src/lib/server/external/api.scraping.helpers.ts index 003bd89..3b01f0b 100755 --- a/src/lib/server/external/api.scraping.helpers.ts +++ b/src/lib/server/external/api.scraping.helpers.ts @@ -30,23 +30,26 @@ function dumpDataRaw(data: any, prefix: string) { } export const testIfSessionIsValid = async (jwt: string) => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; - if (!scrapingbeeApiKey) { - logger.error("[testIfSessionIsValid] ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[testIfSessionIsValid] Scrape.do API key not configured"); return false; } try { const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); + const jwtUrlEncoded = encodeURIComponent(jwt); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); - const res = await fetch(scrapingbeeUrl.toString(), { - headers: { "Spb-Authorization": jwt }, + const res = await fetch(finalUrl.toString(), { + headers: { + "sd-Authorization": jwt, + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, + }, }); if (res.status !== 200 || !res.ok) { @@ -75,38 +78,32 @@ export const getSessionToken = async (payload: { code: string; userType: number; }): Promise<{ ok: boolean; message: string }> => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; - if (!scrapingbeeApiKey) { - logger.error("[getSessionToken] ScrapingBee API key not configured"); - return { ok: false, message: "ScrapingBee API key not configured" }; + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getSessionToken] Scrape.do API key not configured"); + return { ok: false, message: "Scrape.do API key not configured" }; } const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`; logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`); - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); - - // Prefix headers with Spb- for ScrapingBee to forward them - const forwardHeaders = Object.fromEntries( - Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]), - ); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); try { - const res = await fetch(scrapingbeeUrl.toString(), { + const res = await fetch(finalUrl.toString(), { method: "POST", body: JSON.stringify(payload), - headers: { "Spb-Content-Type": "application/json", ...forwardHeaders }, + headers: { "sd-Content-Type": "application/json" }, }); if (!res.ok) { const errorText = await res.text().catch(() => "Unknown error"); logger.error( - `[getSessionToken] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`, + `[getSessionToken] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`, ); return { ok: false, @@ -130,30 +127,32 @@ export const getSessionToken = async (payload: { }; export async function getUsersBalance(userId: number, jwt: string) { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; - if (!scrapingbeeApiKey) { - logger.error("[getUsersBalance] ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getUsersBalance] Scrape.do API key not configured"); return false; } - // Prefix headers with Spb- for ScrapingBee to forward them + // Prefix headers with sd- for Scrape.do to forward them const forwardHeaders = Object.fromEntries( - Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]), + Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]), ); try { const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); + const jwtUrlEncoded = encodeURIComponent(jwt); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); - const encodedJwt = encodeURIComponent(jwt); - - const res = await fetch(scrapingbeeUrl.toString(), { - headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` }, + const res = await fetch(finalUrl.toString(), { + headers: { + "sd-Authorization": jwt, + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, + ...forwardHeaders, + }, }); const rj = (await res.json()) as { @@ -176,12 +175,12 @@ export async function getUsersBalance(userId: number, jwt: string) { } export const getDealers = async (jwt: string, distributor_ids: string[]) => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; - if (!scrapingbeeApiKey) { - logger.error("[getDealers] ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getDealers] Scrape.do API key not configured"); return { dealers: [], - errors: [{ message: "ScrapingBee API key not configured" }], + errors: [{ message: "Scrape.do API key not configured" }], }; } @@ -194,23 +193,30 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => { message: string; }> = []; + // Prefix headers with sd- for Scrape.do to forward them + const forwardHeaders = Object.fromEntries( + Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]), + ); + // Process each batch sequentially for (const batch of batches) { const batchRequests = batch.map(async (did, index) => { await sleep(rng(100, 2000)); const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); + const jwtUrlEncoded = encodeURIComponent(jwt); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); - const res = await fetch(scrapingbeeUrl.toString(), { + const res = await fetch(finalUrl.toString(), { method: "POST", headers: { - "Spb-Authorization": jwt, - "Spb-Content-Type": "application/json", + "sd-Authorization": jwt, + "sd-Content-Type": "application/json", + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, + ...forwardHeaders, }, body: JSON.stringify({ page: 1, @@ -280,28 +286,35 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => { }; export const getDistributors = async (jwt: string) => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; - if (!scrapingbeeApiKey) { - logger.error("[getDistributors] ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getDistributors] Scrape.do API key not configured"); return { ok: false, - message: "ScrapingBee API key not configured", + message: "Scrape.do API key not configured", data: [], }; } const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); + const jwtUrlEncoded = encodeURIComponent(jwt); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); - const res = await fetch(scrapingbeeUrl.toString(), { + // Prefix headers with sd- for Scrape.do to forward them + const forwardHeaders = Object.fromEntries( + Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]), + ); + + const res = await fetch(finalUrl.toString(), { method: "POST", headers: { - "Spb-Authorization": jwt, - "Spb-Content-Type": "application/json", + "sd-Authorization": jwt, + "sd-Content-Type": "application/json", + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, + ...forwardHeaders, }, body: JSON.stringify({ page: 1, @@ -333,19 +346,32 @@ export const getDistributors = async (jwt: string) => { }; export const getDraws = async (jwt: string) => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getDraws] Scrape.do API key not configured"); + return { ok: false, message: "Scrape.do API key not configured", data: [] }; + } logger.info(`[getDraws] Fetching draws from the API`); const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - scrapingbeeUrl.searchParams.set("forward_headers_pure", "true"); + const jwtUrlEncoded = encodeURIComponent(jwt); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); - const res = await fetch(scrapingbeeUrl.toString(), { - headers: { "Spb-Authorization": jwt }, + // Prefix headers with sd- for Scrape.do to forward them + const forwardHeaders = Object.fromEntries( + Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]), + ); + + const res = await fetch(finalUrl.toString(), { + headers: { + "sd-Authorization": jwt, + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, + ...forwardHeaders, + }, }); type J = { diff --git a/src/lib/server/postdata/post.handler.ts b/src/lib/server/postdata/post.handler.ts index d98579c..eeb7984 100644 --- a/src/lib/server/postdata/post.handler.ts +++ b/src/lib/server/postdata/post.handler.ts @@ -205,31 +205,27 @@ async function sendBatchRequest( changedBalance: number, body: string, ) { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY; - if (!scrapingbeeApiKey) { - logger.error("[sendBatchRequest] ScrapingBee API key not configured"); - throw new Error("ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[sendBatchRequest] Scrape.do API key not configured"); + throw new Error("Scrape.do API key not configured"); } const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - - // Prefix headers with Spb- for ScrapingBee to forward them - const forwardHeaders = Object.fromEntries( - Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]), - ); + const jwtUrlEncoded = encodeURIComponent(session.sessionToken); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`); - return fetch(scrapingbeeUrl.toString(), { + return fetch(finalUrl.toString(), { method: "POST", headers: { - "Spb-Authorization": session.sessionToken, - "Spb-Content-Type": "application/json", - ...forwardHeaders, + "sd-Authorization": session.sessionToken, + "sd-Content-Type": "application/json", + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, }, body: JSON.stringify({ dealerId, @@ -310,31 +306,27 @@ async function deleteAllBookedEntries({ drawId: number; closeTime: string; }) { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY; - if (!scrapingbeeApiKey) { - logger.error("[deleteAllBookedEntries] ScrapingBee API key not configured"); - throw new Error("ScrapingBee API key not configured"); + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[deleteAllBookedEntries] Scrape.do API key not configured"); + throw new Error("Scrape.do API key not configured"); } const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`; - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("forward_headers", "true"); - - // Prefix headers with Spb- for ScrapingBee to forward them - const forwardHeaders = Object.fromEntries( - Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]), - ); + const jwtUrlEncoded = encodeURIComponent(session.sessionToken); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); + finalUrl.searchParams.append("extraHeaders", "true"); logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`); - return fetch(scrapingbeeUrl.toString(), { + return fetch(finalUrl.toString(), { method: "POST", headers: { - "Spb-Authorization": session.sessionToken, - "Spb-Content-Type": "application/json", - ...forwardHeaders, + "sd-Authorization": session.sessionToken, + "sd-Content-Type": "application/json", + "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`, }, body: JSON.stringify({ bookIds: data.map((e) => e.bookId), diff --git a/src/lib/trpc/routers/apiauth.router.ts b/src/lib/trpc/routers/apiauth.router.ts index 294814f..703eaca 100755 --- a/src/lib/trpc/routers/apiauth.router.ts +++ b/src/lib/trpc/routers/apiauth.router.ts @@ -17,7 +17,14 @@ import { createTRPCRouter, protectedProcedure } from "../t"; export const apiAuthRouter = createTRPCRouter({ getCaptcha: protectedProcedure.mutation(async () => { - const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? ""; + const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? ""; + if (!scrapeDoApiKey) { + logger.error("[getCaptcha] Scrape.do API key not configured"); + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: "Scrape.do API key not configured", + }); + } try { const uuid = getUUID(); @@ -25,22 +32,17 @@ export const apiAuthRouter = createTRPCRouter({ logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`); - // Build ScrapingBee API URL with params - const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1"); - scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey); - scrapingbeeUrl.searchParams.set("url", targetUrl); - scrapingbeeUrl.searchParams.set("render_js", "false"); - scrapingbeeUrl.searchParams.set("block_resources", "false"); + const finalUrl = new URL("http://api.scrape.do/"); + finalUrl.searchParams.append("url", targetUrl); + finalUrl.searchParams.append("token", scrapeDoApiKey); - const res = await fetch(scrapingbeeUrl.toString()); + const res = await fetch(finalUrl.toString()); if (!res.ok || res.status !== 200) { // Clone response before reading to avoid consuming body const clonedRes = res.clone(); const errorText = await clonedRes.text().catch(() => "Unknown error"); - logger.error( - `[getCaptcha] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`, - ); + logger.error(`[getCaptcha] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`); throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", message: `Failed to fetch captcha image: ${res.status}`,