removed scrapingbee cuse yuh

2026-01-05 23:58:49 +02:00
parent 653fb266e5
commit 905405d0ae
3 changed files with 138 additions and 118 deletions
--- a/src/lib/server/external/api.scraping.helpers.ts
+++ b/src/lib/server/external/api.scraping.helpers.ts
@@ -30,23 +30,26 @@ function dumpDataRaw(data: any, prefix: string) {
 }

 export const testIfSessionIsValid = async (jwt: string) => {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
-  if (!scrapingbeeApiKey) {
-    logger.error("[testIfSessionIsValid] ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[testIfSessionIsValid] Scrape.do API key not configured");
    return false;
  }

  try {
    const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`;

-    const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-    scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-    scrapingbeeUrl.searchParams.set("url", targetUrl);
-    scrapingbeeUrl.searchParams.set("forward_headers", "true");
-    scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
+    const jwtUrlEncoded = encodeURIComponent(jwt);
+    const finalUrl = new URL("http://api.scrape.do/");
+    finalUrl.searchParams.append("url", targetUrl);
+    finalUrl.searchParams.append("token", scrapeDoApiKey);
+    finalUrl.searchParams.append("extraHeaders", "true");

-    const res = await fetch(scrapingbeeUrl.toString(), {
-      headers: { "Spb-Authorization": jwt },
+    const res = await fetch(finalUrl.toString(), {
+      headers: {
+        "sd-Authorization": jwt,
+        "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
+      },
    });

    if (res.status !== 200 || !res.ok) {
@@ -75,38 +78,32 @@ export const getSessionToken = async (payload: {
  code: string;
  userType: number;
 }): Promise<{ ok: boolean; message: string }> => {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
-  if (!scrapingbeeApiKey) {
-    logger.error("[getSessionToken] ScrapingBee API key not configured");
-    return { ok: false, message: "ScrapingBee API key not configured" };
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[getSessionToken] Scrape.do API key not configured");
+    return { ok: false, message: "Scrape.do API key not configured" };
  }

  const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`;

  logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`);

-  const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-  scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-  scrapingbeeUrl.searchParams.set("url", targetUrl);
-  scrapingbeeUrl.searchParams.set("forward_headers", "true");
-  scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
-
-  // Prefix headers with Spb- for ScrapingBee to forward them
-  const forwardHeaders = Object.fromEntries(
-    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
-  );
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");

  try {
-    const res = await fetch(scrapingbeeUrl.toString(), {
+    const res = await fetch(finalUrl.toString(), {
      method: "POST",
      body: JSON.stringify(payload),
-      headers: { "Spb-Content-Type": "application/json", ...forwardHeaders },
+      headers: { "sd-Content-Type": "application/json" },
    });

    if (!res.ok) {
      const errorText = await res.text().catch(() => "Unknown error");
      logger.error(
-        `[getSessionToken] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`,
+        `[getSessionToken] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`,
      );
      return {
        ok: false,
@@ -130,30 +127,32 @@ export const getSessionToken = async (payload: {
 };

 export async function getUsersBalance(userId: number, jwt: string) {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
-  if (!scrapingbeeApiKey) {
-    logger.error("[getUsersBalance] ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[getUsersBalance] Scrape.do API key not configured");
    return false;
  }

-  // Prefix headers with Spb- for ScrapingBee to forward them
+  // Prefix headers with sd- for Scrape.do to forward them
  const forwardHeaders = Object.fromEntries(
-    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
+    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
  );

  try {
    const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`;

-    const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-    scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-    scrapingbeeUrl.searchParams.set("url", targetUrl);
-    scrapingbeeUrl.searchParams.set("forward_headers", "true");
-    scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
+    const jwtUrlEncoded = encodeURIComponent(jwt);
+    const finalUrl = new URL("http://api.scrape.do/");
+    finalUrl.searchParams.append("url", targetUrl);
+    finalUrl.searchParams.append("token", scrapeDoApiKey);
+    finalUrl.searchParams.append("extraHeaders", "true");

-    const encodedJwt = encodeURIComponent(jwt);
-
-    const res = await fetch(scrapingbeeUrl.toString(), {
-      headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` },
+    const res = await fetch(finalUrl.toString(), {
+      headers: {
+        "sd-Authorization": jwt,
+        "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
+        ...forwardHeaders,
+      },
    });

    const rj = (await res.json()) as {
@@ -176,12 +175,12 @@ export async function getUsersBalance(userId: number, jwt: string) {
 }

 export const getDealers = async (jwt: string, distributor_ids: string[]) => {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
-  if (!scrapingbeeApiKey) {
-    logger.error("[getDealers] ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[getDealers] Scrape.do API key not configured");
    return {
      dealers: [],
-      errors: [{ message: "ScrapingBee API key not configured" }],
+      errors: [{ message: "Scrape.do API key not configured" }],
    };
  }

@@ -194,23 +193,30 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
      message: string;
    }> = [];

+    // Prefix headers with sd- for Scrape.do to forward them
+    const forwardHeaders = Object.fromEntries(
+      Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
+    );
+
    // Process each batch sequentially
    for (const batch of batches) {
      const batchRequests = batch.map(async (did, index) => {
        await sleep(rng(100, 2000));

        const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`;
-        const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-        scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-        scrapingbeeUrl.searchParams.set("url", targetUrl);
-        scrapingbeeUrl.searchParams.set("forward_headers", "true");
-        scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
+        const jwtUrlEncoded = encodeURIComponent(jwt);
+        const finalUrl = new URL("http://api.scrape.do/");
+        finalUrl.searchParams.append("url", targetUrl);
+        finalUrl.searchParams.append("token", scrapeDoApiKey);
+        finalUrl.searchParams.append("extraHeaders", "true");

-        const res = await fetch(scrapingbeeUrl.toString(), {
+        const res = await fetch(finalUrl.toString(), {
          method: "POST",
          headers: {
-            "Spb-Authorization": jwt,
-            "Spb-Content-Type": "application/json",
+            "sd-Authorization": jwt,
+            "sd-Content-Type": "application/json",
+            "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
+            ...forwardHeaders,
          },
          body: JSON.stringify({
            page: 1,
@@ -280,28 +286,35 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
 };

 export const getDistributors = async (jwt: string) => {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
-  if (!scrapingbeeApiKey) {
-    logger.error("[getDistributors] ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[getDistributors] Scrape.do API key not configured");
    return {
      ok: false,
-      message: "ScrapingBee API key not configured",
+      message: "Scrape.do API key not configured",
      data: [],
    };
  }

  const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`;
-  const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-  scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-  scrapingbeeUrl.searchParams.set("url", targetUrl);
-  scrapingbeeUrl.searchParams.set("forward_headers", "true");
-  scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
+  const jwtUrlEncoded = encodeURIComponent(jwt);
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");

-  const res = await fetch(scrapingbeeUrl.toString(), {
+  // Prefix headers with sd- for Scrape.do to forward them
+  const forwardHeaders = Object.fromEntries(
+    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
+  );
+
+  const res = await fetch(finalUrl.toString(), {
    method: "POST",
    headers: {
-      "Spb-Authorization": jwt,
-      "Spb-Content-Type": "application/json",
+      "sd-Authorization": jwt,
+      "sd-Content-Type": "application/json",
+      "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
+      ...forwardHeaders,
    },
    body: JSON.stringify({
      page: 1,
@@ -333,19 +346,32 @@ export const getDistributors = async (jwt: string) => {
 };

 export const getDraws = async (jwt: string) => {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[getDraws] Scrape.do API key not configured");
+    return { ok: false, message: "Scrape.do API key not configured", data: [] };
+  }

  logger.info(`[getDraws] Fetching draws from the API`);

  const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`;
-  const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-  scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-  scrapingbeeUrl.searchParams.set("url", targetUrl);
-  scrapingbeeUrl.searchParams.set("forward_headers", "true");
-  scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
+  const jwtUrlEncoded = encodeURIComponent(jwt);
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");

-  const res = await fetch(scrapingbeeUrl.toString(), {
-    headers: { "Spb-Authorization": jwt },
+  // Prefix headers with sd- for Scrape.do to forward them
+  const forwardHeaders = Object.fromEntries(
+    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
+  );
+
+  const res = await fetch(finalUrl.toString(), {
+    headers: {
+      "sd-Authorization": jwt,
+      "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
+      ...forwardHeaders,
+    },
  });

  type J = {
--- a/src/lib/server/postdata/post.handler.ts
+++ b/src/lib/server/postdata/post.handler.ts
@@ -205,31 +205,27 @@ async function sendBatchRequest(
  changedBalance: number,
  body: string,
 ) {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
-  if (!scrapingbeeApiKey) {
-    logger.error("[sendBatchRequest] ScrapingBee API key not configured");
-    throw new Error("ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[sendBatchRequest] Scrape.do API key not configured");
+    throw new Error("Scrape.do API key not configured");
  }

  const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`;
-  const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-  scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-  scrapingbeeUrl.searchParams.set("url", targetUrl);
-  scrapingbeeUrl.searchParams.set("forward_headers", "true");
-
-  // Prefix headers with Spb- for ScrapingBee to forward them
-  const forwardHeaders = Object.fromEntries(
-    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
-  );
+  const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");

  logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`);

-  return fetch(scrapingbeeUrl.toString(), {
+  return fetch(finalUrl.toString(), {
    method: "POST",
    headers: {
-      "Spb-Authorization": session.sessionToken,
-      "Spb-Content-Type": "application/json",
-      ...forwardHeaders,
+      "sd-Authorization": session.sessionToken,
+      "sd-Content-Type": "application/json",
+      "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
    },
    body: JSON.stringify({
      dealerId,
@@ -310,31 +306,27 @@ async function deleteAllBookedEntries({
  drawId: number;
  closeTime: string;
 }) {
-  const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
-  if (!scrapingbeeApiKey) {
-    logger.error("[deleteAllBookedEntries] ScrapingBee API key not configured");
-    throw new Error("ScrapingBee API key not configured");
+  const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+  if (!scrapeDoApiKey) {
+    logger.error("[deleteAllBookedEntries] Scrape.do API key not configured");
+    throw new Error("Scrape.do API key not configured");
  }

  const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`;
-  const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-  scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-  scrapingbeeUrl.searchParams.set("url", targetUrl);
-  scrapingbeeUrl.searchParams.set("forward_headers", "true");
-
-  // Prefix headers with Spb- for ScrapingBee to forward them
-  const forwardHeaders = Object.fromEntries(
-    Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
-  );
+  const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
+  const finalUrl = new URL("http://api.scrape.do/");
+  finalUrl.searchParams.append("url", targetUrl);
+  finalUrl.searchParams.append("token", scrapeDoApiKey);
+  finalUrl.searchParams.append("extraHeaders", "true");

  logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`);

-  return fetch(scrapingbeeUrl.toString(), {
+  return fetch(finalUrl.toString(), {
    method: "POST",
    headers: {
-      "Spb-Authorization": session.sessionToken,
-      "Spb-Content-Type": "application/json",
-      ...forwardHeaders,
+      "sd-Authorization": session.sessionToken,
+      "sd-Content-Type": "application/json",
+      "sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
    },
    body: JSON.stringify({
      bookIds: data.map((e) => e.bookId),
--- a/src/lib/trpc/routers/apiauth.router.ts
+++ b/src/lib/trpc/routers/apiauth.router.ts
@@ -17,7 +17,14 @@ import { createTRPCRouter, protectedProcedure } from "../t";

 export const apiAuthRouter = createTRPCRouter({
  getCaptcha: protectedProcedure.mutation(async () => {
-    const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY ?? "";
+    const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
+    if (!scrapeDoApiKey) {
+      logger.error("[getCaptcha] Scrape.do API key not configured");
+      throw new TRPCError({
+        code: "INTERNAL_SERVER_ERROR",
+        message: "Scrape.do API key not configured",
+      });
+    }

    try {
      const uuid = getUUID();
@@ -25,22 +32,17 @@ export const apiAuthRouter = createTRPCRouter({

      logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`);

-      // Build ScrapingBee API URL with params
-      const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
-      scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
-      scrapingbeeUrl.searchParams.set("url", targetUrl);
-      scrapingbeeUrl.searchParams.set("render_js", "false");
-      scrapingbeeUrl.searchParams.set("block_resources", "false");
+      const finalUrl = new URL("http://api.scrape.do/");
+      finalUrl.searchParams.append("url", targetUrl);
+      finalUrl.searchParams.append("token", scrapeDoApiKey);

-      const res = await fetch(scrapingbeeUrl.toString());
+      const res = await fetch(finalUrl.toString());

      if (!res.ok || res.status !== 200) {
        // Clone response before reading to avoid consuming body
        const clonedRes = res.clone();
        const errorText = await clonedRes.text().catch(() => "Unknown error");
-        logger.error(
-          `[getCaptcha] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`,
-        );
+        logger.error(`[getCaptcha] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`);
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: `Failed to fetch captcha image: ${res.status}`,