Bullshit MF headers not being passed ahead

src/lib/server/external/api.scraping.helpers.ts

@@ -6,6 +6,7 @@ import { baseDistributorId, constants } from "$lib/utils/constants";
 import type { BookingEntry, Draw, LooseApiUser } from "$lib/utils/data.types";
 import { rng } from "$lib/utils/rng";
 import fs from "fs";
+import { HttpProxyAgent } from "http-proxy-agent";
 import fetch from "node-fetch";
 
 // function dumpDistributors(distributors: LooseApiUser[]) {
@@ -371,39 +372,27 @@ export const getData = async (
   drawId: number,
   chosenDate: string,
 ) => {
-  const scraperApiKey = env.SCRAPER_API_KEY ?? "";
+  const scraperApiKey = env.SCRAPERAPI_API_KEY ?? "";
   const targetUrl = `${constants.SCRAP_API_URL}/v1/book/list2`;
 
   logger.info(
     `[getData] Fetching draw data from API for ${chosenDate} ${drawId} for ${userIds.length} users`,
   );
 
-  const proxyConfig = {
-    host: "proxy-server.scraperapi.com",
-    port: 8001,
-    auth: {
-      user: "scraperapi",
-      password: scraperApiKey,
-    },
-    protocol: "http",
-  };
+  const proxyUsername = "scraperapi.keep_headers=true";
 
-  const apiUrl = new URL(`https://api.scraperapi.com/`);
-  apiUrl.searchParams.append("api_key", scraperApiKey);
-  apiUrl.searchParams.append("url", targetUrl);
-  apiUrl.searchParams.append("follow_redirect", "false");
-  apiUrl.searchParams.append("keep_headers", "true");
-  apiUrl.searchParams.append("device_type", "desktop");
+  // Configure HTTP proxy agent
+  const proxyAgent = new HttpProxyAgent(
+    `http://${proxyUsername}:${scraperApiKey}@proxy-server.scraperapi.com:8001`,
+  );
 
-  logger.debug(`[getData] Scraping for data at : ${apiUrl.toString()}`);
+  logger.debug(`[getData] Using proxy to fetch data from : ${targetUrl}`);
 
-  const res = await fetch(apiUrl.toString(), {
+  const res = await fetch(targetUrl, {
     method: "POST",
     headers: {
-      Authorization: jwt,
       "Content-Type": "application/json",
-      ...constants.SCRAP_API_BASE_HEADERS,
-      "Accept-Encoding": "gzip, deflate, br, zstd",
+      Authorization: jwt,
     },
     body: JSON.stringify({
       userType: 3,
@@ -415,6 +404,7 @@ export const getData = async (
       containImported: false,
       keyword: "",
     }),
+    agent: proxyAgent,
   });
 
   type J = {