Compare commits
5 Commits
0111933d92
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
905405d0ae | ||
|
|
653fb266e5 | ||
|
|
f3d186b195 | ||
|
|
0d8cbe295c | ||
|
|
4d4d7cfa93 |
@@ -1,4 +1,4 @@
|
|||||||
FROM node:18-alpine
|
FROM node:24-alpine
|
||||||
|
|
||||||
RUN apk add --no-cache libc6-compat
|
RUN apk add --no-cache libc6-compat
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite dev",
|
"dev": "vite dev",
|
||||||
"start": "HOST=0.0.0.0 PORT=80 node ./build/index.js",
|
"start": "HOST=0.0.0.0 PORT=3000 node ./build/index.js",
|
||||||
"build": "pnpm run check && vite build",
|
"build": "pnpm run check && vite build",
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
|
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
|
||||||
|
|||||||
172
src/lib/server/external/api.scraping.helpers.ts
vendored
172
src/lib/server/external/api.scraping.helpers.ts
vendored
@@ -30,23 +30,26 @@ function dumpDataRaw(data: any, prefix: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const testIfSessionIsValid = async (jwt: string) => {
|
export const testIfSessionIsValid = async (jwt: string) => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[testIfSessionIsValid] ScrapingBee API key not configured");
|
logger.error("[testIfSessionIsValid] Scrape.do API key not configured");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${baseDistributorId}`;
|
||||||
|
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(jwt);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
|
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
const res = await fetch(finalUrl.toString(), {
|
||||||
headers: { "Spb-Authorization": jwt },
|
headers: {
|
||||||
|
"sd-Authorization": jwt,
|
||||||
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status !== 200 || !res.ok) {
|
if (res.status !== 200 || !res.ok) {
|
||||||
@@ -75,38 +78,32 @@ export const getSessionToken = async (payload: {
|
|||||||
code: string;
|
code: string;
|
||||||
userType: number;
|
userType: number;
|
||||||
}): Promise<{ ok: boolean; message: string }> => {
|
}): Promise<{ ok: boolean; message: string }> => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[getSessionToken] ScrapingBee API key not configured");
|
logger.error("[getSessionToken] Scrape.do API key not configured");
|
||||||
return { ok: false, message: "ScrapingBee API key not configured" };
|
return { ok: false, message: "Scrape.do API key not configured" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/auth/login`;
|
||||||
|
|
||||||
logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`);
|
logger.info(`[getSessionToken] Requesting session token for user ${payload.userId}`);
|
||||||
|
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
|
||||||
|
|
||||||
// Prefix headers with Spb- for ScrapingBee to forward them
|
|
||||||
const forwardHeaders = Object.fromEntries(
|
|
||||||
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
const res = await fetch(finalUrl.toString(), {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify(payload),
|
body: JSON.stringify(payload),
|
||||||
headers: { "Spb-Content-Type": "application/json", ...forwardHeaders },
|
headers: { "sd-Content-Type": "application/json" },
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const errorText = await res.text().catch(() => "Unknown error");
|
const errorText = await res.text().catch(() => "Unknown error");
|
||||||
logger.error(
|
logger.error(
|
||||||
`[getSessionToken] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`,
|
`[getSessionToken] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`,
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
@@ -130,30 +127,32 @@ export const getSessionToken = async (payload: {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export async function getUsersBalance(userId: number, jwt: string) {
|
export async function getUsersBalance(userId: number, jwt: string) {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[getUsersBalance] ScrapingBee API key not configured");
|
logger.error("[getUsersBalance] Scrape.do API key not configured");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix headers with Spb- for ScrapingBee to forward them
|
// Prefix headers with sd- for Scrape.do to forward them
|
||||||
const forwardHeaders = Object.fromEntries(
|
const forwardHeaders = Object.fromEntries(
|
||||||
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
|
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/get-balance?userId=${userId}`;
|
||||||
|
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(jwt);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
|
|
||||||
const encodedJwt = encodeURIComponent(jwt);
|
const res = await fetch(finalUrl.toString(), {
|
||||||
|
headers: {
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
"sd-Authorization": jwt,
|
||||||
headers: { "Spb-Authorization": jwt, "Spb-Cookie": `AuthorizationToken=${encodedJwt}` },
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
|
...forwardHeaders,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const rj = (await res.json()) as {
|
const rj = (await res.json()) as {
|
||||||
@@ -176,12 +175,12 @@ export async function getUsersBalance(userId: number, jwt: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const getDealers = async (jwt: string, distributor_ids: string[]) => {
|
export const getDealers = async (jwt: string, distributor_ids: string[]) => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[getDealers] ScrapingBee API key not configured");
|
logger.error("[getDealers] Scrape.do API key not configured");
|
||||||
return {
|
return {
|
||||||
dealers: [],
|
dealers: [],
|
||||||
errors: [{ message: "ScrapingBee API key not configured" }],
|
errors: [{ message: "Scrape.do API key not configured" }],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,23 +193,30 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
|
|||||||
message: string;
|
message: string;
|
||||||
}> = [];
|
}> = [];
|
||||||
|
|
||||||
|
// Prefix headers with sd- for Scrape.do to forward them
|
||||||
|
const forwardHeaders = Object.fromEntries(
|
||||||
|
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
|
||||||
|
);
|
||||||
|
|
||||||
// Process each batch sequentially
|
// Process each batch sequentially
|
||||||
for (const batch of batches) {
|
for (const batch of batches) {
|
||||||
const batchRequests = batch.map(async (did, index) => {
|
const batchRequests = batch.map(async (did, index) => {
|
||||||
await sleep(rng(100, 2000));
|
await sleep(rng(100, 2000));
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/dealer-list`;
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(jwt);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
|
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
const res = await fetch(finalUrl.toString(), {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
"Spb-Authorization": jwt,
|
"sd-Authorization": jwt,
|
||||||
"Spb-Content-Type": "application/json",
|
"sd-Content-Type": "application/json",
|
||||||
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
|
...forwardHeaders,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
page: 1,
|
page: 1,
|
||||||
@@ -280,28 +286,35 @@ export const getDealers = async (jwt: string, distributor_ids: string[]) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export const getDistributors = async (jwt: string) => {
|
export const getDistributors = async (jwt: string) => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[getDistributors] ScrapingBee API key not configured");
|
logger.error("[getDistributors] Scrape.do API key not configured");
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
message: "ScrapingBee API key not configured",
|
message: "Scrape.do API key not configured",
|
||||||
data: [],
|
data: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/user/distributor-list`;
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(jwt);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
|
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
// Prefix headers with sd- for Scrape.do to forward them
|
||||||
|
const forwardHeaders = Object.fromEntries(
|
||||||
|
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
|
||||||
|
);
|
||||||
|
|
||||||
|
const res = await fetch(finalUrl.toString(), {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
"Spb-Authorization": jwt,
|
"sd-Authorization": jwt,
|
||||||
"Spb-Content-Type": "application/json",
|
"sd-Content-Type": "application/json",
|
||||||
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
|
...forwardHeaders,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
page: 1,
|
page: 1,
|
||||||
@@ -333,19 +346,32 @@ export const getDistributors = async (jwt: string) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export const getDraws = async (jwt: string) => {
|
export const getDraws = async (jwt: string) => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
|
if (!scrapeDoApiKey) {
|
||||||
|
logger.error("[getDraws] Scrape.do API key not configured");
|
||||||
|
return { ok: false, message: "Scrape.do API key not configured", data: [] };
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(`[getDraws] Fetching draws from the API`);
|
logger.info(`[getDraws] Fetching draws from the API`);
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/draw/list-my?userId=15`;
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(jwt);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers_pure", "true");
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
|
|
||||||
const res = await fetch(scrapingbeeUrl.toString(), {
|
// Prefix headers with sd- for Scrape.do to forward them
|
||||||
headers: { "Spb-Authorization": jwt },
|
const forwardHeaders = Object.fromEntries(
|
||||||
|
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`sd-${key}`, value]),
|
||||||
|
);
|
||||||
|
|
||||||
|
const res = await fetch(finalUrl.toString(), {
|
||||||
|
headers: {
|
||||||
|
"sd-Authorization": jwt,
|
||||||
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
|
...forwardHeaders,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
type J = {
|
type J = {
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import winston from "winston";
|
|
||||||
import DailyRotateFile from "winston-daily-rotate-file";
|
|
||||||
import util from "util";
|
|
||||||
import { Err } from "./result";
|
|
||||||
import { env } from "$env/dynamic/private";
|
import { env } from "$env/dynamic/private";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
|
import util from "util";
|
||||||
|
import winston from "winston";
|
||||||
|
import DailyRotateFile from "winston-daily-rotate-file";
|
||||||
|
import type { Err } from "./result";
|
||||||
|
|
||||||
process.on("warning", (warning) => {
|
process.on("warning", (warning) => {
|
||||||
const msg = String(warning?.message || "");
|
const msg = String(warning?.message || "");
|
||||||
@@ -73,14 +73,14 @@ const consoleFormat = winston.format.combine(
|
|||||||
: "";
|
: "";
|
||||||
|
|
||||||
return `[${level}] ${timestamp}: ${formattedMessage}${formattedExtra}`;
|
return `[${level}] ${timestamp}: ${formattedMessage}${formattedExtra}`;
|
||||||
})
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
// JSON format for file logging
|
// JSON format for file logging
|
||||||
const fileFormat = winston.format.combine(
|
const fileFormat = winston.format.combine(
|
||||||
winston.format.errors({ stack: true }),
|
winston.format.errors({ stack: true }),
|
||||||
winston.format.timestamp(),
|
winston.format.timestamp(),
|
||||||
winston.format.json()
|
winston.format.json(),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Log directory - use logs folder in project root
|
// Log directory - use logs folder in project root
|
||||||
|
|||||||
@@ -205,31 +205,27 @@ async function sendBatchRequest(
|
|||||||
changedBalance: number,
|
changedBalance: number,
|
||||||
body: string,
|
body: string,
|
||||||
) {
|
) {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[sendBatchRequest] ScrapingBee API key not configured");
|
logger.error("[sendBatchRequest] Scrape.do API key not configured");
|
||||||
throw new Error("ScrapingBee API key not configured");
|
throw new Error("Scrape.do API key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/add-multiple`;
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
// Prefix headers with Spb- for ScrapingBee to forward them
|
|
||||||
const forwardHeaders = Object.fromEntries(
|
|
||||||
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`);
|
logger.debug(`[sendBatchRequest] Sending batch request for dealer ${dealerId}, draw ${draw.id}`);
|
||||||
|
|
||||||
return fetch(scrapingbeeUrl.toString(), {
|
return fetch(finalUrl.toString(), {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
"Spb-Authorization": session.sessionToken,
|
"sd-Authorization": session.sessionToken,
|
||||||
"Spb-Content-Type": "application/json",
|
"sd-Content-Type": "application/json",
|
||||||
...forwardHeaders,
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
dealerId,
|
dealerId,
|
||||||
@@ -310,31 +306,27 @@ async function deleteAllBookedEntries({
|
|||||||
drawId: number;
|
drawId: number;
|
||||||
closeTime: string;
|
closeTime: string;
|
||||||
}) {
|
}) {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
if (!scrapingbeeApiKey) {
|
if (!scrapeDoApiKey) {
|
||||||
logger.error("[deleteAllBookedEntries] ScrapingBee API key not configured");
|
logger.error("[deleteAllBookedEntries] Scrape.do API key not configured");
|
||||||
throw new Error("ScrapingBee API key not configured");
|
throw new Error("Scrape.do API key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`;
|
const targetUrl = `${constants.SCRAP_API_URL}/v1/book/delete-multiple`;
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
const jwtUrlEncoded = encodeURIComponent(session.sessionToken);
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
scrapingbeeUrl.searchParams.set("forward_headers", "true");
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
|
finalUrl.searchParams.append("extraHeaders", "true");
|
||||||
// Prefix headers with Spb- for ScrapingBee to forward them
|
|
||||||
const forwardHeaders = Object.fromEntries(
|
|
||||||
Object.entries(constants.SCRAP_API_BASE_HEADERS).map(([key, value]) => [`Spb-${key}`, value]),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`);
|
logger.debug(`[deleteAllBookedEntries] Deleting ${data.length} entries for dealer ${dealerId}, draw ${drawId}`);
|
||||||
|
|
||||||
return fetch(scrapingbeeUrl.toString(), {
|
return fetch(finalUrl.toString(), {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
"Spb-Authorization": session.sessionToken,
|
"sd-Authorization": session.sessionToken,
|
||||||
"Spb-Content-Type": "application/json",
|
"sd-Content-Type": "application/json",
|
||||||
...forwardHeaders,
|
"sd-Cookie": `AuthorizationToken=${jwtUrlEncoded}`,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
bookIds: data.map((e) => e.bookId),
|
bookIds: data.map((e) => e.bookId),
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
|
import { env } from "$env/dynamic/private";
|
||||||
import { dbApiUser } from "$lib/server/db/apiuser.db";
|
import { dbApiUser } from "$lib/server/db/apiuser.db";
|
||||||
import { getSessionToken } from "$lib/server/external/api.scraping.helpers";
|
import { getSessionToken } from "$lib/server/external/api.scraping.helpers";
|
||||||
|
import { logger } from "$lib/server/logger";
|
||||||
import {
|
import {
|
||||||
isSessionValidInStore,
|
isSessionValidInStore,
|
||||||
removeSessionFromStore,
|
removeSessionFromStore,
|
||||||
@@ -9,37 +11,38 @@ import { getUUID } from "$lib/utils";
|
|||||||
import { constants } from "$lib/utils/constants";
|
import { constants } from "$lib/utils/constants";
|
||||||
import type { ServerError } from "$lib/utils/data.types";
|
import type { ServerError } from "$lib/utils/data.types";
|
||||||
import { TRPCError } from "@trpc/server";
|
import { TRPCError } from "@trpc/server";
|
||||||
|
import fetch from "node-fetch";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { createTRPCRouter, protectedProcedure } from "../t";
|
import { createTRPCRouter, protectedProcedure } from "../t";
|
||||||
import { env } from "$env/dynamic/private";
|
|
||||||
import { logger } from "$lib/server/logger";
|
|
||||||
import fetch from "node-fetch";
|
|
||||||
|
|
||||||
|
|
||||||
export const apiAuthRouter = createTRPCRouter({
|
export const apiAuthRouter = createTRPCRouter({
|
||||||
getCaptcha: protectedProcedure.mutation(async () => {
|
getCaptcha: protectedProcedure.mutation(async () => {
|
||||||
const scrapingbeeApiKey = env.SCRAPINGBEE_API_KEY;
|
const scrapeDoApiKey = env.SCRAPEDO_API_KEY ?? "";
|
||||||
|
if (!scrapeDoApiKey) {
|
||||||
|
logger.error("[getCaptcha] Scrape.do API key not configured");
|
||||||
|
throw new TRPCError({
|
||||||
|
code: "INTERNAL_SERVER_ERROR",
|
||||||
|
message: "Scrape.do API key not configured",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const uuid = getUUID();
|
const uuid = getUUID();
|
||||||
const targetUrl = `${constants.SCRAP_API_URL}/verify/image?uuid=${uuid}`;
|
const targetUrl = `${constants.SCRAP_API_URL}/verify/image?uuid=${uuid}`;
|
||||||
|
|
||||||
logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`);
|
|
||||||
|
|
||||||
// Build ScrapingBee API URL with params
|
|
||||||
const scrapingbeeUrl = new URL("https://app.scrapingbee.com/api/v1");
|
|
||||||
scrapingbeeUrl.searchParams.set("api_key", scrapingbeeApiKey);
|
|
||||||
scrapingbeeUrl.searchParams.set("url", targetUrl);
|
|
||||||
scrapingbeeUrl.searchParams.set("render_js", "false");
|
|
||||||
scrapingbeeUrl.searchParams.set("block_resources", "false");
|
|
||||||
|
|
||||||
const res = await fetch(scrapingbeeUrl.toString());
|
logger.info(`[getCaptcha] Fetching captcha image for uuid: ${uuid}`);
|
||||||
|
|
||||||
|
const finalUrl = new URL("http://api.scrape.do/");
|
||||||
|
finalUrl.searchParams.append("url", targetUrl);
|
||||||
|
finalUrl.searchParams.append("token", scrapeDoApiKey);
|
||||||
|
|
||||||
|
const res = await fetch(finalUrl.toString());
|
||||||
|
|
||||||
if (!res.ok || res.status !== 200) {
|
if (!res.ok || res.status !== 200) {
|
||||||
// Clone response before reading to avoid consuming body
|
// Clone response before reading to avoid consuming body
|
||||||
const clonedRes = res.clone();
|
const clonedRes = res.clone();
|
||||||
const errorText = await clonedRes.text().catch(() => "Unknown error");
|
const errorText = await clonedRes.text().catch(() => "Unknown error");
|
||||||
logger.error(`[getCaptcha] ScrapingBee error ${res.status}: ${errorText.substring(0, 200)}`);
|
logger.error(`[getCaptcha] Scrape.do error ${res.status}: ${errorText.substring(0, 200)}`);
|
||||||
throw new TRPCError({
|
throw new TRPCError({
|
||||||
code: "INTERNAL_SERVER_ERROR",
|
code: "INTERNAL_SERVER_ERROR",
|
||||||
message: `Failed to fetch captcha image: ${res.status}`,
|
message: `Failed to fetch captcha image: ${res.status}`,
|
||||||
@@ -50,8 +53,10 @@ export const apiAuthRouter = createTRPCRouter({
|
|||||||
const arrayBuffer = await res.arrayBuffer();
|
const arrayBuffer = await res.arrayBuffer();
|
||||||
const imageBuffer = Buffer.from(arrayBuffer);
|
const imageBuffer = Buffer.from(arrayBuffer);
|
||||||
const base64String = imageBuffer.toString("base64");
|
const base64String = imageBuffer.toString("base64");
|
||||||
|
|
||||||
logger.info(`[getCaptcha] Successfully fetched captcha image for uuid: ${uuid}, size: ${imageBuffer.length} bytes`);
|
logger.info(
|
||||||
|
`[getCaptcha] Successfully fetched captcha image for uuid: ${uuid}, size: ${imageBuffer.length} bytes`,
|
||||||
|
);
|
||||||
return { id: uuid, image: base64String };
|
return { id: uuid, image: base64String };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error("[getCaptcha] Error getting captcha image", err);
|
logger.error("[getCaptcha] Error getting captcha image", err);
|
||||||
@@ -90,7 +95,7 @@ export const apiAuthRouter = createTRPCRouter({
|
|||||||
password = _user.password;
|
password = _user.password;
|
||||||
logger.info(`[getNewSession] Using specific user: ${userId}`);
|
logger.info(`[getNewSession] Using specific user: ${userId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`[getNewSession] Getting session token for user ${userId}`);
|
logger.info(`[getNewSession] Getting session token for user ${userId}`);
|
||||||
const token = await getSessionToken({
|
const token = await getSessionToken({
|
||||||
code: captchaAnswer,
|
code: captchaAnswer,
|
||||||
@@ -99,7 +104,7 @@ export const apiAuthRouter = createTRPCRouter({
|
|||||||
userType: userType,
|
userType: userType,
|
||||||
password: password,
|
password: password,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!token.ok) {
|
if (!token.ok) {
|
||||||
logger.warn(`[getNewSession] Failed to get session token: ${token.message}`);
|
logger.warn(`[getNewSession] Failed to get session token: ${token.message}`);
|
||||||
return {
|
return {
|
||||||
@@ -107,7 +112,7 @@ export const apiAuthRouter = createTRPCRouter({
|
|||||||
errors: [{ message: token.message }],
|
errors: [{ message: token.message }],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
await setSessionToRedis(token.message, input.userId ?? "");
|
await setSessionToRedis(token.message, input.userId ?? "");
|
||||||
logger.info(`[getNewSession] Successfully created session for user ${userId}`);
|
logger.info(`[getNewSession] Successfully created session for user ${userId}`);
|
||||||
return { success: true, errors: [] as ServerError };
|
return { success: true, errors: [] as ServerError };
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import { dbPresetData } from "$lib/server/db/presetdata.db";
|
import { dbPresetData } from "$lib/server/db/presetdata.db";
|
||||||
|
import { logger } from "$lib/server/logger";
|
||||||
import { zDDFilters, zPresetDataEntry } from "$lib/utils/data.types";
|
import { zDDFilters, zPresetDataEntry } from "$lib/utils/data.types";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { createTRPCRouter, protectedProcedure } from "../t";
|
import { createTRPCRouter, protectedProcedure } from "../t";
|
||||||
import { logger } from "$lib/server/logger";
|
|
||||||
|
|
||||||
export const presetDataRouter = createTRPCRouter({
|
export const presetDataRouter = createTRPCRouter({
|
||||||
getAll: protectedProcedure.input(zDDFilters).mutation(async ({ input }) => {
|
getAll: protectedProcedure.input(zDDFilters).mutation(async ({ input }) => {
|
||||||
@@ -21,23 +21,23 @@ export const presetDataRouter = createTRPCRouter({
|
|||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
|
|
||||||
insert: protectedProcedure
|
insert: protectedProcedure.input(z.array(zPresetDataEntry)).mutation(async ({ input }) => {
|
||||||
.input(z.array(zPresetDataEntry))
|
logger.info(`[presetData.insert] Inserting ${input.length} preset data entries`);
|
||||||
.mutation(async ({ input }) => {
|
const data = await dbPresetData.insertData(input);
|
||||||
logger.info(`[presetData.insert] Inserting ${input.length} preset data entries`);
|
logger.info(`[presetData.insert] Successfully inserted ${data?.length} entries`);
|
||||||
const data = await dbPresetData.insertData(input);
|
return {
|
||||||
logger.info(`[presetData.insert] Successfully inserted ${data.length} entries`);
|
ok: true,
|
||||||
return {
|
detail: "Data inserted",
|
||||||
ok: true,
|
data,
|
||||||
detail: "Data inserted",
|
};
|
||||||
data,
|
}),
|
||||||
};
|
|
||||||
}),
|
|
||||||
|
|
||||||
delete: protectedProcedure
|
delete: protectedProcedure
|
||||||
.input(z.object({ date: z.string(), ids: z.array(z.string()) }))
|
.input(z.object({ date: z.string(), ids: z.array(z.string()) }))
|
||||||
.mutation(async ({ input }) => {
|
.mutation(async ({ input }) => {
|
||||||
logger.info(`[presetData.delete] Deleting ${input.ids.length} preset data entries for date ${input.date}`);
|
logger.info(
|
||||||
|
`[presetData.delete] Deleting ${input.ids.length} preset data entries for date ${input.date}`,
|
||||||
|
);
|
||||||
await dbPresetData.deleteDataByIds(input.date, input.ids);
|
await dbPresetData.deleteDataByIds(input.date, input.ids);
|
||||||
logger.info("[presetData.delete] Successfully deleted preset data entries");
|
logger.info("[presetData.delete] Successfully deleted preset data entries");
|
||||||
return { ok: true, detail: "Data deleted" };
|
return { ok: true, detail: "Data deleted" };
|
||||||
|
|||||||
Reference in New Issue
Block a user