old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: share scraper http fetching

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-29 13:14:20 -04:00
parent 22eb65d4a2
commit 6e50ebf901
6 changed files with 121 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/core/test/ebay-core.test.ts
View File

@@ -32,6 +32,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () => Promise.resolve("<html><body></body></html>"),
}),
) as unknown as typeof fetch;
@@ -64,6 +65,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -88,6 +90,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -114,6 +117,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -146,6 +150,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -188,6 +193,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -214,6 +220,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -243,6 +250,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -272,6 +280,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -301,6 +310,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -343,6 +353,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -375,6 +386,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -407,6 +419,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -440,6 +453,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -467,6 +481,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -499,6 +514,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -529,6 +545,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -574,6 +591,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>
@@ -612,6 +630,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () =>
Promise.resolve(`
<html><body>

19
packages/core/test/http.test.ts
View File

@@ -38,4 +38,23 @@ describe("fetchHtml", () => {
expect(scheduledDelays).not.toContain(1000);
});
test("fetchHtml returns responseUrl when includeResponseUrl is true", async () => {
process.env.NODE_ENV = "test";
global.fetch = mock(() =>
Promise.resolve({
ok: true,
status: 200,
url: "https://example.test/final",
headers: { get: () => null },
text: () => Promise.resolve("<html></html>"),
}),
) as unknown as typeof fetch;
const result = await fetchHtml("https://example.test", 0, {
includeResponseUrl: true,
});
expect(result.html).toBe("<html></html>");
expect(result.responseUrl).toBe("https://example.test/final");
});
});