old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(core): parse Kijiji StandardListing records

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-28 21:57:10 -04:00
parent 2a5701aeb9
commit 11dce39428
2 changed files with 180 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
packages/core/src/scrapers/kijiji.ts
View File

@@ -46,6 +46,17 @@ interface ApolloSearchItem {
[k: string]: unknown; [k: string]: unknown;
} }
type ListingAttribute = {
canonicalName?: string;
canonicalValues?: string[];
};
type ListingAttributes =
| ListingAttribute[]
| {
all?: ListingAttribute[];
};
interface ApolloListingRoot { interface ApolloListingRoot {
url?: string; url?: string;
title?: string; title?: string;
@@ -68,7 +79,7 @@ interface ApolloListingRoot {
adSource?: string; adSource?: string;
flags?: { topAd?: boolean; priceDrop?: boolean }; flags?: { topAd?: boolean; priceDrop?: boolean };
posterInfo?: { posterId?: string; rating?: number }; posterInfo?: { posterId?: string; rating?: number };
attributes?: Array<{ canonicalName?: string; canonicalValues?: string[] }>; attributes?: ListingAttributes;
[k: string]: unknown; [k: string]: unknown;
} }
@@ -323,13 +334,22 @@ function findApolloListingKey(
predicate: (value: Record<string, unknown>) => boolean, predicate: (value: Record<string, unknown>) => boolean,
): string | undefined { ): string | undefined {
return Object.keys(apolloState).find((key) => { return Object.keys(apolloState).find((key) => {
if (!key.startsWith("Listing:")) return false; if (!isListingRecordKey(key)) return false;
const value = apolloState[key]; const value = apolloState[key];
return isRecord(value) && predicate(value); return isRecord(value) && predicate(value);
}); });
} }
function isListingRecordKey(key: string): boolean {
return key.startsWith("Listing:") || key.startsWith("StandardListing:");
}
function getListingAttributes(attributes: ListingAttributes | undefined) {
if (Array.isArray(attributes)) return attributes;
return attributes?.all ?? [];
}
/** /**
* Slugifies a string for Kijiji search URLs * Slugifies a string for Kijiji search URLs
*/ */
@@ -532,7 +552,7 @@ export function parseSearch(
const results: SearchListing[] = []; const results: SearchListing[] = [];
for (const [key, value] of Object.entries(apolloState)) { for (const [key, value] of Object.entries(apolloState)) {
if (!key.startsWith("Listing:")) continue; if (!isListingRecordKey(key)) continue;
if (!isRecord(value)) continue; if (!isRecord(value)) continue;
const item = value as ApolloSearchItem; const item = value as ApolloSearchItem;
@@ -689,13 +709,11 @@ export async function parseDetailedListing(
// Extract attributes as key-value pairs // Extract attributes as key-value pairs
const attributeMap: Record<string, string[]> = {}; const attributeMap: Record<string, string[]> = {};
if (Array.isArray(attributes)) { for (const attr of getListingAttributes(attributes)) {
for (const attr of attributes) { if (attr.canonicalName && Array.isArray(attr.canonicalValues)) {
if (attr?.canonicalName && Array.isArray(attr.canonicalValues)) {
attributeMap[attr.canonicalName] = attr.canonicalValues; attributeMap[attr.canonicalName] = attr.canonicalValues;
} }
} }
}
// Extract seller info based on depth setting // Extract seller info based on depth setting
let sellerInfo: DetailedListing["sellerInfo"]; let sellerInfo: DetailedListing["sellerInfo"];

154
packages/core/test/kijiji-integration.test.ts
View File

@@ -149,6 +149,48 @@ describe("HTML Parsing Integration", () => {
expect(results[0]?.name).toBe("iPhone 13 Pro"); expect(results[0]?.name).toBe("iPhone 13 Pro");
}); });
test("should parse current StandardListing search records", () => {
const mockHtml = `
<html>
<script id="__NEXT_DATA__" type="application/json">
${JSON.stringify({
props: {
pageProps: {
__APOLLO_STATE__: {
ROOT_QUERY: { test: "value" },
"StandardListing:123": {
__typename: "StandardListing",
url: "https://www.kijiji.ca/v-cell-phone/city-of-toronto/iphone-13/123",
title: "iPhone 13",
},
"StandardListing:456": {
__typename: "StandardListing",
url: "/v-cell-phone/city-of-toronto/iphone-14/456",
title: "iPhone 14",
},
},
},
},
})}
</script>
</html>
`;
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
expect(results).toEqual([
{
name: "iPhone 13",
listingLink:
"https://www.kijiji.ca/v-cell-phone/city-of-toronto/iphone-13/123",
},
{
name: "iPhone 14",
listingLink:
"https://www.kijiji.ca/v-cell-phone/city-of-toronto/iphone-14/456",
},
]);
});
test("should return empty array for invalid HTML", () => { test("should return empty array for invalid HTML", () => {
const results = parseSearch( const results = parseSearch(
"<html><body>Invalid</body></html>", "<html><body>Invalid</body></html>",
@@ -303,6 +345,118 @@ describe("HTML Parsing Integration", () => {
expect(result).toBeNull(); expect(result).toBeNull();
}); });
test("should parse current StandardListing detail records", async () => {
const mockHtml = `
<html>
<script id="__NEXT_DATA__" type="application/json">
${JSON.stringify({
props: {
pageProps: {
__APOLLO_STATE__: {
"StandardListing:123": {
__typename: "StandardListing",
url: "https://www.kijiji.ca/v-cell-phone/city-of-toronto/iphone-13/123",
title: "iPhone 13",
description: "Lightly used iPhone 13",
price: {
__typename: "AmountPrice",
amount: 45000,
currency: "CAD",
type: "FIXED",
},
type: "OFFER",
status: "ACTIVE",
activationDate: "2026-04-20T10:00:00.000Z",
metrics: { views: "12" },
location: {
id: 1700273,
name: "City of Toronto",
address: "Toronto, ON",
coordinates: {
latitude: 43.6532,
longitude: -79.3832,
},
},
imageUrls: ["https://media.kijiji.ca/api/v1/image1.jpg"],
categoryId: 760,
adSource: "ORGANIC",
flags: {
topAd: false,
priceDrop: false,
},
posterInfo: {
posterId: "user123",
rating: 4.5,
},
attributes: {
__typename: "StandardListingAttributes",
all: [
{
__typename: "ListingAttributeV2",
canonicalName: "forsaleby",
canonicalValues: ["ownr"],
},
{
__typename: "ListingAttributeV2",
canonicalName: "phonebrand",
canonicalValues: ["apple"],
},
],
},
},
},
},
},
})}
</script>
</html>
`;
const result = await parseDetailedListing(
mockHtml,
"https://www.kijiji.ca",
);
expect(result).toEqual({
url: "https://www.kijiji.ca/v-cell-phone/city-of-toronto/iphone-13/123",
title: "iPhone 13",
description: "Lightly used iPhone 13",
listingPrice: {
amountFormatted: "$450.00",
cents: 45000,
currency: "CAD",
},
listingType: "OFFER",
listingStatus: "ACTIVE",
creationDate: "2026-04-20T10:00:00.000Z",
endDate: undefined,
numberOfViews: 12,
address: "Toronto, ON",
images: ["https://media.kijiji.ca/api/v1/image1.jpg"],
categoryId: 760,
adSource: "ORGANIC",
flags: {
topAd: false,
priceDrop: false,
},
attributes: {
forsaleby: ["ownr"],
phonebrand: ["apple"],
},
location: {
id: 1700273,
name: "City of Toronto",
coordinates: {
latitude: 43.6532,
longitude: -79.3832,
},
},
sellerInfo: {
posterId: "user123",
rating: 4.5,
},
});
});
test("should handle missing optional fields", async () => { test("should handle missing optional fields", async () => {
const mockHtml = ` const mockHtml = `
<html> <html>