UNPKG

recipe-scrapers-js

Version:
1,455 lines (1,431 loc) 56.3 kB
import * as cheerio from "cheerio"; import z$1, { z } from "zod"; import { parse, toSeconds } from "iso8601-duration"; import { parseIngredient } from "parse-ingredient"; //#region src/utils/index.ts function isDefined(value) { return value !== void 0; } function isNull(value) { return value === null; } function isFunction(value) { return typeof value === "function"; } function isNumber(value) { return typeof value === "number"; } function isPlainObject(value) { return typeof value === "object" && value !== null && Object.getPrototypeOf(value) === Object.prototype; } function isString(value) { return typeof value === "string"; } /** * Extracts the host name from a URL string * and removes 'www.' prefix if present. * Throws an error if the input is not a valid URL. */ function getHostName(value) { try { return new URL(value.replace("www.", "")).host; } catch { throw new Error(`Invalid URL: ${value}`); } } //#endregion //#region src/schemas/common.schema.ts const MAX_STRING_LENGTH = 5e3; /** * Helper to create a required, non-empty string field * Note: Returns the base ZodString so additional methods can be chained */ const zString = (fieldName, { min = 1, max = 0 } = {}) => z.string(`${fieldName} must be a string`).min(min, `${fieldName} cannot be empty`).max(max > 0 ? max : MAX_STRING_LENGTH, `${fieldName} must be less than ${max} characters`).transform((s) => s.trim()); /** * Helper to create a URL string field */ const zHttpUrl = (fieldName) => z.httpUrl(`${fieldName} must be a valid URL`); /** * Helper to create a positive integer field */ const zPositiveInteger = (fieldName) => z.int(`${fieldName} must be an integer`).positive(`${fieldName} must be positive`).nullable(); const zNonEmptyArray = (schema, fieldName) => z.array(schema, `${fieldName} items must be an array`).min(1, `${fieldName} group must have at least one item`); //#endregion //#region src/schemas/recipe.schema.ts /** * Current schema version for recipe objects. * Increment this when making breaking changes to the schema. * * Version history: * - 1.0.0: Initial schema version */ const RECIPE_SCHEMA_VERSION = "1.0.0"; /** * Schema for a parsed ingredient from the parse-ingredient library. * This represents the structured data extracted from an ingredient string. * @see https://github.com/jakeboone02/parse-ingredient */ const ParsedIngredientSchema = z.object({ quantity: z.number().nullable(), quantity2: z.number().nullable(), unitOfMeasureID: z.string().nullable(), unitOfMeasure: z.string().nullable(), description: z.string(), isGroupHeader: z.boolean() }); /** * Schema for a single ingredient item */ const IngredientItemSchema = z.object({ value: zString("Ingredient value"), parsed: ParsedIngredientSchema.optional().nullable() }); /** * Schema for a group of ingredients */ const IngredientGroupSchema = z.object({ name: zString("Ingredient group name").nullable(), items: zNonEmptyArray(IngredientItemSchema, "Ingredient") }); /** * Schema for all recipe ingredients * Must have at least one group with at least one ingredient */ const IngredientsSchema = z.array(IngredientGroupSchema, "Ingredients must be an array").min(1, "Recipe must have at least one ingredient group"); /** * Schema for a single instruction step */ const InstructionItemSchema = z.object({ value: zString("Instruction value") }); /** * Schema for a group of instruction steps */ const InstructionGroupSchema = z.object({ name: zString("Instruction group name").nullable(), items: zNonEmptyArray(InstructionItemSchema, "Instruction") }); /** * Schema for all recipe instructions * Must have at least one group with at least one step */ const InstructionsSchema = z.array(InstructionGroupSchema, "Instructions must be an array").min(1, "Recipe must have at least one instruction group"); /** * Schema for a link object */ const LinkSchema = z.object({ href: zHttpUrl("Link href"), text: zString("Link text") }); /** * Base RecipeObject schema without cross-field validations. * Use this schema when you need to extend the recipe object with custom fields. * * @example * ```ts * import { RecipeObjectBaseSchema, applyRecipeValidations } from 'recipe-scrapers-js' * * const MyCustomRecipeSchema = RecipeObjectBaseSchema.extend({ * customField: z.string(), * }) * * // Apply the standard recipe validations * const MyValidatedRecipeSchema = applyRecipeValidations(MyCustomRecipeSchema) * ``` */ const RecipeObjectBaseSchema = z.object({ schemaVersion: z.literal(RECIPE_SCHEMA_VERSION).default(RECIPE_SCHEMA_VERSION).describe("Schema version for recipe data migrations"), host: z.hostname("Host must be a valid hostname"), title: zString("Title", { max: 500 }), author: zString("Author", { max: 255 }), ingredients: IngredientsSchema, instructions: InstructionsSchema, canonicalUrl: zHttpUrl("Canonical URL"), image: zHttpUrl("Image"), totalTime: zPositiveInteger("Total time"), cookTime: zPositiveInteger("Cook time"), prepTime: zPositiveInteger("Prep time"), ratings: z.number("Ratings must be a number").min(0, "Ratings must be at least 0").max(5, "Ratings must be at most 5").default(0), ratingsCount: z.int("Ratings count must be an integer").nonnegative("Ratings count must be non-negative").default(0), yields: zString("Yields"), description: zString("Description"), language: zString("Language", { min: 2 }).optional().default("en"), siteName: zString("Site name").nullable(), cookingMethod: zString("Cooking method").nullable(), category: z.array(zString("Category item"), "Category must be an array").default([]), cuisine: z.array(zString("Cuisine item"), "Cuisine must be an array").default([]), keywords: z.array(zString("Keyword item"), "Keywords must be an array").default([]), dietaryRestrictions: z.array(zString("Dietary restriction item"), "Dietary restrictions must be an array").default([]), equipment: z.array(zString("Equipment item"), "Equipment must be an array").default([]), links: z.array(LinkSchema, "Links must be an array").optional(), nutrients: z.record(z.string(), z.string(), "Nutrients must be an object").default({}), reviews: z.record(z.string(), z.string(), "Reviews must be an object").default({}) }); /** * Applies recipe-specific transformations and validations to a schema. * Use this when extending RecipeObjectBaseSchema with custom fields. * * @param schema - A Zod object schema that includes * all RecipeObjectBaseSchema fields * @returns A schema with transforms and field validations applied * * @example * ```ts * const CustomSchema = RecipeObjectBaseSchema.extend({ * tags: z.array(z.string()), * }) * * const ValidatedCustomSchema = applyRecipeValidations(CustomSchema) * ``` */ function applyRecipeValidations(schema) { return schema.transform((data) => { if (!data.totalTime && !isNull(data.cookTime) && !isNull(data.prepTime)) data.totalTime = data.cookTime + data.prepTime; return data; }).refine(({ totalTime, cookTime, prepTime }) => { if (!isNull(totalTime) && !isNull(cookTime) && !isNull(prepTime)) return totalTime >= cookTime + prepTime; return true; }, { message: "Total time should be at least the sum of cook time and prep time", path: ["totalTime"] }).refine((data) => { return data.ratings === 0 || data.ratingsCount > 0; }, { message: "Ratings count should be greater than 0 when ratings exist", path: ["ratingsCount"] }); } /** * Strict RecipeObject schema with all validations enforced. * This is the standard schema used by recipe scrapers. * * For custom extensions, use RecipeObjectBaseSchema.extend() and then * apply validations with applyRecipeValidations(). */ const RecipeObjectSchema = applyRecipeValidations(RecipeObjectBaseSchema); //#endregion //#region src/exceptions/index.ts var ExtractorNotFoundException = class extends Error { constructor(field) { super(`No extractor found for field: ${field}`); this.name = "ExtractorNotFoundException"; } }; var NotImplementedException = class extends Error { constructor(method) { super(`Method should be implemented: ${method}`); this.name = "NotImplementedException"; } }; var UnsupportedFieldException = class extends Error { constructor(field) { super(`Extraction not supported for field: ${field}`); this.name = "UnsupportedFieldException"; } }; var ExtractionFailedException = class extends Error { constructor(field, value) { const msg = isDefined(value) ? `Invalid value for "${field}": ${String(value)}` : `No value found for "${field}"`; super(msg); this.name = "ExtractionFailedException"; } }; var NoIngredientsFoundException = class extends ExtractionFailedException { constructor() { super("ingredients"); this.name = "NoIngredientsFoundException"; } }; //#endregion //#region src/logger.ts let LogLevel = /* @__PURE__ */ function(LogLevel$1) { LogLevel$1[LogLevel$1["VERBOSE"] = 0] = "VERBOSE"; LogLevel$1[LogLevel$1["DEBUG"] = 1] = "DEBUG"; LogLevel$1[LogLevel$1["INFO"] = 2] = "INFO"; LogLevel$1[LogLevel$1["WARN"] = 3] = "WARN"; LogLevel$1[LogLevel$1["ERROR"] = 4] = "ERROR"; return LogLevel$1; }({}); var Logger = class { constructor(context, logLevel = LogLevel.WARN) { this.context = context; this.logLevel = logLevel; } verbose(...args) { if (this.logLevel > LogLevel.VERBOSE) return; console.log(`[VERBOSE][${this.context}]`, ...args); } debug(...args) { if (this.logLevel > LogLevel.DEBUG) return; console.debug(`[DEBUG][${this.context}]`, ...args); } log(...args) { if (this.logLevel > LogLevel.INFO) return; console.log(`[INFO][${this.context}]`, ...args); } info(...args) { if (this.logLevel > LogLevel.INFO) return; console.info(`[INFO][${this.context}]`, ...args); } warn(...args) { if (this.logLevel > LogLevel.WARN) return; console.warn(`[WARN][${this.context}]`, ...args); } error(...args) { console.error(`[ERROR][${this.context}]`, ...args); } }; //#endregion //#region src/plugin-manager.ts var PluginManager = class { extractorPlugins; postProcessorPlugins; constructor(baseExtractors, basePostProcessors, extraExtractors = [], extraPostProcessors = []) { this.extractorPlugins = [...baseExtractors, ...extraExtractors].sort((a, b) => b.priority - a.priority); this.postProcessorPlugins = [...basePostProcessors, ...extraPostProcessors].sort((a, b) => b.priority - a.priority); } getExtractors() { return this.extractorPlugins; } getPostProcessors() { return this.postProcessorPlugins; } }; //#endregion //#region src/abstract-postprocessor-plugin.ts var PostProcessorPlugin = class {}; //#endregion //#region src/utils/parsing.ts /******************************************************************************* * Utility functions for common parsing tasks ******************************************************************************/ function normalizeString(str) { return str?.trim().replace(/\s+/g, " ").replace(/\s+,/g, ",") ?? ""; } function splitToList(value, separator) { if (!value) return []; const items = []; for (const item of value.split(separator)) { const str = normalizeString(item); if (str) items.push(str); } return items; } /** * @TODO Implement [Temporal.Duration](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Temporal/Duration) once it lands. */ function parseMinutes(value) { const totalSeconds = toSeconds(parse(value)); return Math.round(totalSeconds / 60); } //#endregion //#region src/utils/ingredients.ts const DEFAULT_GROUPING_SELECTORS = { wprm: { headingSelectors: [".wprm-recipe-ingredient-group h4", ".wprm-recipe-group-name"], itemSelectors: [".wprm-recipe-ingredient", ".wprm-recipe-ingredients li"] }, tasty: { headingSelectors: [".tasty-recipes-ingredients-body p strong", ".tasty-recipes-ingredients h4"], itemSelectors: [".tasty-recipes-ingredients-body ul li", ".tasty-recipes-ingredients ul li"] } }; /** * Creates an IngredientItem. */ function createIngredientItem(value) { return { value }; } /** * Creates an IngredientGroup. */ function createIngredientGroup(name, items = []) { return { name, items }; } /** * Type guard to check if value is an IngredientItem. */ function isIngredientItem(value) { return isPlainObject(value) && "value" in value && isString(value.value); } /** * Type guard to check if value is an IngredientGroup. */ function isIngredientGroup(value) { return isPlainObject(value) && "name" in value && "items" in value && Array.isArray(value.items) && value.items.every(isIngredientItem); } /** * Type guard to check if value is an Ingredients array. */ function isIngredients(value) { return Array.isArray(value) && value.every(isIngredientGroup); } /** * Extracts the flat list of ingredient values from an Ingredients array. * Useful when scrapers need to re-group ingredients using HTML structure. */ function flattenIngredients(ingredients) { return ingredients.flatMap((group) => group.items.map((item) => item.value)); } /** * Converts an array of strings to an Ingredients array with a single * default group. */ function stringsToIngredients(values, groupName = null) { return [createIngredientGroup(groupName, values.map(createIngredientItem))]; } function scoreSentenceSimilarity(first, second) { if (first === second) return 1; if (first.length < 2 || second.length < 2) return 0; const bigrams = (s) => new Set(Array.from({ length: s.length - 1 }, (_, i) => s.slice(i, i + 2))); const firstBigrams = bigrams(first); const secondBigrams = bigrams(second); return 2 * [...firstBigrams].filter((b) => secondBigrams.has(b)).length / (firstBigrams.size + secondBigrams.size); } function bestMatch(testString, targetStrings) { if (targetStrings.length === 0) throw new Error("targetStrings cannot be empty"); const scores = targetStrings.map((t) => scoreSentenceSimilarity(testString, t)); let bestIndex = 0; let bestScore = scores[0]; for (let i = 1; i < scores.length; i++) if (scores[i] > bestScore) { bestScore = scores[i]; bestIndex = i; } return targetStrings[bestIndex]; } function findSelectors($, initialHeading, initialItem) { if (initialHeading && initialItem) { if ($(initialHeading).length && $(initialItem).length) return [initialHeading, initialItem]; return null; } const values = Object.values(DEFAULT_GROUPING_SELECTORS); for (const { headingSelectors, itemSelectors } of values) for (const heading of headingSelectors) for (const item of itemSelectors) if ($(heading).length && $(item).length) return [heading, item]; return null; } /** * Groups ingredients based on the provided selectors. * If no selectors are provided, it will try to find the best matching * selectors from the default grouping selectors. * * @param $ Cheerio instance * @param ingredientValues Array of ingredient strings to group * @param headingSelector Optional custom heading selector * @param itemSelector Optional custom item selector * @returns Ingredients array with groups */ function groupIngredients($, ingredientValues, headingSelector, itemSelector) { const selectors = findSelectors($, headingSelector, itemSelector); if (!selectors) return stringsToIngredients(ingredientValues); const [groupNameSelector, ingredientSelector] = selectors; if (new Set($(ingredientSelector).toArray().map((el) => $(el).text().trim()).filter(Boolean)).size !== ingredientValues.length) return stringsToIngredients(ingredientValues); const groupings = /* @__PURE__ */ new Map(); let currentHeading = null; const elements = $(`${groupNameSelector}, ${ingredientSelector}`).toArray(); for (const el of elements) { const $el = $(el); if ($el.is(groupNameSelector)) { currentHeading = normalizeString($el.text()).replace(/:$/, "") || null; if (!groupings.has(currentHeading)) groupings.set(currentHeading, []); } else if ($el.is(ingredientSelector)) { const text = normalizeString($el.text()); if (!text) continue; const matched = bestMatch(text, ingredientValues); const heading = currentHeading ?? null; if (!groupings.has(heading)) groupings.set(heading, []); groupings.get(heading)?.push(matched); } } const result = []; for (const [name, items] of groupings.entries()) result.push(createIngredientGroup(name, items.map(createIngredientItem))); return result; } //#endregion //#region src/utils/instructions.ts /** * List of possible headings to remove from instructions. */ const INSTRUCTION_HEADINGS = [ "Preparation", "Directions", "Instructions", "Method", "Steps" ]; /** * Creates an InstructionItem. */ function createInstructionItem(value) { return { value }; } /** * Creates an InstructionGroup. */ function createInstructionGroup(name, items = []) { return { name, items }; } /** * Type guard to check if value is an InstructionItem. */ function isInstructionItem(value) { return isPlainObject(value) && "value" in value && isString(value.value); } /** * Type guard to check if value is an InstructionGroup. */ function isInstructionGroup(value) { return isPlainObject(value) && "name" in value && "items" in value && Array.isArray(value.items) && value.items.every(isInstructionItem); } /** * Type guard to check if value is an Instructions array. */ function isInstructions(value) { return Array.isArray(value) && value.every(isInstructionGroup); } /** * Removes any heading from the start of the instructions string. */ function removeInstructionHeading(value) { for (const heading of INSTRUCTION_HEADINGS) { const regex = new RegExp(`^\\s*${heading}\\s*:?\\s*`, "i"); if (regex.test(value)) return value.replace(regex, ""); } return value; } /** * Splits a recipe instructions string into an array of steps. * Removes known headings and trims whitespace. */ function splitInstructions(value) { if (!value) return []; const cleaned = removeInstructionHeading(value).trim(); let steps = splitToList(cleaned, /\n\s*\n+/); if (steps.length === 1) steps = splitToList(cleaned, /(?<=\.)\s+(?=[A-Z])/); return steps; } //#endregion //#region src/plugins/html-stripper.processor.ts var HtmlStripperPlugin = class extends PostProcessorPlugin { name = "HtmlStripper"; priority = 100; fieldsToProcess = [ "title", "description", "instructions", "ingredients" ]; shouldProcess(field) { return this.fieldsToProcess.includes(field); } process(field, value) { if (!this.shouldProcess(field)) return value; if (isString(value)) return this.stripHtml(value); if (field === "instructions" && isInstructions(value)) return this.processInstructions(value); if (field === "ingredients" && isIngredients(value)) return this.processIngredients(value); return value; } processIngredients(ingredients) { return ingredients.map((group) => ({ name: group.name === null ? null : this.stripHtml(group.name), items: group.items.map((item) => ({ value: this.stripHtml(item.value) })) })); } processInstructions(instructions) { return instructions.map((group) => ({ name: group.name === null ? null : this.stripHtml(group.name), items: group.items.map((item) => ({ value: this.stripHtml(item.value) })) })); } stripHtml(html) { return html.replace(/<[^>]*>/g, "").replace(/&amp;/g, "&").replace(/&nbsp;/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, "\"").replace(/&#34;/g, "\"").replace(/&#39;/g, "'").trim(); } }; //#endregion //#region src/plugins/ingredient-parser.processor.ts /** * Post-processor plugin that parses ingredient strings into structured data. * Uses the parse-ingredient library to extract quantity, unit, and description. * * @see https://github.com/jakeboone02/parse-ingredient */ var IngredientParserPlugin = class extends PostProcessorPlugin { name = "IngredientParser"; priority = 50; constructor(options = {}) { super(); this.options = options; } shouldProcess(field) { return field === "ingredients"; } process(field, value) { if (!this.shouldProcess(field)) return value; if (isIngredients(value)) return this.processIngredients(value); return value; } processIngredients(ingredients) { return ingredients.map((group) => ({ name: group.name, items: group.items.map((item) => this.parseItem(item)) })); } parseItem(item) { const parsedIngredient = parseIngredient(item.value, this.options)[0] ?? null; return { value: item.value, parsed: parsedIngredient }; } }; //#endregion //#region src/abstract-plugin.ts var AbstractPlugin = class { constructor($) { this.$ = $; } }; //#endregion //#region src/abstract-extractor-plugin.ts var ExtractorPlugin = class extends AbstractPlugin {}; //#endregion //#region src/plugins/opengraph.extractor.ts var OpenGraphException = class extends ExtractionFailedException { constructor(name) { super(name); this.name = "OpenGraphException"; } }; var OpenGraphPlugin = class OpenGraphPlugin extends ExtractorPlugin { name = OpenGraphPlugin.name; priority = 60; extractors = { image: this.image.bind(this), siteName: this.siteName.bind(this) }; supports(field) { return Object.keys(this.extractors).includes(field); } extract(field) { const extractor = this.extractors[field]; if (!extractor) throw new ExtractorNotFoundException(field); return extractor(); } siteName() { const meta = this.$("meta[property=\"og:site_name\"]").attr("content") || this.$("meta[name=\"og:site_name\"]").attr("content"); if (!meta) throw new OpenGraphException("siteName"); return meta; } image() { const image = this.$("meta[property=\"og:image\"][content]").attr("content"); if (!image?.startsWith("http")) throw new OpenGraphException("image"); return image; } }; //#endregion //#region src/utils/microdata.ts const addProperty = (obj, key, value) => { if (obj[key] === void 0) obj[key] = value; else if (Array.isArray(obj[key])) obj[key].push(value); else obj[key] = [obj[key], value]; }; const extractValueFromElement = (element) => { if (element.is("meta")) return element.attr("content"); if (element.is("time")) return element.attr("datetime") || element.text().trim(); if (element.is("img")) return element.attr("src"); if (element.is("a")) return element.attr("href"); return element.text().trim(); }; const extractSchemaType = (itemType) => { return itemType.match(/schema\.org\/(\w+)/)?.[1]; }; /** * Extracts microdata from HTML elements using itemtype and itemprop attributes * * @param $ - Cheerio instance * @param selector - Selector to find elements with microdata * @returns Array of extracted microdata objects */ function extractMicrodata($, selector) { const results = []; $(selector).each((_, el) => { const $element = $(el); const itemType = $element.attr("itemtype"); const rootObject = {}; if (itemType) { const schemaType = extractSchemaType(itemType); if (schemaType) rootObject["@type"] = schemaType; } const allProps = $element.find("[itemprop]").addBack("[itemprop]"); const nestedItemTypes = $element.find("[itemtype]"); allProps.filter((_$1, propEl) => { const $prop = $(propEl); if ($prop.attr("itemtype")) return true; return !nestedItemTypes.toArray().some((nestedEl) => $(nestedEl).find($prop).length > 0); }).each((_$1, propEl) => { const $prop = $(propEl); const propName = $prop.attr("itemprop"); if (!propName) return; let propValue; const nestedItemType = $prop.attr("itemtype"); if (nestedItemType) { const nestedObject = {}; const nestedSchemaType = extractSchemaType(nestedItemType); if (nestedSchemaType) nestedObject["@type"] = nestedSchemaType; $prop.find("[itemprop]").each((_$2, nestedEl) => { const $nested = $(nestedEl); const nestedProp = $nested.attr("itemprop"); if (!nestedProp) return; const nestedValue = extractValueFromElement($nested); if (nestedValue && nestedValue !== "") addProperty(nestedObject, nestedProp, nestedValue); }); propValue = nestedObject; } else propValue = extractValueFromElement($prop); if (propValue !== void 0 && propValue !== "") addProperty(rootObject, propName, propValue); }); if (Object.keys(rootObject).length > 1 || Object.keys(rootObject).length === 1 && !rootObject["@type"]) results.push(rootObject); }); return results; } /** * Extracts Recipe microdata specifically * * @param $ - Cheerio instance * @returns Array of recipe microdata objects */ function extractRecipeMicrodata($) { return extractMicrodata($, "[itemtype*=\"schema.org/Recipe\"], [itemtype*=\"Recipe\"]"); } //#endregion //#region src/utils/parse-yields.ts const SERVE_REGEX_NUMBER = /(?:\D*(?<items>\d+(?:\.\d*)?)\D*)/; const SERVE_REGEX_ITEMS = /\bsandwiches\b|\btacquitos\b|\bmakes\b|\bcups\b|\bappetizer\b|\bporzioni\b|\bcookies\b|\b(large |small )?buns\b/gi; const SERVE_REGEX_TO = /\d+(\s+to\s+|-)\d+/gi; const RECIPE_YIELD_TYPES = [ ["dozen", "dozen"], ["batch", "batches"], ["cake", "cakes"], ["sandwich", "sandwiches"], ["bun", "buns"], ["cookie", "cookies"], ["muffin", "muffins"], ["cupcake", "cupcakes"], ["loaf", "loaves"], ["pie", "pies"], ["cup", "cups"], ["pint", "pints"], ["gallon", "gallons"], ["ounce", "ounces"], ["pound", "pounds"], ["gram", "grams"], ["liter", "liters"], ["piece", "pieces"], ["layer", "layers"], ["scoop", "scoops"], ["bar", "bars"], ["patty", "patties"], ["hamburger bun", "hamburger buns"], ["pancake", "pancakes"], ["item", "items"] ]; /** * Returns a string of servings or items. If the recipe is for a number of * items (not servings), it returns "x item(s)" where x is the quantity. * This function handles cases where the yield is in dozens, * such as "4 dozen cookies", returning "4 dozen" instead of "4 servings". * Additionally accommodates yields specified in batches * (e.g., "2 batches of brownies"), returning the yield as stated. * * @param value The yield string from the recipe * @returns The number of servings, items, dozen, batches, etc... */ function parseYields(element) { if (!element) throw new Error("Element is required"); let serveText = element; if (SERVE_REGEX_TO.test(serveText)) { const splitMatch = serveText.match(SERVE_REGEX_TO); if (splitMatch && splitMatch.index !== void 0) serveText = serveText.slice(splitMatch.index + splitMatch[0].length).trim(); } const matched = serveText.match(SERVE_REGEX_NUMBER)?.groups?.items || "0"; const serveTextLower = serveText.toLowerCase(); let bestMatch$1 = null; let bestMatchLength = 0; for (const [singular, plural$1] of RECIPE_YIELD_TYPES) if (serveTextLower.includes(singular) || serveTextLower.includes(plural$1)) { const matchLength = serveTextLower.includes(singular) ? singular.length : plural$1.length; if (matchLength > bestMatchLength) { bestMatchLength = matchLength; bestMatch$1 = `${matched} ${Number.parseFloat(matched) === 1 ? singular : plural$1}`; } } if (bestMatch$1) { const parenMatch = serveText.match(/\(.*\)/); if (parenMatch) bestMatch$1 += ` ${parenMatch[0]}`; return bestMatch$1; } const plural = Number.parseFloat(matched) > 1 || Number.parseFloat(matched) === 0 ? "s" : ""; if (SERVE_REGEX_ITEMS.test(serveText)) return `${matched} item${plural}`; return `${matched} serving${plural}`; } //#endregion //#region src/plugins/schema-org.extractor/type-predicates.ts function hasId(obj) { return "@id" in obj && typeof obj["@id"] === "string"; } function isGraphType(obj) { return isPlainObject(obj) && "@graph" in obj && Array.isArray(obj["@graph"]); } function isBaseType(obj) { return isPlainObject(obj) && "@type" in obj && (isString(obj["@type"]) || Array.isArray(obj["@type"])); } function isSchemaOrgData(obj) { return isGraphType(obj) || isBaseType(obj); } function isThingType(obj, type) { if (!isBaseType(obj)) return false; return (Array.isArray(obj["@type"]) ? obj["@type"][0] : obj["@type"]) === type; } function isAggregateRating(obj) { return isThingType(obj, "AggregateRating"); } function isHowToSection(obj) { return isThingType(obj, "HowToSection"); } function isHowToStep(obj) { return isThingType(obj, "HowToStep"); } function isOrganization(obj) { return isThingType(obj, "Organization"); } function isPerson(obj) { return isThingType(obj, "Person"); } function isRecipe(obj) { return isThingType(obj, "Recipe"); } function isWebPage(obj) { return isThingType(obj, "WebPage"); } function isWebSite(obj) { return isThingType(obj, "WebSite"); } //#endregion //#region src/plugins/schema-org.extractor/index.ts var SchemaOrgException = class extends ExtractionFailedException { constructor(field, value) { super(field, value); this.name = "SchemaOrgException"; } }; var SchemaOrgPlugin = class SchemaOrgPlugin extends ExtractorPlugin { name = SchemaOrgPlugin.name; priority = 90; logger; schemaData = []; recipe = { "@type": "Recipe" }; people = {}; ratingsData = {}; websiteName = null; extractors = { siteName: this.siteName.bind(this), language: this.language.bind(this), title: this.title.bind(this), author: this.author.bind(this), description: this.description.bind(this), image: this.image.bind(this), ingredients: this.ingredients.bind(this), instructions: this.instructions.bind(this), category: this.category.bind(this), yields: this.yields.bind(this), totalTime: this.totalTime.bind(this), cookTime: this.cookTime.bind(this), prepTime: this.prepTime.bind(this), cuisine: this.cuisine.bind(this), cookingMethod: this.cookingMethod.bind(this), ratings: this.ratings.bind(this), ratingsCount: this.ratingsCount.bind(this), nutrients: this.nutrients.bind(this), keywords: this.keywords.bind(this), dietaryRestrictions: this.dietaryRestrictions.bind(this) }; constructor($, logLevel) { super($); this.logger = new Logger(SchemaOrgPlugin.name, logLevel); this.extractJsonLdData(); this.extractMicrodataData(); this.processSchemaData(); } supports(field) { return Object.keys(this.extractors).includes(field); } extract(field) { const extractor = this.extractors[field]; if (!isFunction(extractor)) throw new UnsupportedFieldException(field); return extractor(); } /** * Extracts structured JSON-LD data from the page. */ extractJsonLdData() { this.$("script[type=\"application/ld+json\"]").each((_, el) => { try { const json = this.$(el).html()?.trim(); if (json) { const data = JSON.parse(json); if (Array.isArray(data)) { for (const item of data) if (isSchemaOrgData(item)) this.schemaData.push(item); } else if (isSchemaOrgData(data)) this.schemaData.push(data); } } catch (error) { this.logger.warn("Failed to parse JSON-LD", error); } }); } /** * Extracts microdata from the page. */ extractMicrodataData() { const microdataObjects = extractRecipeMicrodata(this.$); for (const obj of microdataObjects) this.schemaData.push(obj); } pickFromObject(obj, props) { if (!isPlainObject(obj)) return void 0; for (const prop of props) if (isString(obj[prop])) return obj[prop]; } getSchemaTextValue(value, props = [ "textValue", "name", "title", "@id" ]) { let text; if (isString(value)) text = value; else if (isNumber(value)) text = value.toString(); else if (Array.isArray(value)) text = this.getSchemaTextValue(value[0], props); else text = this.pickFromObject(value, props); return normalizeString(text); } schemaValueToList(value) { let list = []; if (Array.isArray(value)) for (const item of value) { const itemValue = this.getSchemaTextValue(item); if (itemValue) list.push(itemValue); } else if (isString(value)) list = splitToList(this.getSchemaTextValue(value), ","); return new Set(list); } findEntity(item, schemaType) { if (isThingType(item, schemaType)) return item; if (isGraphType(item)) { for (const graphItem of item["@graph"]) if (isThingType(graphItem, schemaType)) return graphItem; } return null; } getIdOrUrl(obj) { return hasId(obj) ? obj["@id"] : isString(obj.url) ? obj.url : null; } processSchemaData() { for (const item of this.schemaData) if (isGraphType(item)) for (const graphItem of item["@graph"]) this.processSchemaItem(graphItem); else this.processSchemaItem(item); } processSchemaItem(obj) { if (isRecipe(obj)) return this.processRecipe(obj); return this.processNonRecipeThing(obj); } processRecipe(obj) { const recipe = this.findEntity(obj, "Recipe"); this.recipe = { ...this.recipe, ...recipe }; } processNonRecipeThing(obj) { if (isWebSite(obj)) this.websiteName = this.getSchemaTextValue(obj); if (isWebPage(obj) && isBaseType(obj.mainEntity)) this.processSchemaItem(obj.mainEntity); if (isPerson(obj)) { const key = this.getIdOrUrl(obj); if (key) this.people[key] = obj; } if (isAggregateRating(obj)) { const key = obj["@id"]; if (key) this.ratingsData[key] = obj; } } parseDurationField(key) { const value = this.recipe[key]; if (!value) return null; if (isNumber(value)) { this.logger.warn(`Duration field "${key}" is a number: ${value}`); return value; } if (isString(value)) return parseMinutes(value); if (isBaseType(value) && "maxValue" in value) return parseMinutes(this.getSchemaTextValue(value.maxValue)); return null; } parseInstructions(value) { if (isString(value)) return [createInstructionGroup(null, splitInstructions(value).map(createInstructionItem))]; const instructions = Array.isArray(value) ? value.flat() : [value].flat(); const groups = []; let currentGroup = { name: null, items: [] }; for (const item of instructions) { const name = this.getSchemaTextValue(item, ["name"]); const text = this.getSchemaTextValue(item, ["text"]); if (isString(item)) currentGroup.items.push(normalizeString(item)); else if (isHowToStep(item)) { if (name && text && !text.startsWith(name.replace(/\.$/, ""))) currentGroup.items.push(name); if (text) currentGroup.items.push(text); } else if (isHowToSection(item)) { if (currentGroup.items.length > 0) groups.push(createInstructionGroup(currentGroup.name, currentGroup.items.filter(Boolean).map(createInstructionItem))); currentGroup = { name: name || null, items: [] }; if (item.itemListElement) { const nestedResult = this.parseInstructions(item.itemListElement); for (const nestedGroup of nestedResult) currentGroup.items.push(...nestedGroup.items.map((i) => i.value)); } } else if (text) currentGroup.items.push(text); } if (currentGroup.items.length > 0) groups.push(createInstructionGroup(currentGroup.name, currentGroup.items.filter(Boolean).map(createInstructionItem))); return groups; } /***************************************************************************** * Extractor methods ****************************************************************************/ siteName() { if (isOrganization(this.recipe.publisher)) { const publisherName = this.getSchemaTextValue(this.recipe.publisher, ["name", "alternateName"]); if (publisherName) return publisherName; } if (!this.websiteName) throw new SchemaOrgException("siteName"); return this.websiteName; } language() { const language = this.getSchemaTextValue(this.recipe.inLanguage); if (!language) throw new SchemaOrgException("language"); return language; } title() { const title = this.getSchemaTextValue(this.recipe.name); if (!title) throw new SchemaOrgException("title"); return title; } author() { let author = this.recipe.author; let authorName; if (Array.isArray(author) && author.length > 0) author = author[0]; if (isBaseType(author)) { const key = this.getIdOrUrl(author); if (key && this.people[key]) author = this.people[key]; authorName = author.name?.toString(); } authorName = normalizeString(authorName); if (!authorName) throw new SchemaOrgException("author"); return authorName; } description() { const desc = this.getSchemaTextValue(this.recipe.description); if (!desc) throw new SchemaOrgException("description"); return desc; } image() { const image = this.getSchemaTextValue(this.recipe.image, ["url", "contentUrl"]); if (!image.startsWith("http")) throw new SchemaOrgException("image", image); return image; } ingredients() { const ingredients = this.recipe.recipeIngredient ?? this.recipe.ingredients ?? []; if (!Array.isArray(ingredients)) throw new SchemaOrgException("ingredients", ingredients); const flatIngredients = ingredients.flat(); const uniqueIngredients = /* @__PURE__ */ new Set(); for (const item of flatIngredients) { const ingredient = this.getSchemaTextValue(item); if (ingredient) uniqueIngredients.add(ingredient); } return stringsToIngredients([...uniqueIngredients]); } instructions() { const instructions = this.parseInstructions(this.recipe.recipeInstructions); if (instructions.length === 0) throw new SchemaOrgException("instructions"); return instructions; } category() { const category = this.recipe.recipeCategory; if (!category) throw new SchemaOrgException("category"); return this.schemaValueToList(category); } yields() { const yields = this.getSchemaTextValue(this.recipe.recipeYield ?? this.recipe.yield); if (!yields) throw new SchemaOrgException("yields", yields); return parseYields(yields); } totalTime() { const totalTime = this.parseDurationField("totalTime"); if (totalTime) return totalTime; const prepTime = this.parseDurationField("prepTime") ?? 0; const cookTime = this.parseDurationField("cookTime") ?? 0; if (prepTime || cookTime) return prepTime + cookTime; throw new SchemaOrgException("totalTime"); } cookTime() { return this.parseDurationField("cookTime"); } prepTime() { return this.parseDurationField("prepTime"); } cuisine() { const cuisine = this.recipe.recipeCuisine; if (!cuisine) throw new SchemaOrgException("cuisine"); return this.schemaValueToList(cuisine); } cookingMethod() { const cookingMethod = this.getSchemaTextValue(this.recipe.cookingMethod); if (!cookingMethod) throw new SchemaOrgException("cookingMethod"); return cookingMethod; } ratings() { let ratings = this.recipe.aggregateRating ?? this.findEntity(this.recipe, "AggregateRating"); let ratingValue; if (isAggregateRating(ratings)) { const ratingId = ratings["@id"]; if (ratingId && this.ratingsData[ratingId]) ratings = this.ratingsData[ratingId]; ratingValue = this.getSchemaTextValue(ratings.ratingValue); } if (!ratingValue) throw new SchemaOrgException("ratings"); return Math.round(Number.parseFloat(ratingValue) * 100) / 100; } ratingsCount() { let ratings = this.recipe.aggregateRating ?? this.findEntity(this.recipe, "AggregateRating"); let ratingsCount; if (isAggregateRating(ratings)) { const ratingId = ratings["@id"]; if (ratingId && this.ratingsData[ratingId]) ratings = this.ratingsData[ratingId]; ratingsCount = this.getSchemaTextValue(ratings.ratingCount) || this.getSchemaTextValue(ratings.reviewCount); } if (!ratingsCount) throw new SchemaOrgException("ratingsCount"); const count = Number.parseFloat(ratingsCount); return count !== 0 ? Math.floor(count) : 0; } nutrients() { const nutrients = this.recipe.nutrition; if (!isPlainObject(nutrients)) throw new SchemaOrgException("nutrients", nutrients); const cleanedNutrients = /* @__PURE__ */ new Map(); for (const [key, value] of Object.entries(nutrients)) { if (!key || key.startsWith("@") || !value) continue; cleanedNutrients.set(key, this.getSchemaTextValue(value)); } return cleanedNutrients; } keywords() { const keywords = this.recipe.keywords; if (!keywords) throw new SchemaOrgException("keywords"); return this.schemaValueToList(keywords); } dietaryRestrictions() { const dietaryRestrictions = this.recipe.suitableForDiet; if (!dietaryRestrictions) throw new SchemaOrgException("dietaryRestrictions"); const restrictionList = /* @__PURE__ */ new Set(); const list = this.schemaValueToList(dietaryRestrictions); for (const item of list) { const value = item.replace(/^https?:\/\/schema\.org\//, ""); if (value) restrictionList.add(value); } return restrictionList; } }; //#endregion //#region src/constants.ts const OPTIONAL_RECIPE_FIELD_DEFAULT_VALUES = { siteName: null, category: /* @__PURE__ */ new Set(), cookTime: null, prepTime: null, totalTime: null, cuisine: /* @__PURE__ */ new Set(), cookingMethod: null, ratings: 0, ratingsCount: 0, equipment: /* @__PURE__ */ new Set(), reviews: /* @__PURE__ */ new Map(), nutrients: /* @__PURE__ */ new Map(), dietaryRestrictions: /* @__PURE__ */ new Set(), keywords: /* @__PURE__ */ new Set() }; //#endregion //#region src/recipe-extractor.ts var RecipeExtractor = class RecipeExtractor { logger; constructor(plugins, scraperName, options = {}) { this.plugins = plugins; this.scraperName = scraperName; this.options = options; this.logger = new Logger(this.getContext(), this.options.logLevel); this.plugins.sort((a, b) => b.priority - a.priority); } getContext(context) { return `${this.scraperName}.${RecipeExtractor.name}${context ? `.${context}` : ""}`; } async extract(field, extractor) { let result; this.logger.debug(`Extracting field: ${field}`); for (const plugin of this.plugins) { const pluginLogger = new Logger(this.getContext(plugin.name), this.options.logLevel); if (plugin.supports(field) && !isDefined(result)) try { result = await plugin.extract(field); } catch (err) { if (err instanceof ExtractionFailedException) pluginLogger.verbose(err.message); else pluginLogger.error(err); } else pluginLogger.verbose(`Field is not supported: ${field}`); } if (extractor) { this.logger.debug(`Using site-specific extractor for: ${field}`); this.logger.verbose("Current result: ", result); try { result = await extractor(result); this.logger.verbose(`Site result for ${field}: `, result); } catch (err) { this.logger.error(err); } } if (!result && field in OPTIONAL_RECIPE_FIELD_DEFAULT_VALUES) { this.logger.debug(`Using default value for: ${field}`); result = OPTIONAL_RECIPE_FIELD_DEFAULT_VALUES[field]; } if (isDefined(result)) return result; throw new ExtractorNotFoundException(field); } }; //#endregion //#region src/abstract-scraper.ts var AbstractScraper = class { logger; pluginManager; recipeExtractor; $; recipeData = null; constructor(html, url, options = {}) { this.html = html; this.url = url; this.options = options; const { extraExtractors = [], extraPostProcessors = [], logLevel = LogLevel.WARN, parseIngredients = false } = options; this.logger = new Logger(this.constructor.name, logLevel); this.$ = cheerio.load(html); const baseExtractors = [new OpenGraphPlugin(this.$), new SchemaOrgPlugin(this.$, logLevel)]; const basePostProcessors = [new HtmlStripperPlugin()]; if (parseIngredients) { const parserOptions = isPlainObject(parseIngredients) ? parseIngredients : {}; basePostProcessors.push(new IngredientParserPlugin(parserOptions)); } this.pluginManager = new PluginManager(baseExtractors, basePostProcessors, extraExtractors, extraPostProcessors); this.recipeExtractor = new RecipeExtractor(this.pluginManager.getExtractors(), this.constructor.name, { logLevel }); } /** * Main extraction method - tries site-specific first, then plugins, * then applies post-processing. */ async extract(field) { let value = await this.recipeExtractor.extract(field, this.extractors[field]); for (const processor of this.pluginManager.getPostProcessors()) value = await processor.process(field, value); return value; } /** * Static method to get the host of the scraper. * This should be implemented by subclasses to return the specific host. */ static host() { throw new NotImplementedException("host"); } /***************************************************************************** * Default implementations for common fields that can be overridden * by subclasses. ****************************************************************************/ canonicalUrl() { const canonicalLink = this.$("link[rel=\"canonical\"]").attr("href"); const base = new URL(this.url.startsWith("http") ? this.url : `https://${this.url}`); return canonicalLink ? new URL(canonicalLink, base).href : base.href; } language() { const langAttr = this.$("html").attr("lang"); if (langAttr) return langAttr; const metaLang = this.$("meta[http-equiv=\"content-language\"]").attr("content"); if (metaLang) return metaLang.split(",")[0]; this.logger.warn("Could not determine language"); return "en"; } links() { if (!this.options.linksEnabled) return void 0; return this.$("a[href]").map((_, el) => { const href = this.$(el).attr("href"); if (!href?.startsWith("http")) return null; return { href, text: this.$(el).text().trim() }; }).get().filter(Boolean); } /** * Scrape's the recipe and caches the data. */ async scrape() { const instance = this.constructor; if (this.recipeData) return this.recipeData; this.recipeData = { author: await this.extract("author"), canonicalUrl: this.canonicalUrl(), category: await this.extract("category"), cookTime: await this.extract("cookTime"), cookingMethod: await this.extract("cookingMethod"), cuisine: await this.extract("cuisine"), description: await this.extract("description"), dietaryRestrictions: await this.extract("dietaryRestrictions"), equipment: await this.extract("equipment"), host: instance.host(), image: await this.extract("image"), ingredients: await this.extract("ingredients"), instructions: await this.extract("instructions"), keywords: await this.extract("keywords"), language: this.language(), links: this.links(), nutrients: await this.extract("nutrients"), prepTime: await this.extract("prepTime"), ratings: await this.extract("ratings"), ratingsCount: await this.extract("ratingsCount"), reviews: await this.extract("reviews"), siteName: await this.extract("siteName"), title: await this.extract("title"), totalTime: await this.extract("totalTime"), yields: await this.extract("yields") }; return this.recipeData; } /** * Converts the scraper's data into a JSON-serializable object. * Note: schemaVersion is added during validation by parse() or safeParse(). */ async toRecipeObject() { const { category, cuisine, dietaryRestrictions, equipment, keywords, nutrients, reviews, ...rest } = await this.scrape(); return { ...rest, category: Array.from(category), cuisine: Array.from(cuisine), dietaryRestrictions: Array.from(dietaryRestrictions), equipment: Array.from(equipment), keywords: Array.from(keywords), nutrients: Object.fromEntries(nutrients), reviews: Object.fromEntries(reviews) }; } /** * Get the Zod schema to use for validation. * Subclasses can override to provide custom schemas. */ getSchema() { return RecipeObjectSchema; } /** * Extract and validate recipe data. * Throws ZodError if validation fails. * * @returns Validated recipe object * @throws {ZodError} If validation fails */ async parse() { const raw = await this.toRecipeObject(); return this.getSchema().parse(raw); } /** * Extract and validate recipe data without throwing. * Returns a result object indicating success or failure. * * @returns Result object with either data or error */ async safeParse() { const raw = await this.toRecipeObject(); return this.getSchema().safeParse(raw); } }; //#endregion //#region src/scrapers/allrecipes.ts var AllRecipes = class extends AbstractScraper { static host() { return "allrecipes.com"; } extractors = {}; }; //#endregion //#region src/scrapers/americastestkitchen.ts const recipeIngredientItemSchema = z$1.object({ fields: z$1.object({ qty: z$1.string(), preText: z$1.string(), postText: z$1.string(), measurement: z$1.string().nullable(), pluralIngredient: z$1.boolean(), ingredient: z$1.object({ contentType: z$1.string(), fields: z$1.object({ title: z$1.string(), pluralTitle: z$1.string(), kind: z$1.string() }) }) }) }); const recipeIngredientGroupSchema = z$1.object({ fields: z$1.object({ title: z$1.string(), recipeIngredientItems: z$1.array(recipeIngredientItemSchema) }) }); const recipeInstructionSchema = z$1.object({ fields: z$1.object({ content: z$1.string() }) }); const recipeDataSchema = z$1.object({ totalCookTime: z$1.number(), recipeTimeNote: z$1.string().optional(), ingredientGroups: z$1.array(recipeIngredientGroupSchema), headnote: z$1.string().optional(), instructions: z$1.array(recipeInstructionSchema), metaData: z$1.object({ fields: z$1.object({ photo: z$1.object({ url: z$1.url() }) }) }) }); const pagePropsDataSchema = z$1.object({ props: z$1.object({ pageProps: z$1.object({ data: recipeDataSchema }) }) }); var AmericasTestKitchen = class extends AbstractScraper { data = null; static host() { return "americastestkitchen.com"; } extractors = { image: this.image.bind(this), ingredients: this.ingredients.bind(this), instructions: this.instructions.bind(this), siteName: this.siteName.bind(this) }; siteName() { return "America's Test Kitchen"; } image(prevValue) { const data = this.getRecipeData(); if (!data) { if (prevValue) return prevValue; throw new Error("Failed to extract image"); } return data.metaData.fields.photo.url; } ingredients(prevValue) { let ingredients = this.parseIngredients(); if (!ingredients) ingredients = this.parseHtmlIngredients(prevValue); if (!ingredients) throw new Error("Failed to extract ingredients"); return ingredients; } instructions(prevValue) { const data = this.getRecipeData(); if (!data) { if (prevValue) return prevValue; throw new Error("Failed to extract instructions"); } const { headnote } = data; const items = []; if (headnote) items.push(`Note: ${normalizeString(headnote)}`); for (const instruction of data.instructions) items.push(normalizeString(instruction.fields.content)); return [createInstructionGroup(null, items.map(createInstructionItem))]; } parseHtmlIngredients(