website-validator

import { UrlRole, FoundPageFetchResult, FileFetchResult } from "./index.js"; import { DeepReadonly } from "ts-essentials"; import { Pool } from "./worker-runner.js"; export declare const recursiveFetchFiles: (pool: Pool, fetchFile: (url: string) => Promise<DeepReadonly<FileFetchResult>>, baseUrl: string, indexName: string) => (startUrls: DeepReadonly<{ url: string; role: UrlRole; }[]>) => Promise<({ url: string; role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | { readonly type: "json"; readonly extractConfigs: readonly { readonly jmespath: string; readonly asserts: readonly ({ readonly type: "image"; } | { readonly type: "video"; } | { readonly type: "font"; } | { readonly type: "imageSize"; readonly width: number; readonly height: number; } | { readonly type: "content-type"; readonly contentType: readonly string[]; } | { readonly type: "permanent"; } | { readonly type: "document"; })[]; readonly role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | /*elided*/ any; }[]; }; res: FoundPageFetchResult; links: readonly { readonly url: string; readonly role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | { readonly type: "json"; readonly extractConfigs: readonly { readonly jmespath: string; readonly asserts: readonly ({ readonly type: "image"; } | { readonly type: "video"; } | { readonly type: "font"; } | { readonly type: "imageSize"; readonly width: number; readonly height: number; } | { readonly type: "content-type"; readonly contentType: readonly string[]; } | { readonly type: "permanent"; } | { readonly type: "document"; })[]; readonly role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | /*elided*/ any; }[]; }; readonly asserts: readonly ({ readonly type: "image"; } | { readonly type: "video"; } | { readonly type: "font"; } | { readonly type: "imageSize"; readonly width: number; readonly height: number; } | { readonly type: "content-type"; readonly contentType: readonly string[]; } | { readonly type: "permanent"; } | { readonly type: "document"; })[]; readonly location: { readonly type: "html"; readonly element: { readonly outerHTML: string; readonly selector: string; }; } | { readonly type: "robotssitemap"; readonly index: number; } | { readonly type: "sitemaptxt"; readonly sitemaplocation: { readonly url: string; } | { readonly extrasitemapIndex: number; }; readonly index: number; } | { readonly type: "sitemapxml"; readonly sitemaplocation: { readonly url: string; } | { readonly extrasitemapIndex: number; }; readonly urlsetIndex: number; readonly urlIndex: number; } | { readonly type: "rss"; readonly rssurl: string; readonly channelIndex: number; readonly linkIndex: number; } | { readonly type: "atom"; readonly atomurl: string; readonly entryIndex: number; readonly linkIndex: number; } | { readonly type: "json"; readonly jsonurl: string; readonly jmespath: string; readonly index: number; } | { readonly type: "css"; readonly position: string; readonly target: string; } | { readonly type: "extraurl"; readonly index: number; } | { readonly type: "redirect"; }; }[]; } | { url: string; role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | { readonly type: "json"; readonly extractConfigs: readonly { readonly jmespath: string; readonly asserts: readonly ({ readonly type: "image"; } | { readonly type: "video"; } | { readonly type: "font"; } | { readonly type: "imageSize"; readonly width: number; readonly height: number; } | { readonly type: "content-type"; readonly contentType: readonly string[]; } | { readonly type: "permanent"; } | { readonly type: "document"; })[]; readonly role: { readonly type: "document"; } | { readonly type: "stylesheet"; } | { readonly type: "asset"; } | { readonly type: "sitemap"; } | { readonly type: "robotstxt"; } | { readonly type: "rss"; } | { readonly type: "atom"; } | /*elided*/ any; }[]; }; res: { readonly url: string; readonly headers: { readonly [x: string]: string; }; readonly status: number; readonly data: { readonly path: string; readonly mtime: number; } | null; }; links: null; })[]>;