@siren-js/client/dist/crawl/crawler.js

Siren API client library

import { follow } from '../follow';
import { parse } from '../parse';
import { isHyperlink } from '../utils';
export class Crawler {
    politenessPolicy;
    revisitPolicy;
    selectionPolicy;
    visitor;
    constructor(politenessPolicy, revisitPolicy, selectionPolicy, visitor) {
        this.politenessPolicy = politenessPolicy;
        this.revisitPolicy = revisitPolicy;
        this.selectionPolicy = selectionPolicy;
        this.visitor = visitor;
    }
    async crawl(entryPoint) {
        const crawled = new Set();
        const targets = [entryPoint];
        while (targets.length > 0) {
            const [target] = targets.splice(0, 1);
            const url = targetUrl(target);
            if (!this.politenessPolicy.canCrawl(target) || (crawled.has(url) && !this.revisitPolicy.canRevisit(target)))
                continue;
            await wait(this.politenessPolicy.delay);
            const response = await follow(target);
            crawled.add(url);
            if (isParsable(response)) {
                const entity = await parse(response);
                await entity.accept(this.visitor);
                const selectedTargets = await this.selectionPolicy.selectFrom(entity);
                targets.push(...selectedTargets);
            }
        }
    }
}
const targetUrl = (target) => (isHyperlink(target) ? target.href.toString() : target.toString());
const wait = (amountMs) => new Promise((resolve) => setTimeout(resolve, amountMs));
const sirenMediaTypeRegExp = /^application\/vnd\.siren\+json/;
const isParsable = (response) => {
    const contentType = response.headers.get('Content-Type');
    return contentType != null && (sirenMediaTypeRegExp.test(contentType) || contentType.includes('json'));
};