krawala/dist/utils.d.ts

Simple javascript web-crawler with command line interface

/// <reference types="underscore" />
import * as _ from 'underscore';
import { Attributes, Crawled, LinkScriptImageOrHref, Options, Output, Page, PageData, PartiallyCrawled, Progress } from './types';
export declare const error: (message: string) => void;
export declare const isSameDomain: (url: string, parentUrl: string) => boolean;
export declare const resolveUrl: (url: string, parentUrl?: string | undefined) => string;
export declare const validateBaseUrl: (url: string) => true | void;
export declare const padLeft: (value: string | number) => string;
export declare const clearProgress: () => void;
export declare const createProgressMessage: (options: Options, progress: Progress, currentTask: string) => string;
export declare const updateProgress: (options: Options, progress: Progress, currentTask: string) => void;
export declare const resolveLinkScriptImageOrHref: (attribsList: Attributes[], url: string, baseUrl: string, linkKey: string) => {
    url: string;
    resolved: string;
    references: number;
    attributes: Attributes;
}[];
export declare const groupHrefs: (resolvedHrefs: LinkScriptImageOrHref[], baseUrl: string) => _.Dictionary<LinkScriptImageOrHref[]>;
export declare const collectData: (node: Crawled & Partial<Page>, text: string, baseUrl: string) => PageData;
export declare const getOutput: (options: Options, pages: any[] | {
    [index: string]: any;
}) => string;
export declare const complete: (options: Options, pages: Output) => void;
export declare const getFailed: (crawled: {
    [index: string]: PartiallyCrawled | undefined;
}) => PartiallyCrawled[];