oxylabs-ai-studio
Version:
JavaScript SDK for Oxylabs AI Studio API services
95 lines • 3.79 kB
JavaScript
/**
* AI-Crawl Service
* Handles all AI-Crawl related API calls
*/
export class AiCrawlerService {
constructor(client) {
this.client = client;
}
/**
* Generate schema for crawling (POST /crawl/generate-params)
*/
async generateSchema(options) {
return await this.client.post('/crawl/generate-params', options);
}
/**
* Submit crawling request (POST /crawl/run)
*/
async submitCrawlRequest(options) {
const payload = {
url: options.url,
user_prompt: options.user_prompt,
output_format: options.output_format || "markdown",
render_javascript: options.render_javascript || false,
return_sources_limit: options.return_sources_limit ?? 25,
geo_location: options.geo_location || undefined,
};
if (options.max_credits !== undefined) {
payload.max_credits = options.max_credits;
}
// Only include openapi_schema if output_format is json, csv or toon
if ((options.output_format === "json" || options.output_format === "csv" || options.output_format === "toon") && options.schema) {
payload.openapi_schema = options.schema;
}
return await this.client.post('/crawl/run', payload);
}
/**
* Get crawling run data/results (GET /crawl/run/data)
*/
async getCrawlRunData(runId) {
if (!runId) {
throw new Error('run_id is required');
}
const params = new URLSearchParams();
params.append('run_id', runId);
const url = `/crawl/run/data?${params.toString()}`;
return await this.client.get(url);
}
/**
* Synchronous crawling (wait for results)
*/
async crawl(options, timeout = 120000, pollInterval = 5000) {
const submitResult = await this.submitCrawlRequest(options);
const runId = submitResult.run_id || submitResult.id;
if (!runId) {
throw new Error('No run ID returned from crawl request');
}
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
const runData = await this.getCrawlRunData(runId);
const status = (runData && (runData.status || (runData.run && runData.run.status)));
if (status === 'completed') {
return runData;
}
else if (status === 'failed' || status === 'error') {
const message = ((runData.error_code || runData.message)) || 'Unknown error';
throw new Error(`Crawling failed: ${message}`);
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
throw new Error(`Crawling timeout after ${timeout}ms`);
}
/**
* Complete workflow with auto-schema and sync results
*/
async crawlWithAutoSchema(options, timeout = 60000) {
// Generate schema first
const schemaResult = await this.generateSchema({
user_prompt: options.parse_prompt
});
// save schema to file locally
console.log('schemaResult', JSON.stringify(schemaResult.openapi_schema, null, 2));
// Then perform synchronous crawling
return await this.crawl({
url: options.url,
user_prompt: options.user_prompt || "",
output_format: options.output_format || "markdown",
schema: schemaResult.openapi_schema,
render_javascript: options.render_javascript || false,
return_sources_limit: options.return_sources_limit || 25,
geo_location: options.geo_location || undefined,
max_credits: options.max_credits ?? undefined,
}, timeout);
}
}
//# sourceMappingURL=aiCrawler.js.map