oxylabs-ai-studio
Version:
JavaScript SDK for Oxylabs AI Studio API services
97 lines • 3.52 kB
JavaScript
/**
* AI-Scrape Service
* Handles all AI-Scrape related API calls
*/
export class AiScrapeService {
constructor(client) {
this.client = client;
}
/**
* Generate schema for scraping (POST /scrape/schema)
*/
async generateSchema(options) {
return await this.client.post('/scrape/schema', options);
}
/**
* Submit scraping request (POST /scrape)
*/
async submitScrapeRequest(options) {
console.log('Submitting scrape request with options:', options);
const payload = {
url: options.url,
output_format: options.output_format || "markdown",
render_html: options.render_html || false
};
// Only include openapi_schema if output_format is json
if (options.output_format === "json" && options.openapi_schema) {
payload.openapi_schema = options.openapi_schema;
}
return await this.client.post('/scrape', payload);
}
/**
* Get scraping run status (GET /scrape/run)
*/
async getScrapeRun(runId) {
if (!runId) {
throw new Error('run_id is required');
}
const params = new URLSearchParams();
params.append('run_id', runId);
const url = `/scrape/run?${params.toString()}`;
return await this.client.get(url);
}
/**
* Get scraping run data/results (GET /scrape/run/data)
*/
async getScrapeRunData(runId) {
if (!runId) {
throw new Error('run_id is required');
}
const params = new URLSearchParams();
params.append('run_id', runId);
const url = `/scrape/run/data?${params.toString()}`;
return await this.client.get(url);
}
/**
* Synchronous scraping (wait for results)
*/
async scrape(options, timeout = 60000, pollInterval = 5000) {
const submitResult = await this.submitScrapeRequest(options);
const runId = submitResult.run_id || submitResult.id;
if (!runId) {
throw new Error('No run ID returned from scrape request');
}
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
const runStatus = await this.getScrapeRun(runId);
const run_status = runStatus.status;
console.log('Run status:', run_status);
if (run_status === 'completed' || run_status === 'success') {
return await this.getScrapeRunData(runId);
}
else if (run_status === 'failed' || run_status === 'error') {
throw new Error(`Scraping failed: ${runStatus.error || runStatus.message || 'Unknown error'}`);
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
throw new Error(`Scraping timeout after ${timeout}ms`);
}
/**
* Complete workflow with auto-schema and sync results
*/
async scrapeWithAutoSchema(options, timeout = 60000) {
// Generate schema first
const schemaResult = await this.generateSchema({
user_prompt: options.user_prompt
});
// Then perform synchronous scraping
return await this.scrape({
url: options.url,
user_prompt: "",
output_format: options.output_format || "markdown",
openapi_schema: schemaResult.openapi_schema,
render_html: options.render_html || false
}, timeout);
}
}
//# sourceMappingURL=aiScrape.js.map