@kareemaly/researcher/dist/commands/search.js

CLI tool for web research

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.searchCommand = searchCommand;
const searchApiProvider_1 = require("../services/searchApiProvider");
const turndownContentProcessor_1 = require("../services/turndownContentProcessor");
const fileSystemStorage_1 = require("../services/fileSystemStorage");
const logger_1 = require("../utils/logger");
const path_1 = __importDefault(require("path"));
const log = (0, logger_1.createLogger)("cli:search");
function searchCommand(program) {
    program
        .command("search")
        .description("Search and process results")
        .argument("<query>", "search query")
        .option("-t, --type <type>", "search type (web, news)", "web")
        .option("-l, --limit <number>", "number of results", "5")
        .option("--location <location>", "location for search results")
        .option("--language <lang>", "language for search results", "en")
        .action(async (query, options, command) => {
        try {
            const globalOpts = command.parent?.opts();
            const outputDir = path_1.default.resolve(globalOpts?.output || "research");
            log("Starting search with options: %O", {
                query,
                ...options,
                outputDir,
            });
            // Initialize services
            const searchProvider = new searchApiProvider_1.SearchApiProvider({
                apiKey: process.env.SEARCH_API_KEY,
                endpoint: process.env.SEARCH_API_ENDPOINT,
                rateLimitConfig: {
                    requestsPerPeriod: 1,
                    periodSeconds: 30,
                },
            });
            const contentProcessor = new turndownContentProcessor_1.TurndownContentProcessor();
            const storage = new fileSystemStorage_1.FileSystemStorage({ basePath: outputDir });
            // Initialize storage
            await storage.initialize();
            // Perform search
            const searchResults = await searchProvider.search(query, {
                type: options.type,
                limit: parseInt(options.limit),
                location: options.location,
                language: options.language,
            });
            // Save search results
            const searchId = await storage.saveSearch(searchResults);
            log("Search results saved with ID: %s", searchId);
            // Process each result
            console.log("\nProcessing search results...");
            for (const [index, result] of searchResults.results.entries()) {
                try {
                    console.log(`\nProcessing (${index + 1}/${searchResults.results.length}): ${result.title}`);
                    const content = await contentProcessor.process(result.url);
                    await storage.saveContent(searchId, content);
                    console.log("✓ Processed successfully");
                }
                catch (error) {
                    console.error(`✗ Failed to process ${result.url}:`, error);
                }
            }
            // Print summary
            const stats = await storage.getStats();
            console.log("\nSearch complete!");
            console.log(`- Search ID: ${searchId}`);
            console.log(`- Results processed: ${searchResults.results.length}`);
            console.log(`- Total storage size: ${(stats.totalSizeBytes / 1024 / 1024).toFixed(2)} MB`);
            console.log(`\nView results with: researcher show ${searchId}`);
        }
        catch (error) {
            log.error("Search failed: %O", error);
            console.error("Search failed:", error instanceof Error ? error.message : error);
            process.exit(1);
        }
    });
}