clusterkw
Version:
A package for clustering keywords using OpenAI embeddings
294 lines (293 loc) • 12.6 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const commander_1 = require("commander");
const fs_extra_1 = __importDefault(require("fs-extra"));
const path = __importStar(require("path"));
const csv_parser_1 = __importDefault(require("csv-parser"));
const dotenv = __importStar(require("dotenv"));
const keyword_clusterer_1 = require("./keyword-clusterer");
// Load environment variables
dotenv.config();
const program = new commander_1.Command();
// Set up CLI options
program
.name('clusterkw')
.description('Cluster keywords using OpenAI embeddings')
.version('0.1.0');
program
.option('-f, --file <path>', 'Path to file containing keywords (supports txt, csv, json)')
.option('-c, --column <name>', 'Column name containing keywords (for CSV files)', 'keyword')
.option('-o, --output <path>', 'Output file path (supports json, csv)')
.option('--api-key <apiKey>', 'OpenAI API key (overrides OPENAI_API_KEY env variable)')
.option('-m, --min-cluster-size <number>', 'Minimum cluster size', '2')
.option('-d, --distance <number>', 'Maximum distance threshold', '0.3')
.option('-e, --embedding-model <model>', 'OpenAI embedding model', 'text-embedding-3-small')
.option('-g, --gpt-model <model>', 'OpenAI completion model', 'gpt-4o-mini-2024-07-18')
.option('-a, --algorithm <algorithm>', 'Clustering algorithm (simple, kmeans, hierarchical, direct)', 'simple')
.option('-k, --clusters <number>', 'Number of clusters for k-means algorithm', '5')
.option('--max-iterations <number>', 'Maximum iterations for k-means algorithm', '100')
.option('--linkage <method>', 'Linkage method for hierarchical clustering (single, complete, average)', 'average')
.option('--context <description>', 'Context description to guide clustering (e.g., "AI chat topics")')
.option('--delimiter <char>', 'CSV delimiter', ',')
.option('--no-header', 'CSV file has no header row');
program.parse(process.argv);
const options = program.opts();
async function main() {
try {
// Check if file is provided
if (!options.file) {
console.error('Error: File path is required. Use --file or -f option.');
program.help();
process.exit(1);
}
// Load .env file first, regardless of whether we already have an API key
try {
// Check if .env file exists
if (fs_extra_1.default.existsSync('.env')) {
console.log('Found .env file, loading environment variables...');
// Force reload of .env file
dotenv.config({ override: true, path: '.env' });
}
}
catch (error) {
if (error instanceof Error) {
console.warn('Error loading .env file:', error.message);
}
else {
console.warn('Error loading .env file');
}
}
// Get API key with enhanced handling - check all possible sources
let apiKey = options.apiKey ||
process.env.OPENAI_API_KEY ||
process.env.OPENAI_KEY;
// Debug output to help troubleshoot
if (!apiKey) {
console.log('API key not found. Checking environment variables:');
console.log('- OPENAI_API_KEY:', process.env.OPENAI_API_KEY ? 'Found' : 'Not found');
console.log('- OPENAI_KEY:', process.env.OPENAI_KEY ? 'Found' : 'Not found');
console.log('- --api-key option:', options.apiKey ? 'Provided' : 'Not provided');
console.log('- .env file:', fs_extra_1.default.existsSync('.env') ? 'Found' : 'Not found');
}
if (!apiKey) {
console.error('Error: OpenAI API key is required. You can provide it via:');
console.error(' 1. --api-key command line option');
console.error(' 2. OPENAI_API_KEY environment variable');
console.error(' 3. OPENAI_KEY environment variable');
console.error(' 4. .env file with OPENAI_API_KEY=your-key or OPENAI_KEY=your-key');
process.exit(1);
}
// Check if file exists
if (!fs_extra_1.default.existsSync(options.file)) {
console.error(`Error: File not found: ${options.file}`);
process.exit(1);
}
console.log(`Reading keywords from: ${options.file}`);
// Read keywords from file
const keywords = await readKeywordsFromFile(options.file, options);
if (keywords.length === 0) {
console.error('Error: No keywords found in the file.');
process.exit(1);
}
console.log(`Found ${keywords.length} keywords.`);
// Initialize clusterer
const clusterer = new keyword_clusterer_1.KeywordClusterer({
apiKey,
embeddingModel: options.embeddingModel,
completionModel: options.gptModel,
minClusterSize: parseInt(options.minClusterSize, 10),
distanceThreshold: parseFloat(options.distance),
algorithm: options.algorithm,
k: parseInt(options.clusters, 10),
maxIterations: parseInt(options.maxIterations, 10),
linkage: options.linkage,
context: options.context
});
if (options.context) {
console.log(`Using context: "${options.context}"`);
}
console.log(`Using clustering algorithm: ${options.algorithm}`);
console.log('Clustering keywords...');
// Cluster keywords
const clusters = await clusterer.clusterKeywords(keywords);
console.log(`\nFound ${clusters.length} clusters:\n`);
// Display clusters
clusters.forEach((cluster, index) => {
console.log(`Cluster ${index + 1}: ${cluster.name || 'Unnamed Cluster'}`);
console.log(`Description: ${cluster.description || 'No description'}`);
console.log(`Items (${cluster.items.length}):`);
// Only show first 5 items if there are more than 10
const displayItems = cluster.items.length > 10
? [...cluster.items.slice(0, 5), `... and ${cluster.items.length - 5} more`]
: cluster.items;
displayItems.forEach(item => console.log(` - ${item}`));
console.log('');
});
// Save output if specified
if (options.output) {
await saveOutput(clusters, options.output);
console.log(`Results saved to: ${options.output}`);
}
}
catch (error) {
console.error('Error:', error);
process.exit(1);
}
}
async function readKeywordsFromFile(filePath, options) {
const fileExt = path.extname(filePath).toLowerCase();
// Read based on file extension
switch (fileExt) {
case '.csv':
return readFromCSV(filePath, options);
case '.json':
return readFromJSON(filePath, options);
case '.txt':
default:
return readFromTXT(filePath);
}
}
async function readFromCSV(filePath, options) {
return new Promise((resolve, reject) => {
const results = [];
const column = options.column;
// Set up CSV parser
const parser = fs_extra_1.default.createReadStream(filePath)
.pipe((0, csv_parser_1.default)({
separator: options.delimiter,
headers: options.header !== false
}));
parser.on('data', (data) => {
// If headers are used, look for the specified column
if (options.header !== false) {
if (data[column]) {
results.push(data[column]);
}
}
else {
// If no headers, take the first value from each row
const firstValue = Object.values(data)[0];
if (firstValue) {
results.push(firstValue);
}
}
});
parser.on('end', () => {
resolve(results);
});
parser.on('error', (error) => {
reject(error);
});
});
}
async function readFromJSON(filePath, options) {
try {
const data = await fs_extra_1.default.readJSON(filePath);
// Handle different JSON formats
if (Array.isArray(data)) {
// If it's an array of strings
if (data.length > 0 && typeof data[0] === 'string') {
return data;
}
// If it's an array of objects
if (data.length > 0 && typeof data[0] === 'object') {
const column = options.column;
return data
.filter((item) => item[column])
.map((item) => item[column]);
}
}
// If it's an object with a keywords array
if (data.keywords && Array.isArray(data.keywords)) {
return data.keywords;
}
throw new Error('Invalid JSON format. Expected an array of strings, array of objects with a keyword property, or an object with a keywords array.');
}
catch (error) {
if (error instanceof Error) {
throw new Error(`Failed to parse JSON file: ${error.message}`);
}
throw new Error('Failed to parse JSON file');
}
}
async function readFromTXT(filePath) {
try {
const content = await fs_extra_1.default.readFile(filePath, 'utf8');
// Split by newlines and filter out empty lines
return content.split('\n')
.map((line) => line.trim())
.filter((line) => line.length > 0);
}
catch (error) {
if (error instanceof Error) {
throw new Error(`Failed to read text file: ${error.message}`);
}
throw new Error('Failed to read text file');
}
}
async function saveOutput(clusters, outputPath) {
const fileExt = path.extname(outputPath).toLowerCase();
try {
switch (fileExt) {
case '.json':
await fs_extra_1.default.writeJSON(outputPath, clusters, { spaces: 2 });
break;
case '.csv':
// Create CSV content
const csvContent = [
'cluster_id,cluster_name,cluster_description,keyword',
...clusters.flatMap((cluster, index) => cluster.items.map((item) => `${index + 1},"${(cluster.name || '').replace(/"/g, '""')}","${(cluster.description || '').replace(/"/g, '""')}","${item.replace(/"/g, '""')}"`))
].join('\n');
await fs_extra_1.default.writeFile(outputPath, csvContent);
break;
default:
// Default to JSON if extension is not recognized
await fs_extra_1.default.writeJSON(`${outputPath}.json`, clusters, { spaces: 2 });
break;
}
}
catch (error) {
if (error instanceof Error) {
throw new Error(`Failed to save output: ${error.message}`);
}
throw new Error('Failed to save output');
}
}
main();