indexnow-submitter
Version:
An IndexNow Submission module with caching and analytics
236 lines (235 loc) • 10.6 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IndexNowSubmitter = void 0;
const axios_1 = __importDefault(require("axios"));
const commander_1 = require("commander");
const fs = __importStar(require("fs/promises"));
const xml2js_1 = require("xml2js");
const util_1 = require("util");
const winston_1 = require("winston");
const dotenv = __importStar(require("dotenv"));
const node_cache_1 = __importDefault(require("node-cache"));
dotenv.config();
const parseXml = (0, util_1.promisify)(xml2js_1.parseString);
const logger = (0, winston_1.createLogger)({
level: 'info',
format: winston_1.format.combine(winston_1.format.timestamp(), winston_1.format.json()),
transports: [
new winston_1.transports.Console(),
new winston_1.transports.File({ filename: 'indexnow.log' })
]
});
const defaultConfig = {
engine: 'api.indexnow.org',
key: process.env.INDEXNOW_KEY || '',
keyPath: process.env.INDEXNOW_KEY_PATH || `https://${process.env.INDEXNOW_HOST}/${process.env.INDEXNOW_KEY}.txt`,
host: process.env.INDEXNOW_HOST || '',
batchSize: 100,
rateLimit: 1000,
cacheTTL: 86400 // 24 hours
};
class IndexNowSubmitter {
constructor(config = {}) {
this.config = Object.assign(Object.assign({}, defaultConfig), config);
const missingConfig = [];
if (!this.config.key)
missingConfig.push('key');
if (!this.config.host)
missingConfig.push('host');
if (missingConfig.length > 0) {
throw new Error(`Missing required config: ${missingConfig.join(', ')}`);
}
this.cache = new node_cache_1.default({ stdTTL: this.config.cacheTTL });
this.analytics = {
totalSubmissions: 0,
successfulSubmissions: 0,
failedSubmissions: 0,
averageResponseTime: 0
};
}
delay(ms) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => setTimeout(resolve, ms));
});
}
submitBatch(urls) {
return __awaiter(this, void 0, void 0, function* () {
if (urls.length === 0)
return; // Don't submit empty batches
const endpoint = `https://${this.config.engine}/IndexNow`;
const payload = {
host: this.config.host,
key: this.config.key,
keyPath: this.config.keyPath,
urlList: urls
};
logger.info(`Submitting batch of ${urls.length} urls: ${urls}`);
try {
const startTime = Date.now();
const response = yield axios_1.default.post(endpoint, payload);
const endTime = Date.now();
this.updateAnalytics(urls.length, endTime - startTime);
logger.info(`Batch submitted successfully: ${response.status}`);
}
catch (error) {
this.analytics.totalSubmissions += urls.length;
this.analytics.failedSubmissions += urls.length;
logger.error('Error submitting batch:', error);
throw error;
}
});
}
updateAnalytics(urlCount, responseTime) {
this.analytics.totalSubmissions += urlCount;
this.analytics.successfulSubmissions += urlCount;
this.analytics.averageResponseTime =
(this.analytics.averageResponseTime * (this.analytics.totalSubmissions - urlCount) + responseTime) /
this.analytics.totalSubmissions;
}
processBatch(urls) {
return __awaiter(this, void 0, void 0, function* () {
if (urls.length === 0)
return; // Don't process empty batches
const batch = urls.slice(0, this.config.batchSize);
yield this.submitBatch(batch);
batch.forEach(url => this.cache.set(url, true));
if (urls.length > this.config.batchSize) {
yield this.delay(this.config.rateLimit);
yield this.processBatch(urls.slice(this.config.batchSize));
}
});
}
submitUrls(urls) {
return __awaiter(this, void 0, void 0, function* () {
const uncachedUrls = urls.filter(url => !this.cache.get(url));
logger.info(`Submitting ${uncachedUrls.length} uncached URLs`);
if (uncachedUrls.length > 0) {
yield this.processBatch(uncachedUrls);
}
});
}
submitSingleUrl(url) {
return __awaiter(this, void 0, void 0, function* () {
logger.info(`Checking cache for URL: ${url}`);
if (!this.cache.get(url)) {
yield this.submitUrls([url]);
}
else {
logger.info(`URL ${url} already submitted recently. Skipping.`);
}
});
}
submitFromSitemap(sitemapUrl, modifiedSince) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
try {
const response = yield axios_1.default.get(sitemapUrl);
const result = yield parseXml(response.data);
const urls = ((_b = (_a = result.urlset) === null || _a === void 0 ? void 0 : _a.url) === null || _b === void 0 ? void 0 : _b.filter((entry) => {
if (!modifiedSince)
return true;
if (!entry.lastmod || !entry.lastmod[0])
return false; // ignore if lastmod entry is not present
const lastmod = new Date(entry.lastmod[0]);
return lastmod >= modifiedSince;
}).map((entry) => entry.loc && entry.loc[0]).filter(Boolean)) || [];
logger.info(`Found ${urls.length} URLs in sitemap`);
if (urls.length > 0) {
yield this.submitUrls(urls);
}
}
catch (error) {
logger.error('Error processing sitemap:', error);
throw error;
}
});
}
getAnalytics() {
return Object.assign({}, this.analytics);
}
}
exports.IndexNowSubmitter = IndexNowSubmitter;
function runCli() {
return __awaiter(this, void 0, void 0, function* () {
commander_1.program
.version('1.3.1')
.option('-e, --engine <engine>', 'Search engine domain')
.option('-k, --key <key>', 'IndexNow API key')
.option('-h, --host <host>', 'Your website host')
.option('-p, --key-path <key-path>', 'IndexNow API key path')
.option('-b, --batch-size <size>', 'Batch size for URL submission, default is 100')
.option('-r, --rate-limit <delay>', 'Delay between batches in milliseconds, default is 1000')
.option('-c, --cache-ttl <ttl>', 'Cache TTL in seconds, default is (24 hours)');
commander_1.program
.command('submit <url>')
.description('Submit a single URL')
.action((url) => __awaiter(this, void 0, void 0, function* () {
const submitter = new IndexNowSubmitter(commander_1.program.opts());
yield submitter.submitSingleUrl(url);
console.log('Analytics:', submitter.getAnalytics());
}));
commander_1.program
.command('submit-file <file>')
.description('Submit URLs from a file, with each url in a single line')
.action((file) => __awaiter(this, void 0, void 0, function* () {
const submitter = new IndexNowSubmitter(commander_1.program.opts());
const content = yield fs.readFile(file, 'utf-8');
const urls = content.split('\n').filter(url => url.trim() !== '');
yield submitter.submitUrls(urls);
console.log('Analytics:', submitter.getAnalytics());
}));
commander_1.program
.command('submit-sitemap <url>')
.option('-d, --modified-since <date>', 'Only submit URLs modified since this date')
.description('Submit URLs from a sitemap')
.action((url, options) => __awaiter(this, void 0, void 0, function* () {
const submitter = new IndexNowSubmitter(commander_1.program.opts());
const modifiedSince = options.modifiedSince ? new Date(options.modifiedSince) : undefined;
yield submitter.submitFromSitemap(url, modifiedSince);
console.log('Analytics:', submitter.getAnalytics());
}));
yield commander_1.program.parseAsync(process.argv);
});
}
if (require.main === module) {
runCli().catch(error => {
logger.error('CLI execution error:', error);
process.exit(1);
});
}