cdk-tweet-queue
Version:
Defines an SQS queue with tweet stream from a search
76 lines • 10.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
// eslint-disable-next-line import/no-extraneous-dependencies
const aws_sdk_1 = require("aws-sdk");
// eslint-disable-next-line import/no-extraneous-dependencies
// eslint-disable-next-line @typescript-eslint/no-require-imports
const Twitter = require("twitter");
const checkpoint_1 = require("./checkpoint");
const util_1 = require("./util");
const BATCH_SIZE = 100;
const MAX_RESULTS = 500;
exports.handler = async function (event, _context) {
console.log('event:', JSON.stringify(event));
const sqs = new aws_sdk_1.SQS();
const secretsManager = new aws_sdk_1.SecretsManager();
const checkpoint = new checkpoint_1.CheckpointTable();
// get secret from secrets manager and create twitter client
const secretId = (0, util_1.getEnv)('CREDENTIALS_SECRET');
const result = await secretsManager.getSecretValue({ SecretId: secretId }).promise();
const creds = JSON.parse(result.SecretString);
const twitter = new Twitter(creds);
const query = (0, util_1.getEnv)('TWITTER_QUERY');
// read last checkpoint (can be undefined)
const cursor_tail = await checkpoint.getLastCheckpoint();
console.log('cursor_tail (last checkpoint):', cursor_tail);
let next_cursor_tail = 0;
let cursor_head = undefined;
const results = {};
while (true) {
const req = { q: query, count: BATCH_SIZE, max_id: cursor_head, since_id: cursor_tail };
console.log('twitter search:', JSON.stringify(req));
const res = await twitter.get('search/tweets', req);
let min_id = res.search_metadata.max_id;
let new_tweets = 0;
for (const status of res.statuses) {
if (status.id in results) {
continue; // duplicate (possible)
}
results[status.id] = true;
new_tweets++;
// send tweet to queue
await sqs.sendMessage({
QueueUrl: process.env.QUEUE_URL,
MessageBody: JSON.stringify(status),
}).promise();
// track min_id to query next page
if (status.id < min_id) {
min_id = status.id;
}
// keep track of next cursor tail (the maximum id we processed)
if (status.id > next_cursor_tail) {
next_cursor_tail = status.id;
}
}
console.log(`published ${new_tweets} new tweets`);
// if we haven't received any new tweets, we are done here
if (new_tweets === 0) {
break;
}
if (Object.keys(results).length > MAX_RESULTS) {
break;
}
// bring in the next page
cursor_head = min_id;
}
if (cursor_tail !== next_cursor_tail) {
console.log('storing max_id checkpoint:', next_cursor_tail);
await checkpoint.checkpoint(next_cursor_tail);
}
else {
console.log('no new checkpoint');
}
console.log('results:', Object.keys(results).length);
return { count: results.length };
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJpbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOztBQUFBLDZEQUE2RDtBQUM3RCxxQ0FBOEM7QUFDOUMsNkRBQTZEO0FBQzdELGlFQUFpRTtBQUNqRSxtQ0FBb0M7QUFDcEMsNkNBQStDO0FBQy9DLGlDQUFnQztBQUVoQyxNQUFNLFVBQVUsR0FBRyxHQUFHLENBQUM7QUFDdkIsTUFBTSxXQUFXLEdBQUcsR0FBRyxDQUFDO0FBRXhCLE9BQU8sQ0FBQyxPQUFPLEdBQUcsS0FBSyxXQUFVLEtBQVUsRUFBRSxRQUFhO0lBQ3hELE9BQU8sQ0FBQyxHQUFHLENBQUMsUUFBUSxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQztJQUU3QyxNQUFNLEdBQUcsR0FBRyxJQUFJLGFBQUcsRUFBRSxDQUFDO0lBQ3RCLE1BQU0sY0FBYyxHQUFHLElBQUksd0JBQWMsRUFBRSxDQUFDO0lBQzVDLE1BQU0sVUFBVSxHQUFHLElBQUksNEJBQWUsRUFBRSxDQUFDO0lBRXpDLDREQUE0RDtJQUM1RCxNQUFNLFFBQVEsR0FBRyxJQUFBLGFBQU0sRUFBQyxvQkFBb0IsQ0FBQyxDQUFDO0lBQzlDLE1BQU0sTUFBTSxHQUFHLE1BQU0sY0FBYyxDQUFDLGNBQWMsQ0FBQyxFQUFFLFFBQVEsRUFBRSxRQUFRLEVBQUUsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDO0lBQ3JGLE1BQU0sS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLFlBQWEsQ0FBQyxDQUFDO0lBQy9DLE1BQU0sT0FBTyxHQUFHLElBQUksT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBR25DLE1BQU0sS0FBSyxHQUFHLElBQUEsYUFBTSxFQUFDLGVBQWUsQ0FBQyxDQUFDO0lBRXRDLDBDQUEwQztJQUMxQyxNQUFNLFdBQVcsR0FBRyxNQUFNLFVBQVUsQ0FBQyxpQkFBaUIsRUFBRSxDQUFDO0lBQ3pELE9BQU8sQ0FBQyxHQUFHLENBQUMsZ0NBQWdDLEVBQUUsV0FBVyxDQUFDLENBQUM7SUFFM0QsSUFBSSxnQkFBZ0IsR0FBRyxDQUFDLENBQUM7SUFDekIsSUFBSSxXQUFXLEdBQUcsU0FBUyxDQUFDO0lBQzVCLE1BQU0sT0FBTyxHQUE4QixFQUFHLENBQUM7SUFFL0MsT0FBTyxJQUFJLEVBQUUsQ0FBQztRQUNaLE1BQU0sR0FBRyxHQUFHLEVBQUUsQ0FBQyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsUUFBUSxFQUFFLFdBQVcsRUFBRSxDQUFDO1FBQ3hGLE9BQU8sQ0FBQyxHQUFHLENBQUMsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDO1FBQ3BELE1BQU0sR0FBRyxHQUFRLE1BQU0sT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLEVBQUUsR0FBRyxDQUFDLENBQUM7UUFFekQsSUFBSSxNQUFNLEdBQUcsR0FBRyxDQUFDLGVBQWUsQ0FBQyxNQUFNLENBQUM7UUFDeEMsSUFBSSxVQUFVLEdBQUcsQ0FBQyxDQUFDO1FBQ25CLEtBQUssTUFBTSxNQUFNLElBQUksR0FBRyxDQUFDLFFBQVEsRUFBRSxDQUFDO1lBQ2xDLElBQUksTUFBTSxDQUFDLEVBQUUsSUFBSSxPQUFPLEVBQUUsQ0FBQztnQkFDekIsU0FBUyxDQUFDLHVCQUF1QjtZQUNuQyxDQUFDO1lBRUQsT0FBTyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsR0FBRyxJQUFJLENBQUM7WUFDMUIsVUFBVSxFQUFFLENBQUM7WUFFYixzQkFBc0I7WUFDdEIsTUFBTSxHQUFHLENBQUMsV0FBVyxDQUFDO2dCQUNwQixRQUFRLEVBQUUsT0FBTyxDQUFDLEdBQUcsQ0FBQyxTQUFVO2dCQUNoQyxXQUFXLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUM7YUFDcEMsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBRWIsa0NBQWtDO1lBQ2xDLElBQUksTUFBTSxDQUFDLEVBQUUsR0FBRyxNQUFNLEVBQUUsQ0FBQztnQkFDdkIsTUFBTSxHQUFHLE1BQU0sQ0FBQyxFQUFFLENBQUM7WUFDckIsQ0FBQztZQUVELCtEQUErRDtZQUMvRCxJQUFJLE1BQU0sQ0FBQyxFQUFFLEdBQUcsZ0JBQWdCLEVBQUUsQ0FBQztnQkFDakMsZ0JBQWdCLEdBQUcsTUFBTSxDQUFDLEVBQUUsQ0FBQztZQUMvQixDQUFDO1FBQ0gsQ0FBQztRQUVELE9BQU8sQ0FBQyxHQUFHLENBQUMsYUFBYSxVQUFVLGFBQWEsQ0FBQyxDQUFDO1FBRWxELDBEQUEwRDtRQUMxRCxJQUFJLFVBQVUsS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUNyQixNQUFNO1FBQ1IsQ0FBQztRQUVELElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQyxNQUFNLEdBQUcsV0FBVyxFQUFFLENBQUM7WUFDOUMsTUFBTTtRQUNSLENBQUM7UUFFRCx5QkFBeUI7UUFDekIsV0FBVyxHQUFHLE1BQU0sQ0FBQztJQUN2QixDQUFDO0lBRUQsSUFBSSxXQUFXLEtBQUssZ0JBQWdCLEVBQUUsQ0FBQztRQUNyQyxPQUFPLENBQUMsR0FBRyxDQUFDLDRCQUE0QixFQUFFLGdCQUFnQixDQUFDLENBQUM7UUFDNUQsTUFBTSxVQUFVLENBQUMsVUFBVSxDQUFDLGdCQUFnQixDQUFDLENBQUM7SUFDaEQsQ0FBQztTQUFNLENBQUM7UUFDTixPQUFPLENBQUMsR0FBRyxDQUFDLG1CQUFtQixDQUFDLENBQUM7SUFDbkMsQ0FBQztJQUVELE9BQU8sQ0FBQyxHQUFHLENBQUMsVUFBVSxFQUFFLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDckQsT0FBTyxFQUFFLEtBQUssRUFBRSxPQUFPLENBQUMsTUFBTSxFQUFFLENBQUM7QUFDbkMsQ0FBQyxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLy8gZXNsaW50LWRpc2FibGUtbmV4dC1saW5lIGltcG9ydC9uby1leHRyYW5lb3VzLWRlcGVuZGVuY2llc1xuaW1wb3J0IHsgU2VjcmV0c01hbmFnZXIsIFNRUyB9IGZyb20gJ2F3cy1zZGsnO1xuLy8gZXNsaW50LWRpc2FibGUtbmV4dC1saW5lIGltcG9ydC9uby1leHRyYW5lb3VzLWRlcGVuZGVuY2llc1xuLy8gZXNsaW50LWRpc2FibGUtbmV4dC1saW5lIEB0eXBlc2NyaXB0LWVzbGludC9uby1yZXF1aXJlLWltcG9ydHNcbmltcG9ydCBUd2l0dGVyID0gcmVxdWlyZSgndHdpdHRlcicpO1xuaW1wb3J0IHsgQ2hlY2twb2ludFRhYmxlIH0gZnJvbSAnLi9jaGVja3BvaW50JztcbmltcG9ydCB7IGdldEVudiB9IGZyb20gJy4vdXRpbCc7XG5cbmNvbnN0IEJBVENIX1NJWkUgPSAxMDA7XG5jb25zdCBNQVhfUkVTVUxUUyA9IDUwMDtcblxuZXhwb3J0cy5oYW5kbGVyID0gYXN5bmMgZnVuY3Rpb24oZXZlbnQ6IGFueSwgX2NvbnRleHQ6IGFueSkge1xuICBjb25zb2xlLmxvZygnZXZlbnQ6JywgSlNPTi5zdHJpbmdpZnkoZXZlbnQpKTtcblxuICBjb25zdCBzcXMgPSBuZXcgU1FTKCk7XG4gIGNvbnN0IHNlY3JldHNNYW5hZ2VyID0gbmV3IFNlY3JldHNNYW5hZ2VyKCk7XG4gIGNvbnN0IGNoZWNrcG9pbnQgPSBuZXcgQ2hlY2twb2ludFRhYmxlKCk7XG5cbiAgLy8gZ2V0IHNlY3JldCBmcm9tIHNlY3JldHMgbWFuYWdlciBhbmQgY3JlYXRlIHR3aXR0ZXIgY2xpZW50XG4gIGNvbnN0IHNlY3JldElkID0gZ2V0RW52KCdDUkVERU5USUFMU19TRUNSRVQnKTtcbiAgY29uc3QgcmVzdWx0ID0gYXdhaXQgc2VjcmV0c01hbmFnZXIuZ2V0U2VjcmV0VmFsdWUoeyBTZWNyZXRJZDogc2VjcmV0SWQgfSkucHJvbWlzZSgpO1xuICBjb25zdCBjcmVkcyA9IEpTT04ucGFyc2UocmVzdWx0LlNlY3JldFN0cmluZyEpO1xuICBjb25zdCB0d2l0dGVyID0gbmV3IFR3aXR0ZXIoY3JlZHMpO1xuXG5cbiAgY29uc3QgcXVlcnkgPSBnZXRFbnYoJ1RXSVRURVJfUVVFUlknKTtcblxuICAvLyByZWFkIGxhc3QgY2hlY2twb2ludCAoY2FuIGJlIHVuZGVmaW5lZClcbiAgY29uc3QgY3Vyc29yX3RhaWwgPSBhd2FpdCBjaGVja3BvaW50LmdldExhc3RDaGVja3BvaW50KCk7XG4gIGNvbnNvbGUubG9nKCdjdXJzb3JfdGFpbCAobGFzdCBjaGVja3BvaW50KTonLCBjdXJzb3JfdGFpbCk7XG5cbiAgbGV0IG5leHRfY3Vyc29yX3RhaWwgPSAwO1xuICBsZXQgY3Vyc29yX2hlYWQgPSB1bmRlZmluZWQ7XG4gIGNvbnN0IHJlc3VsdHM6IHsgW2lkOiBzdHJpbmddOiBib29sZWFuIH0gPSB7IH07XG5cbiAgd2hpbGUgKHRydWUpIHtcbiAgICBjb25zdCByZXEgPSB7IHE6IHF1ZXJ5LCBjb3VudDogQkFUQ0hfU0laRSwgbWF4X2lkOiBjdXJzb3JfaGVhZCwgc2luY2VfaWQ6IGN1cnNvcl90YWlsIH07XG4gICAgY29uc29sZS5sb2coJ3R3aXR0ZXIgc2VhcmNoOicsIEpTT04uc3RyaW5naWZ5KHJlcSkpO1xuICAgIGNvbnN0IHJlczogYW55ID0gYXdhaXQgdHdpdHRlci5nZXQoJ3NlYXJjaC90d2VldHMnLCByZXEpO1xuXG4gICAgbGV0IG1pbl9pZCA9IHJlcy5zZWFyY2hfbWV0YWRhdGEubWF4X2lkO1xuICAgIGxldCBuZXdfdHdlZXRzID0gMDtcbiAgICBmb3IgKGNvbnN0IHN0YXR1cyBvZiByZXMuc3RhdHVzZXMpIHtcbiAgICAgIGlmIChzdGF0dXMuaWQgaW4gcmVzdWx0cykge1xuICAgICAgICBjb250aW51ZTsgLy8gZHVwbGljYXRlIChwb3NzaWJsZSlcbiAgICAgIH1cblxuICAgICAgcmVzdWx0c1tzdGF0dXMuaWRdID0gdHJ1ZTtcbiAgICAgIG5ld190d2VldHMrKztcblxuICAgICAgLy8gc2VuZCB0d2VldCB0byBxdWV1ZVxuICAgICAgYXdhaXQgc3FzLnNlbmRNZXNzYWdlKHtcbiAgICAgICAgUXVldWVVcmw6IHByb2Nlc3MuZW52LlFVRVVFX1VSTCEsXG4gICAgICAgIE1lc3NhZ2VCb2R5OiBKU09OLnN0cmluZ2lmeShzdGF0dXMpLFxuICAgICAgfSkucHJvbWlzZSgpO1xuXG4gICAgICAvLyB0cmFjayBtaW5faWQgdG8gcXVlcnkgbmV4dCBwYWdlXG4gICAgICBpZiAoc3RhdHVzLmlkIDwgbWluX2lkKSB7XG4gICAgICAgIG1pbl9pZCA9IHN0YXR1cy5pZDtcbiAgICAgIH1cblxuICAgICAgLy8ga2VlcCB0cmFjayBvZiBuZXh0IGN1cnNvciB0YWlsICh0aGUgbWF4aW11bSBpZCB3ZSBwcm9jZXNzZWQpXG4gICAgICBpZiAoc3RhdHVzLmlkID4gbmV4dF9jdXJzb3JfdGFpbCkge1xuICAgICAgICBuZXh0X2N1cnNvcl90YWlsID0gc3RhdHVzLmlkO1xuICAgICAgfVxuICAgIH1cblxuICAgIGNvbnNvbGUubG9nKGBwdWJsaXNoZWQgJHtuZXdfdHdlZXRzfSBuZXcgdHdlZXRzYCk7XG5cbiAgICAvLyBpZiB3ZSBoYXZlbid0IHJlY2VpdmVkIGFueSBuZXcgdHdlZXRzLCB3ZSBhcmUgZG9uZSBoZXJlXG4gICAgaWYgKG5ld190d2VldHMgPT09IDApIHtcbiAgICAgIGJyZWFrO1xuICAgIH1cblxuICAgIGlmIChPYmplY3Qua2V5cyhyZXN1bHRzKS5sZW5ndGggPiBNQVhfUkVTVUxUUykge1xuICAgICAgYnJlYWs7XG4gICAgfVxuXG4gICAgLy8gYnJpbmcgaW4gdGhlIG5leHQgcGFnZVxuICAgIGN1cnNvcl9oZWFkID0gbWluX2lkO1xuICB9XG5cbiAgaWYgKGN1cnNvcl90YWlsICE9PSBuZXh0X2N1cnNvcl90YWlsKSB7XG4gICAgY29uc29sZS5sb2coJ3N0b3JpbmcgbWF4X2lkIGNoZWNrcG9pbnQ6JywgbmV4dF9jdXJzb3JfdGFpbCk7XG4gICAgYXdhaXQgY2hlY2twb2ludC5jaGVja3BvaW50KG5leHRfY3Vyc29yX3RhaWwpO1xuICB9IGVsc2Uge1xuICAgIGNvbnNvbGUubG9nKCdubyBuZXcgY2hlY2twb2ludCcpO1xuICB9XG5cbiAgY29uc29sZS5sb2coJ3Jlc3VsdHM6JywgT2JqZWN0LmtleXMocmVzdWx0cykubGVuZ3RoKTtcbiAgcmV0dXJuIHsgY291bnQ6IHJlc3VsdHMubGVuZ3RoIH07XG59O1xuIl19