microfrontier
Version:
A web crawler frontier implementation in TypeScript backed by Redis. MicroFrontier is a scalable and distributed frontier implemented through Redis Queues.
88 lines (87 loc) • 15.7 kB
JavaScript
"use strict";
var _ioredis = require("ioredis");
var _urlfrontier = require("./URLFrontier");
let FrontendWorker = class FrontendWorker {
start() {
this.stopped = false;
this.work();
}
stop() {
this.stopped = true;
}
pickPriorityQueueByPrioritizationStrategy() {
// Custom Strategy
if (this.config.frontendPrioritizationStrategy) return this.config.frontendPrioritizationStrategy(this.config);
// Default Strategy: Random Partition
const rand = Math.random();
var picked;
for (var op of this.orderedPriority){
if (rand < op.probability) {
picked = op.name;
break;
}
}
// Last element not covered by the for loop above
if (!picked) {
picked = this.orderedPriority[this.orderedPriority.length - 1].name;
}
return picked;
}
async workOnce(forcedPriority) {
const queue = forcedPriority ? _urlfrontier.URLFrontier.getFrontendQueueName(this.config, forcedPriority) : _urlfrontier.URLFrontier.getFrontendQueueName(this.config, this.pickPriorityQueueByPrioritizationStrategy());
const poppedItem = await this.redisClient.rpop(queue);
if (!poppedItem) return false;
const { url , meta } = JSON.parse(poppedItem);
const parsedUrl = new URL(url);
// Since ioredis-mock doesn't support multi() & transaction
if (process.env.JEST_WORKER_ID === undefined) {
await this.redisClient.multi().lpush(_urlfrontier.URLFrontier.getBackendQueueName(this.config, parsedUrl.hostname), JSON.stringify({
url,
meta
})) // add to the backend queue
.zincrby(_urlfrontier.URLFrontier.getHostnameUrlCountName(this.config), 1, parsedUrl.hostname) // Increment by one the hostname urls
.zadd(_urlfrontier.URLFrontier.getHeapName(this.config), 'NX', new Date().getTime(), parsedUrl.hostname) // In case of new hostname in the frontier, add it to the heap
.exec();
} else {
await this.redisClient.lpush(_urlfrontier.URLFrontier.getBackendQueueName(this.config, parsedUrl.hostname), JSON.stringify({
url,
meta
})) // add to the backend queue
;
await this.redisClient.zincrby(_urlfrontier.URLFrontier.getHostnameUrlCountName(this.config), 1, parsedUrl.hostname) // Increment by one the hostname urls
;
await this.redisClient.zadd(_urlfrontier.URLFrontier.getHeapName(this.config), 'NX', new Date().getTime(), parsedUrl.hostname) // In case of new hostname in the frontier, add it to the heap
;
}
return true;
}
async work() {
while(true && !this.stopped){
if (!await this.workOnce()) {
// If nothing found slow down requests to avoid DDoSing Redis
await new Promise((resolve, reject)=>{
setTimeout(resolve, 100);
});
}
}
}
constructor(configuration){
const { config , redisClient , redisOptions } = configuration;
if (!config) throw new Error("FrontendWorker needs a URLFrontier configuration to be started");
if (!redisOptions && !redisClient || redisOptions && redisClient) throw new Error("FrontendWorker needs at most one of redisClient or redisOptions to connect to redis, not both or none.");
this.config = config;
this.stopped = false;
// Sort priorities to be used later
this.orderedPriority = Object.keys(this.config.priorities).map((k)=>{
return {
probability: this.config.priorities[k].probability,
name: k
};
}).sort((a, b)=>a.probability - b.probability
);
if (redisOptions) this.redisClient = new _ioredis.default(redisOptions);
if (redisClient) this.redisClient = redisClient;
}
};
exports.FrontendWorker = FrontendWorker;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIi4uL3NyYy9Gcm9udGVuZFdvcmtlci50cyJdLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgSU9SZWRpcyBmcm9tIFwiaW9yZWRpc1wiO1xuaW1wb3J0IHsgVVJMRnJvbnRpZXIgfSBmcm9tIFwiLi9VUkxGcm9udGllclwiO1xuaW1wb3J0IHsgVVJMRnJvbnRpZXJDb25maWcgfSBmcm9tIFwiLi9VUkxGcm9udGllckNvbmZpZ1wiO1xuXG5cbmV4cG9ydCBjbGFzcyBGcm9udGVuZFdvcmtlciB7XG4gICAgcmVkaXNDbGllbnQ6IElPUmVkaXMuUmVkaXNcbiAgICBjb25maWc6IFVSTEZyb250aWVyQ29uZmlnXG4gICAgc3RvcHBlZDogYm9vbGVhbiBcbiAgICBvcmRlcmVkUHJpb3JpdHk6IHtwcm9iYWJpbGl0eTogbnVtYmVyLCBuYW1lOiBzdHJpbmd9W11cblxuICAgIGNvbnN0cnVjdG9yKGNvbmZpZ3VyYXRpb246IHtjb25maWc6IFVSTEZyb250aWVyQ29uZmlnLCByZWRpc0NsaWVudD86IElPUmVkaXMuUmVkaXMsIHJlZGlzT3B0aW9ucz86IElPUmVkaXMuUmVkaXNPcHRpb25zfSkge1xuICAgICAgICBjb25zdCB7Y29uZmlnLCByZWRpc0NsaWVudCwgcmVkaXNPcHRpb25zfSA9IGNvbmZpZ3VyYXRpb25cbiAgICAgICAgaWYgKCFjb25maWcpXG4gICAgICAgICAgICB0aHJvdyBuZXcgRXJyb3IoXCJGcm9udGVuZFdvcmtlciBuZWVkcyBhIFVSTEZyb250aWVyIGNvbmZpZ3VyYXRpb24gdG8gYmUgc3RhcnRlZFwiKVxuICAgICAgICBpZiAoIXJlZGlzT3B0aW9ucyAmJiAhcmVkaXNDbGllbnQgfHwgcmVkaXNPcHRpb25zICYmIHJlZGlzQ2xpZW50KVxuICAgICAgICAgICAgdGhyb3cgbmV3IEVycm9yKFwiRnJvbnRlbmRXb3JrZXIgbmVlZHMgYXQgbW9zdCBvbmUgb2YgcmVkaXNDbGllbnQgb3IgcmVkaXNPcHRpb25zIHRvIGNvbm5lY3QgdG8gcmVkaXMsIG5vdCBib3RoIG9yIG5vbmUuXCIpXG5cbiAgICAgICAgdGhpcy5jb25maWcgPSBjb25maWdcbiAgICAgICAgdGhpcy5zdG9wcGVkID0gZmFsc2U7XG5cbiAgICAgICAgLy8gU29ydCBwcmlvcml0aWVzIHRvIGJlIHVzZWQgbGF0ZXJcbiAgICAgICAgdGhpcy5vcmRlcmVkUHJpb3JpdHkgPSBPYmplY3Qua2V5cyh0aGlzLmNvbmZpZy5wcmlvcml0aWVzKS5tYXAoKGspID0-IHtcbiAgICAgICAgICAgIHJldHVybiB7cHJvYmFiaWxpdHk6IHRoaXMuY29uZmlnLnByaW9yaXRpZXNba10ucHJvYmFiaWxpdHksIG5hbWU6IGt9XG4gICAgICAgIH0pLnNvcnQoKGEsYikgPT4gYS5wcm9iYWJpbGl0eS1iLnByb2JhYmlsaXR5KVxuXG4gICAgICAgIGlmIChyZWRpc09wdGlvbnMpXG4gICAgICAgICAgICB0aGlzLnJlZGlzQ2xpZW50ID0gbmV3IElPUmVkaXMocmVkaXNPcHRpb25zKVxuXG4gICAgICAgIGlmIChyZWRpc0NsaWVudClcbiAgICAgICAgICAgIHRoaXMucmVkaXNDbGllbnQgPSByZWRpc0NsaWVudFxuICAgIH1cblxuICAgIHB1YmxpYyBzdGFydCgpIHtcbiAgICAgICAgdGhpcy5zdG9wcGVkID0gZmFsc2U7XG4gICAgICAgIHRoaXMud29yaygpO1xuICAgIH1cblxuICAgIHB1YmxpYyBzdG9wKCkge1xuICAgICAgICB0aGlzLnN0b3BwZWQgPSB0cnVlO1xuICAgIH1cblxuICAgIHByaXZhdGUgcGlja1ByaW9yaXR5UXVldWVCeVByaW9yaXRpemF0aW9uU3RyYXRlZ3koKTogc3RyaW5nIHtcbiAgICAgICAgLy8gQ3VzdG9tIFN0cmF0ZWd5XG4gICAgICAgIGlmKHRoaXMuY29uZmlnLmZyb250ZW5kUHJpb3JpdGl6YXRpb25TdHJhdGVneSlcbiAgICAgICAgcmV0dXJuIHRoaXMuY29uZmlnLmZyb250ZW5kUHJpb3JpdGl6YXRpb25TdHJhdGVneSh0aGlzLmNvbmZpZylcblxuICAgICAgICAvLyBEZWZhdWx0IFN0cmF0ZWd5OiBSYW5kb20gUGFydGl0aW9uXG4gICAgICAgIGNvbnN0IHJhbmQgPSBNYXRoLnJhbmRvbSgpO1xuICAgICAgICB2YXIgcGlja2VkXG4gICAgICAgIGZvcih2YXIgb3Agb2YgdGhpcy5vcmRlcmVkUHJpb3JpdHkpe1xuICAgICAgICAgICAgaWYocmFuZCA8IG9wLnByb2JhYmlsaXR5KXtcbiAgICAgICAgICAgICAgICBwaWNrZWQgPSBvcC5uYW1lXG4gICAgICAgICAgICAgICAgYnJlYWs7XG4gICAgICAgICAgICB9XG4gICAgICAgIH1cbiAgICAgICAgLy8gTGFzdCBlbGVtZW50IG5vdCBjb3ZlcmVkIGJ5IHRoZSBmb3IgbG9vcCBhYm92ZVxuICAgICAgICBpZighcGlja2VkKXtcbiAgICAgICAgICAgIHBpY2tlZCA9IHRoaXMub3JkZXJlZFByaW9yaXR5W3RoaXMub3JkZXJlZFByaW9yaXR5Lmxlbmd0aC0xXS5uYW1lXG4gICAgICAgIH1cbiAgICAgICBcbiAgICAgICAgcmV0dXJuIHBpY2tlZFxuICAgIH1cblxuICAgIGFzeW5jIHdvcmtPbmNlKGZvcmNlZFByaW9yaXR5Pzogc3RyaW5nKSB7XG4gICAgICAgIGNvbnN0IHF1ZXVlID0gZm9yY2VkUHJpb3JpdHkgP1xuICAgICAgICAgICAgVVJMRnJvbnRpZXIuZ2V0RnJvbnRlbmRRdWV1ZU5hbWUodGhpcy5jb25maWcsIGZvcmNlZFByaW9yaXR5KSA6XG4gICAgICAgICAgICBVUkxGcm9udGllci5nZXRGcm9udGVuZFF1ZXVlTmFtZSh0aGlzLmNvbmZpZywgdGhpcy5waWNrUHJpb3JpdHlRdWV1ZUJ5UHJpb3JpdGl6YXRpb25TdHJhdGVneSgpKTtcblxuICAgICAgICBjb25zdCBwb3BwZWRJdGVtID0gYXdhaXQgdGhpcy5yZWRpc0NsaWVudC5ycG9wKHF1ZXVlKTtcbiAgICAgICAgaWYgKCFwb3BwZWRJdGVtKVxuICAgICAgICAgICAgcmV0dXJuIGZhbHNlXG5cbiAgICAgICAgY29uc3QgeyB1cmwsIG1ldGEgfSA9IEpTT04ucGFyc2UocG9wcGVkSXRlbSk7XG4gICAgICAgIGNvbnN0IHBhcnNlZFVybCA9IG5ldyBVUkwodXJsKTtcblxuICAgICAgICAvLyBTaW5jZSBpb3JlZGlzLW1vY2sgZG9lc24ndCBzdXBwb3J0IG11bHRpKCkgJiB0cmFuc2FjdGlvblxuICAgICAgICBpZihwcm9jZXNzLmVudi5KRVNUX1dPUktFUl9JRCA9PT0gdW5kZWZpbmVkKXtcbiAgICAgICAgICAgIGF3YWl0IHRoaXMucmVkaXNDbGllbnQubXVsdGkoKVxuICAgICAgICAgICAgLmxwdXNoKFVSTEZyb250aWVyLmdldEJhY2tlbmRRdWV1ZU5hbWUodGhpcy5jb25maWcsIHBhcnNlZFVybC5ob3N0bmFtZSksIEpTT04uc3RyaW5naWZ5KHsgdXJsLCBtZXRhIH0pKSAvLyBhZGQgdG8gdGhlIGJhY2tlbmQgcXVldWVcbiAgICAgICAgICAgIC56aW5jcmJ5KFVSTEZyb250aWVyLmdldEhvc3RuYW1lVXJsQ291bnROYW1lKHRoaXMuY29uZmlnKSwgMSwgcGFyc2VkVXJsLmhvc3RuYW1lKSAvLyBJbmNyZW1lbnQgYnkgb25lIHRoZSBob3N0bmFtZSB1cmxzXG4gICAgICAgICAgICAuemFkZChVUkxGcm9udGllci5nZXRIZWFwTmFtZSh0aGlzLmNvbmZpZyksJ05YJywgbmV3IERhdGUoKS5nZXRUaW1lKCksIHBhcnNlZFVybC5ob3N0bmFtZSkgLy8gSW4gY2FzZSBvZiBuZXcgaG9zdG5hbWUgaW4gdGhlIGZyb250aWVyLCBhZGQgaXQgdG8gdGhlIGhlYXBcbiAgICAgICAgICAgIC5leGVjKClcbiAgICAgICAgfWVsc2V7XG4gICAgICAgICAgICBhd2FpdCB0aGlzLnJlZGlzQ2xpZW50LmxwdXNoKFVSTEZyb250aWVyLmdldEJhY2tlbmRRdWV1ZU5hbWUodGhpcy5jb25maWcsIHBhcnNlZFVybC5ob3N0bmFtZSksIEpTT04uc3RyaW5naWZ5KHsgdXJsLCBtZXRhIH0pKSAvLyBhZGQgdG8gdGhlIGJhY2tlbmQgcXVldWVcbiAgICAgICAgICAgIGF3YWl0IHRoaXMucmVkaXNDbGllbnQuemluY3JieShVUkxGcm9udGllci5nZXRIb3N0bmFtZVVybENvdW50TmFtZSh0aGlzLmNvbmZpZyksIDEsIHBhcnNlZFVybC5ob3N0bmFtZSkgLy8gSW5jcmVtZW50IGJ5IG9uZSB0aGUgaG9zdG5hbWUgdXJsc1xuICAgICAgICAgICAgYXdhaXQgdGhpcy5yZWRpc0NsaWVudC56YWRkKFVSTEZyb250aWVyLmdldEhlYXBOYW1lKHRoaXMuY29uZmlnKSwnTlgnLCBuZXcgRGF0ZSgpLmdldFRpbWUoKSwgcGFyc2VkVXJsLmhvc3RuYW1lKSAvLyBJbiBjYXNlIG9mIG5ldyBob3N0bmFtZSBpbiB0aGUgZnJvbnRpZXIsIGFkZCBpdCB0byB0aGUgaGVhcFxuICAgICAgICB9XG4gICAgICAgIHJldHVybiB0cnVlXG4gICAgfVxuXG4gICAgcHJpdmF0ZSBhc3luYyB3b3JrKCkge1xuICAgICAgICB3aGlsZSAodHJ1ZSAmJiAhdGhpcy5zdG9wcGVkKSB7XG4gICAgICAgICAgICBpZiAoIWF3YWl0IHRoaXMud29ya09uY2UoKSkge1xuICAgICAgICAgICAgICAgIC8vIElmIG5vdGhpbmcgZm91bmQgc2xvdyBkb3duIHJlcXVlc3RzIHRvIGF2b2lkIEREb1NpbmcgUmVkaXNcbiAgICAgICAgICAgICAgICBhd2FpdCBuZXcgUHJvbWlzZSgocmVzb2x2ZSwgcmVqZWN0KSA9PiB7XG4gICAgICAgICAgICAgICAgICAgIHNldFRpbWVvdXQocmVzb2x2ZSwgMTAwKVxuICAgICAgICAgICAgICAgIH0pXG4gICAgICAgICAgICB9XG4gICAgICAgIH1cbiAgICB9XG59Il0sIm5hbWVzIjpbIkZyb250ZW5kV29ya2VyIiwic3RhcnQiLCJzdG9wcGVkIiwid29yayIsInN0b3AiLCJwaWNrUHJpb3JpdHlRdWV1ZUJ5UHJpb3JpdGl6YXRpb25TdHJhdGVneSIsImNvbmZpZyIsImZyb250ZW5kUHJpb3JpdGl6YXRpb25TdHJhdGVneSIsInJhbmQiLCJNYXRoIiwicmFuZG9tIiwicGlja2VkIiwib3AiLCJvcmRlcmVkUHJpb3JpdHkiLCJwcm9iYWJpbGl0eSIsIm5hbWUiLCJsZW5ndGgiLCJ3b3JrT25jZSIsImZvcmNlZFByaW9yaXR5IiwicXVldWUiLCJnZXRGcm9udGVuZFF1ZXVlTmFtZSIsInBvcHBlZEl0ZW0iLCJyZWRpc0NsaWVudCIsInJwb3AiLCJ1cmwiLCJtZXRhIiwiSlNPTiIsInBhcnNlIiwicGFyc2VkVXJsIiwiVVJMIiwicHJvY2VzcyIsImVudiIsIkpFU1RfV09SS0VSX0lEIiwidW5kZWZpbmVkIiwibXVsdGkiLCJscHVzaCIsImdldEJhY2tlbmRRdWV1ZU5hbWUiLCJob3N0bmFtZSIsInN0cmluZ2lmeSIsInppbmNyYnkiLCJnZXRIb3N0bmFtZVVybENvdW50TmFtZSIsInphZGQiLCJnZXRIZWFwTmFtZSIsIkRhdGUiLCJnZXRUaW1lIiwiZXhlYyIsIlByb21pc2UiLCJyZXNvbHZlIiwicmVqZWN0Iiwic2V0VGltZW91dCIsImNvbmZpZ3VyYXRpb24iLCJyZWRpc09wdGlvbnMiLCJFcnJvciIsIk9iamVjdCIsImtleXMiLCJwcmlvcml0aWVzIiwibWFwIiwiayIsInNvcnQiLCJhIiwiYiJdLCJtYXBwaW5ncyI6IjtBQUFvQixHQUFTLENBQVQsUUFBUztBQUNELEdBQWUsQ0FBZixZQUFlO0FBSXBDLEdBQUssQ0FBQ0EsY0FBYyxTQUFkQSxjQUFjO0lBNEJoQkMsS0FBSyxHQUFHLENBQUM7UUFDWixJQUFJLENBQUNDLE9BQU8sR0FBRyxLQUFLO1FBQ3BCLElBQUksQ0FBQ0MsSUFBSTtJQUNiLENBQUM7SUFFTUMsSUFBSSxHQUFHLENBQUM7UUFDWCxJQUFJLENBQUNGLE9BQU8sR0FBRyxJQUFJO0lBQ3ZCLENBQUM7SUFFT0cseUNBQXlDLEdBQVcsQ0FBQztRQUN6RCxFQUFrQixBQUFsQixnQkFBa0I7UUFDbEIsRUFBRSxFQUFDLElBQUksQ0FBQ0MsTUFBTSxDQUFDQyw4QkFBOEIsRUFDN0MsTUFBTSxDQUFDLElBQUksQ0FBQ0QsTUFBTSxDQUFDQyw4QkFBOEIsQ0FBQyxJQUFJLENBQUNELE1BQU07UUFFN0QsRUFBcUMsQUFBckMsbUNBQXFDO1FBQ3JDLEtBQUssQ0FBQ0UsSUFBSSxHQUFHQyxJQUFJLENBQUNDLE1BQU07UUFDeEIsR0FBRyxDQUFDQyxNQUFNO1FBQ1YsR0FBRyxFQUFDLEdBQUcsQ0FBQ0MsRUFBRSxJQUFJLElBQUksQ0FBQ0MsZUFBZSxDQUFDLENBQUM7WUFDaEMsRUFBRSxFQUFDTCxJQUFJLEdBQUdJLEVBQUUsQ0FBQ0UsV0FBVyxFQUFDLENBQUM7Z0JBQ3RCSCxNQUFNLEdBQUdDLEVBQUUsQ0FBQ0csSUFBSTtnQkFDaEIsS0FBSztZQUNULENBQUM7UUFDTCxDQUFDO1FBQ0QsRUFBaUQsQUFBakQsK0NBQWlEO1FBQ2pELEVBQUUsR0FBRUosTUFBTSxFQUFDLENBQUM7WUFDUkEsTUFBTSxHQUFHLElBQUksQ0FBQ0UsZUFBZSxDQUFDLElBQUksQ0FBQ0EsZUFBZSxDQUFDRyxNQUFNLEdBQUMsQ0FBQyxFQUFFRCxJQUFJO1FBQ3JFLENBQUM7UUFFRCxNQUFNLENBQUNKLE1BQU07SUFDakIsQ0FBQztVQUVLTSxRQUFRLENBQUNDLGNBQXVCLEVBQUUsQ0FBQztRQUNyQyxLQUFLLENBQUNDLEtBQUssR0FBR0QsY0FBYyxHQWhFUixZQUFlLGFBaUVuQkUsb0JBQW9CLENBQUMsSUFBSSxDQUFDZCxNQUFNLEVBQUVZLGNBQWMsSUFqRTVDLFlBQWUsYUFrRW5CRSxvQkFBb0IsQ0FBQyxJQUFJLENBQUNkLE1BQU0sRUFBRSxJQUFJLENBQUNELHlDQUF5QztRQUVoRyxLQUFLLENBQUNnQixVQUFVLEdBQUcsS0FBSyxDQUFDLElBQUksQ0FBQ0MsV0FBVyxDQUFDQyxJQUFJLENBQUNKLEtBQUs7UUFDcEQsRUFBRSxHQUFHRSxVQUFVLEVBQ1gsTUFBTSxDQUFDLEtBQUs7UUFFaEIsS0FBSyxDQUFDLENBQUMsQ0FBQ0csR0FBRyxHQUFFQyxJQUFJLEVBQUMsQ0FBQyxHQUFHQyxJQUFJLENBQUNDLEtBQUssQ0FBQ04sVUFBVTtRQUMzQyxLQUFLLENBQUNPLFNBQVMsR0FBRyxHQUFHLENBQUNDLEdBQUcsQ0FBQ0wsR0FBRztRQUU3QixFQUEyRCxBQUEzRCx5REFBMkQ7UUFDM0QsRUFBRSxFQUFDTSxPQUFPLENBQUNDLEdBQUcsQ0FBQ0MsY0FBYyxLQUFLQyxTQUFTLEVBQUMsQ0FBQztZQUN6QyxLQUFLLENBQUMsSUFBSSxDQUFDWCxXQUFXLENBQUNZLEtBQUssR0FDM0JDLEtBQUssQ0E5RVUsWUFBZSxhQThFWkMsbUJBQW1CLENBQUMsSUFBSSxDQUFDOUIsTUFBTSxFQUFFc0IsU0FBUyxDQUFDUyxRQUFRLEdBQUdYLElBQUksQ0FBQ1ksU0FBUyxDQUFDLENBQUM7Z0JBQUNkLEdBQUc7Z0JBQUVDLElBQUk7WUFBQyxDQUFDLEVBQUcsQ0FBMkIsQUFBM0IsRUFBMkIsQUFBM0IseUJBQTJCO2FBQ2xJYyxPQUFPLENBL0VRLFlBQWUsYUErRVZDLHVCQUF1QixDQUFDLElBQUksQ0FBQ2xDLE1BQU0sR0FBRyxDQUFDLEVBQUVzQixTQUFTLENBQUNTLFFBQVEsQ0FBRSxDQUFxQyxBQUFyQyxFQUFxQyxBQUFyQyxtQ0FBcUM7YUFDdEhJLElBQUksQ0FoRlcsWUFBZSxhQWdGYkMsV0FBVyxDQUFDLElBQUksQ0FBQ3BDLE1BQU0sR0FBRSxDQUFJLEtBQUUsR0FBRyxDQUFDcUMsSUFBSSxHQUFHQyxPQUFPLElBQUloQixTQUFTLENBQUNTLFFBQVEsQ0FBRSxDQUE4RCxBQUE5RCxFQUE4RCxBQUE5RCw0REFBOEQ7YUFDeEpRLElBQUk7UUFDVCxDQUFDLE1BQUksQ0FBQztZQUNGLEtBQUssQ0FBQyxJQUFJLENBQUN2QixXQUFXLENBQUNhLEtBQUssQ0FuRlosWUFBZSxhQW1GVUMsbUJBQW1CLENBQUMsSUFBSSxDQUFDOUIsTUFBTSxFQUFFc0IsU0FBUyxDQUFDUyxRQUFRLEdBQUdYLElBQUksQ0FBQ1ksU0FBUyxDQUFDLENBQUM7Z0JBQUNkLEdBQUc7Z0JBQUVDLElBQUk7WUFBQyxDQUFDLEVBQUcsQ0FBMkIsQUFBM0IsRUFBMkIsQUFBM0IseUJBQTJCOztZQUN6SixLQUFLLENBQUMsSUFBSSxDQUFDSCxXQUFXLENBQUNpQixPQUFPLENBcEZkLFlBQWUsYUFvRllDLHVCQUF1QixDQUFDLElBQUksQ0FBQ2xDLE1BQU0sR0FBRyxDQUFDLEVBQUVzQixTQUFTLENBQUNTLFFBQVEsQ0FBRSxDQUFxQyxBQUFyQyxFQUFxQyxBQUFyQyxtQ0FBcUM7O1lBQzdJLEtBQUssQ0FBQyxJQUFJLENBQUNmLFdBQVcsQ0FBQ21CLElBQUksQ0FyRlgsWUFBZSxhQXFGU0MsV0FBVyxDQUFDLElBQUksQ0FBQ3BDLE1BQU0sR0FBRSxDQUFJLEtBQUUsR0FBRyxDQUFDcUMsSUFBSSxHQUFHQyxPQUFPLElBQUloQixTQUFTLENBQUNTLFFBQVEsQ0FBRSxDQUE4RCxBQUE5RCxFQUE4RCxBQUE5RCw0REFBOEQ7O1FBQ25MLENBQUM7UUFDRCxNQUFNLENBQUMsSUFBSTtJQUNmLENBQUM7VUFFYWxDLElBQUksR0FBRyxDQUFDO2NBQ1gsSUFBSSxLQUFLLElBQUksQ0FBQ0QsT0FBTyxDQUFFLENBQUM7WUFDM0IsRUFBRSxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUNlLFFBQVEsSUFBSSxDQUFDO2dCQUN6QixFQUE2RCxBQUE3RCwyREFBNkQ7Z0JBQzdELEtBQUssQ0FBQyxHQUFHLENBQUM2QixPQUFPLEVBQUVDLE9BQU8sRUFBRUMsTUFBTSxHQUFLLENBQUM7b0JBQ3BDQyxVQUFVLENBQUNGLE9BQU8sRUFBRSxHQUFHO2dCQUMzQixDQUFDO1lBQ0wsQ0FBQztRQUNMLENBQUM7SUFDTCxDQUFDO2dCQXpGV0csYUFBNEcsQ0FBRSxDQUFDO1FBQ3ZILEtBQUssQ0FBQyxDQUFDNUMsQ0FBQUEsTUFBTSxHQUFFZ0IsV0FBVyxHQUFFNkIsWUFBWSxFQUFBLENBQUMsR0FBR0QsYUFBYTtRQUN6RCxFQUFFLEdBQUc1QyxNQUFNLEVBQ1AsS0FBSyxDQUFDLEdBQUcsQ0FBQzhDLEtBQUssQ0FBQyxDQUFnRTtRQUNwRixFQUFFLEdBQUdELFlBQVksS0FBSzdCLFdBQVcsSUFBSTZCLFlBQVksSUFBSTdCLFdBQVcsRUFDNUQsS0FBSyxDQUFDLEdBQUcsQ0FBQzhCLEtBQUssQ0FBQyxDQUF3RztRQUU1SCxJQUFJLENBQUM5QyxNQUFNLEdBQUdBLE1BQU07UUFDcEIsSUFBSSxDQUFDSixPQUFPLEdBQUcsS0FBSztRQUVwQixFQUFtQyxBQUFuQyxpQ0FBbUM7UUFDbkMsSUFBSSxDQUFDVyxlQUFlLEdBQUd3QyxNQUFNLENBQUNDLElBQUksQ0FBQyxJQUFJLENBQUNoRCxNQUFNLENBQUNpRCxVQUFVLEVBQUVDLEdBQUcsRUFBRUMsQ0FBQyxHQUFLLENBQUM7WUFDbkUsTUFBTSxDQUFDLENBQUMzQztnQkFBQUEsV0FBVyxFQUFFLElBQUksQ0FBQ1IsTUFBTSxDQUFDaUQsVUFBVSxDQUFDRSxDQUFDLEVBQUUzQyxXQUFXO2dCQUFFQyxJQUFJLEVBQUUwQyxDQUFDO1lBQUEsQ0FBQztRQUN4RSxDQUFDLEVBQUVDLElBQUksRUFBRUMsQ0FBQyxFQUFDQyxDQUFDLEdBQUtELENBQUMsQ0FBQzdDLFdBQVcsR0FBQzhDLENBQUMsQ0FBQzlDLFdBQVc7O1FBRTVDLEVBQUUsRUFBRXFDLFlBQVksRUFDWixJQUFJLENBQUM3QixXQUFXLEdBQUcsR0FBRyxDQTNCZCxRQUFTLFNBMkJjNkIsWUFBWTtRQUUvQyxFQUFFLEVBQUU3QixXQUFXLEVBQ1gsSUFBSSxDQUFDQSxXQUFXLEdBQUdBLFdBQVc7SUFDdEMsQ0FBQzs7UUExQlF0QixjQUFjLEdBQWRBLGNBQWMifQ==