locusbuilder-utility/server/modules/polling/index.js

Locusbuilder Utility server for asset scraping and processing

/**
 * Polling.js
 * @author Richard Abear <owchzzz@gmail.com>
 * 
 * Polling module to make queues for phantom processing
 */

 // configuration object
 // jshint ignore: start
 const config = require('./config.js');
 
 const HtmlParser = require('../htmlparser');
 const Optimizer = require('../optimizer');
 class Polling {
     constructor(config) {
        let c = this.config = config; 
        this.polls = [];
        for(var i = 0; i < c.max; i++) {
            console.log('Adding poll: ' + parseInt(i));
            let poll = Promise.resolve([]);
            poll.htmlparser = new HtmlParser({id:i});
            poll.htmlparser.init();
            this.polls.push(poll);
        }

        this.activePoll = 0;

     }

     poll(url) {
         let self = this;
        var poll = this.polls[this.activePoll];
        var htmlparser = poll.htmlparser;
        return poll.then( async (val) => {
            console.log('Evaluating '+ url + 'On poll: ' + self.activePoll);
            let content = await htmlparser.openPage(url);

            let $ = await Optimizer({
                'content': content
            });
    
            let images = await $.findAll({search:'img', searchattr: 'src'});

            // Update the active poll
            self.activePoll = (this.activePoll < this.config.max - 1) ? this.activePoll+1 : 0;
            return images;


        });
    
        
     }
 }

 module.exports = Polling;