spas

Version:

spas is a pragmatic tool for bundling multiple API requests into a single request/response for the end user. spas throttles, caches, parses, filters, concatenates and minifies API responses. It serves them all up in one tidy little package resulting in fe

dtex.github.io/spas/

dtex/spas

197 lines (166 loc) • 65.5 kB

HTML

<!DOCTYPE html> <html> <head> <title>engine.js</title> <meta http-equiv="content-type" content="text/html; charset=UTF-8"> <link rel="stylesheet" media="all" href="docco.css" /> </head> <body> <div id="container"> <div id="background"></div> <table cellpadding="0" cellspacing="0"> <thead> <tr> <th class="docs"> <h1> engine.js </h1> </th> <th class="code"> </th> </tr> </thead> <tbody> <tr id="section-1"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-1">¶</a> </div> </td> <td class="code"> <div class="highlight"><pre>var nconf = require('nconf') , redis = require("redis") , _ = require('underscore')._ , neuron = require('neuron')</pre></div> </td> </tr> <tr id="section-2"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-2">¶</a> </div> , crypto = require('crypto') , sha = crypto.createHash('sha1') , zlib = require('zlib') </td> <td class="code"> <div class="highlight"><pre>; require('date-utils'); </pre></div> </td> </tr> <tr id="section-3"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-3">¶</a> </div> Use nconf to grab commandline params and read config.json </td> <td class="code"> <div class="highlight"><pre>nconf.argv().file({ file: 'config.json' });</pre></div> </td> </tr> <tr id="section-4"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-4">¶</a> </div> If --dev is passed use the development config section </td> <td class="code"> <div class="highlight"><pre>var config = nconf.get('dev') ? nconf.get('development') : nconf.get('live');</pre></div> </td> </tr> <tr id="section-5"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-5">¶</a> </div> Connect to redis </td> <td class="code"> <div class="highlight"><pre>var client = redis.createClient(config.redis.port, config.redis.address); if (nconf.get(config.redis.auth)) { client.auth(config.redis.auth, function (err) { if (err) { </pre></div> </td> </tr> <tr id="section-6"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-6">¶</a> </div> handle err; </td> <td class="code"> <div class="highlight"><pre> } }); }</pre></div> </td> </tr> <tr id="section-7"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-7">¶</a> </div> <h2>Recursive function to remove unwanted elements from API response</h2> </td> <td class="code"> <div class="highlight"><pre>var filter = function( source, map) { if (_.isArray(source)) { _.each(source, function(item, index) { filter( item, map[0]); }); } else { if (_.isString(source) || map === true || _.isUndefined(map)) return 0; _.each(source, function(obj, key, source) { if (_.isUndefined(map[key])) { delete source[key]; } else { filter( obj, map[key]); } }); }; }</pre></div> </td> </tr> <tr id="section-8"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-8">¶</a> </div> <h2>Perform scheduled refresh</h2> </td> <td class="code"> <div class="highlight"><pre>exports.refresh = function(api, key, bid, bundle) {</pre></div> </td> </tr> <tr id="section-9"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-9">¶</a> </div> We're forcing a refresh of the content so run the api.code </td> <td class="code"> <div class="highlight"><pre> api.resource( api.params, api.credentials, function( err, res ) { if ( err ) {</pre></div> </td> </tr> <tr id="section-10"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-10">¶</a> </div> We got an error so set our output object to be the error and expire immediately </td> <td class="code"> <div class="highlight"><pre> api.expires = ( new Date() ); var tout = { expires: api.expires, result: err, iid: bid+key, cname: key, scheduled: true }; } else { </pre></div> </td> </tr> <tr id="section-11"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-11">¶</a> </div> Filter the response </td> <td class="code"> <div class="highlight"><pre> if (_.has(api.filter)) { filter ( res, api.filter ); } </pre></div> </td> </tr> <tr id="section-12"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-12">¶</a> </div> Perform cleanup function on API response </td> <td class="code"> <div class="highlight"><pre> if (_.has(api, 'cleanup')) { res = api.cleanup(res); } </pre></div> </td> </tr> <tr id="section-13"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-13">¶</a> </div> Build the stored response </td> <td class="code"> <div class="highlight"><pre> api.expires = ( new Date() ).addSeconds( api.cacheduration ); bundle[key] = api; var tout = { expires: api.expires, result: res, iid: api.iid, cname: key, scheduled: true }; </pre></div> </td> </tr> <tr id="section-14"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-14">¶</a> </div> Save response to Redis </td> <td class="code"> <div class="highlight"><pre> client.set(bid+key, JSON.stringify(tout)); </pre></div> </td> </tr> <tr id="section-15"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-15">¶</a> </div> Update the bundle </td> <td class="code"> <div class="highlight"><pre> client.del('bundle'+bid); } }); }</pre></div> </td> </tr> <tr id="section-16"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-16">¶</a> </div> <h2>Retrieve the requested bundle</h2> </td> <td class="code"> <div class="highlight"><pre>exports.fulfill = function ( myRes, bid, bundle, callback, override ) { if (typeof bundle === 'undefined') { myRes.writeHead(404); myRes.end(); return false; } </pre></div> </td> </tr> <tr id="section-17"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-17">¶</a> </div> Count the number of queries in this bundle so we know when we are ready to respond </td> <td class="code"> <div class="highlight"><pre> var queriesInThisBundle = _.size(bundle), thisResponse = {}; </pre></div> </td> </tr> <tr id="section-18"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-18">¶</a> </div> cleanup is not an API request </td> <td class="code"> <div class="highlight"><pre> if(_.has(bundle, 'cleanup')) { queriesInThisBundle--; } </pre></div> </td> </tr> <tr id="section-19"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-19">¶</a> </div> If override was not passed </td> <td class="code"> <div class="highlight"><pre> if( _.isUndefined( override )) { </pre></div> </td> </tr> <tr id="section-20"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-20">¶</a> </div> Retrieve bundle response from Redis </td> <td class="code"> <div class="highlight"><pre> client.get('bid'+bid, function ( err, doc ) { if ( err || doc === null ) {</pre></div> </td> </tr> <tr id="section-21"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-21">¶</a> </div> There was an error so force refresh on bundle </td> <td class="code"> <div class="highlight"><pre> exports.fulfill( myRes, bid, bundle, callback, true ); } else { doc = JSON.parse( doc ); </pre></div> </td> </tr> <tr id="section-22"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-22">¶</a> </div> If there is a valid expiration date for the bundle </td> <td class="code"> <div class="highlight"><pre> doc.expires = new Date(doc.expires); if ( 'expires' in doc && _.isDate(doc.expires) ) { doc.secleft = doc.expires.getSecondsBetween( new Date() ) * -1; } else {</pre></div> </td> </tr> <tr id="section-23"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-23">¶</a> </div> This should never happen </td> <td class="code"> <div class="highlight"><pre> doc.secleft = 0; } if (Number(doc.secleft) < 0 ) { </pre></div> </td> </tr> <tr id="section-24"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-24">¶</a> </div> The bundle has expired. Force a refresh </td> <td class="code"> <div class="highlight"><pre> exports.fulfill( myRes, bid, bundle, callback, true ); } else { </pre></div> </td> </tr> <tr id="section-25"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-25">¶</a> </div> Respond with the cached data </td> <td class="code"> <div class="highlight"><pre> doc.fromcache = true; var responseType = callback ? 'application/javascript' : 'application/json'; myRes.writeHead(200, {'Content-Type': responseType, 'max-age': doc.secleft, 'cache-control': 'public, max-age='+doc.secleft+', no-transform', "Expires": doc.expires }); </pre></div> </td> </tr> <tr id="section-26"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-26">¶</a> </div> If a callback name was passed, use it. Otherwise, just output the object </td> <td class="code"> <div class="highlight"><pre> var tbuf = new Buffer( ( !callback ) ? JSON.stringify( doc ) : callback + '(' + JSON.stringify( doc ) + ');' ); myRes.end(tbuf); } } }); } else { </pre></div> </td> </tr> <tr id="section-27"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-27">¶</a> </div> <h3>Override was passed so wew ar eforcing a refresh</h3> </td> <td class="code"> <div class="highlight"><pre> var manager = new neuron.JobManager(); manager.on('empty', function (job) { if (job.name === 'finishRequest' && queriesInThisBundle === 0) { var done = true; _.each(manager.job, function(job, key) { if (_.size(job.waiting) !== 0 || _.size(job.running) !== 0 || job.queue.length !== 0) { done = false; } }); if (done) manager.enqueue('sendResponse', bid); } }); manager.addJob('fulfillPart', { work: function(api, key, override, cachedPart) { var self = this; if ( _.isUndefined( override ) ) { </pre></div> </td> </tr> <tr id="section-28"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-28">¶</a> </div> Load the cached api response from Redis </td> <td class="code"> <div class="highlight"><pre> client.get(bid+key, function (err, doc) { if (err || doc === null){ self.finished = true; manager.enqueue('fulfillPart', api, key, true ); } else { doc = JSON.parse( doc ); doc.expires = new Date(doc.expires); if ( ('expires' in doc) && _.isDate(doc.expires) ) { var secleft = doc.expires.getSecondsBetween( new Date() ) * -1; } if (secleft < 0) { self.finished = true; manager.enqueue('fulfillPart', api, key, true, doc ); } else { doc.fromcache = true; manager.enqueue('finishRequest', doc ); self.finished = true; } } }); } else { if (_.has( api, 'auth')) { </pre></div> </td> </tr> <tr id="section-29"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-29">¶</a> </div> If the API request object has an auth scheme defined </td> <td class="code"> <div class="highlight"><pre> api.auth.type.authorize (api, bid, key, function( result ) { if (result === true) { manager.enqueue('startRequest', api, key, cachedPart); } else { manager.enqueue('finishRequest', result ); } self.finished = true; }); } else {</pre></div> </td> </tr> <tr id="section-30"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-30">¶</a> </div> Authentication is not needed </td> <td class="code"> <div class="highlight"><pre> self.finished = true; manager.enqueue('startRequest', api, key, cachedPart); } } } }); manager.addJob('startRequest', { work: function( api, key, cachedPart ) { var self = this; if (_.has( api, 'timeout')) { self.timeout = setTimeout(function(self) { manager.enqueue('finishRequest', cachedPart ); if(_.has(cachedPart)) { cachedPart.timeout = true; cachedPart.fromcache = true; }; self.finished = true; }, api.timeout, self) } api.resource( api.params, api.credentials, function( err, res ) { clearTimeout(self.timeout) delete self.timeout; if ( err ) { api.expires = ( new Date() ); tout = _.isUndefined(cachedPart) ? {} : cachedPart; tout.cname = key; tout.expires = api.expires; tout.fromcache = true; tout.err = err; } else { </pre></div> </td> </tr> <tr id="section-31"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-31">¶</a> </div> Filter the response </td> <td class="code"> <div class="highlight"><pre> if (_.has(api, 'filter')) { filter ( res, api.filter ); } </pre></div> </td> </tr> <tr id="section-32"> <td class="docs"> <div class="pilwrap"> <a class="pilcrow" href="#section-32">¶</a> </div> Perform cleanup function on API response </td> <td class="