celeritas/celeritas.js

This is an API service framework which supports API requests over HTTP & WebSockets.

'use strict';

const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const winston = require('winston');
const uuid = require('uuid');
const crypto = require("crypto");
const cluster = require('cluster');
const os = require('os');

const newrelic = (fs.existsSync("./newrelic.js")) ? require('newrelic') : false;

const Call = require('./lib/call.js');
const CallUtilities = require("./lib/utilities.js");
CallUtilities.validate = require("./lib/validate.js");

const CeleritasWS = require("./lib/websocketserver.js");
const Router = require("./lib/router.js");

/*
	@Celeritas: Creates an instance of class Celeritas; Celeritas is an application container which runs a standard HTTPS/WSS API service. Celeritas should be a singleton.
	arguments: {
		options: {
			api: {
				route: Default route is "api/v1/", this defines the path to access APIs exposed by your application, e.g. "GET http://api.example.com/api/v1/API"
				directory: The directory the application searches for to find API methods. e.g. "./apis",
				timeout: Time in MS before a request to the API times out,
				metaData: TRUE|FALSE, if true, api responses will include meta data about the API result. Otherwise, the response will only contain the contents of the result array,
				castResultsAsArray: TRUE|FALSE, if true, output.data.result will always be an array. Otherwise, it is the raw response.
			},
			autoload: This autoloads modules from directories in your app folder, and assigns them to the key passed. [
				path: directory
			],
			server: {
				protocol: "http" or "https"
				port: 8443; the desired TCP port the api should be available on.
				ws: true|false; if http pass-through via websocket is supported.
			},
			cluster: {
				enabled: true|false, determines if the app will run as a cluster of nodes rather than a single 1-core app,
				forks: the number of child processes that will run the app.
			},
			debug: true|false, determines if debug mode is on for not.
			accessControl: default access control headers for HTTP requests that the API serves. {
				allowOrigin: "*",
				allowHeaders: 'Content-Type, Authorization, Accept',
				exposeHeaders: 'Content-Type, Cache-Control, Authorization',
				allowMethods: "GET, POST, PUT, DELETE, OPTIONS"
			},
			
			//Any other options you define will also be loaded into the Celeritas instance.
		}
	}
*/
class Celeritas {
	constructor (options) {
		var version = require('./package.json').version;

		var numCPUs = os.cpus().length;
		var defaults = {
			id: uuid(),
			name: "celeritas@" + version,
			debug: false,
			newrelic: newrelic,
			validation: "native",
			logMode: "VERBOSE", //QUIET or VERBOSE
			api: {
				route: "v1/",
				directory: "./routes",
				timeout: 60 * 1000,
				metaData: true,
				castResultsAsArray: false,
				alphabetizeResponse: true,
				completeHandler: null
			},
			server: {
				protocol: "https",
				port: process.env.PORT || 8080,
				ws: true,
				enforceHttps: false
			},
			cluster: {
				enabled: (numCPUs !== 1),
				forks: (numCPUs -1) || 1
			},
			accessControl: {
				allowOrigin: "*",
				allowMethods: "GET, POST, PUT, DELETE, OPTIONS",
				allowHeaders: 'Content-Type, Cache-Control, Authorization, Accept, Accept-Encoding, X-Return-Input',
				exposeHeaders: 'Content-Type, Cache-Control, Authorization, X-Request-Id, X-Powered-By'
			},
			blacklist: {
				uris: fs.existsSync("./blacklist.txt") === true ? fs.readFileSync('./blacklist.txt').toString().split("\r\n") : []
			},
			preload: () => {}
		}

		var autoload = options.autoload;
		delete options.autoload;
	
		for (var i in defaults) {
			if (typeof defaults[i] == "object" || typeof options[i] == "object")
				Object.assign(defaults[i], options[i] || {});
			else
				defaults[i] = options[i];
		}

		for (var i in options) {
			if (typeof defaults[i] == "undefined")
				defaults[i] = options[i];
		}

		Object.assign(this, defaults);

		for (var i in autoload)
			this._autoload(autoload[i], i);

		this._version = version;

		if (!fs.existsSync("./logs"))
			fs.mkdirSync("./logs", parseInt('0744', 8));

		this._log = new (winston.Logger)({
		    transports: [
		        new (winston.transports.Console)({
		            timestamp: true,
		            colorize: true
		        }),
		        new (winston.transports.File)({
		        	name: "Info-Log",
		        	timestamp: true,
		        	filename: "./logs/infos.log",
		        	level: 'info'
		        }),
		         new (winston.transports.File)({
		         	name: "Warning-Log",
		        	timestamp: true,
		        	filename: "./logs/warnings.log",
		        	level: 'warning'
		        }),
		        new (winston.transports.File)({
		        	name: "Error-Log",
		        	timestamp: true,
		        	filename: "./logs/errors.log",
		        	level: 'error'
		        })
		    ]
		});

		this._calls = {};

		this._createAPIRoutes(this.api.directory);

		this._startup();
	}

	async _startup () {
		if (typeof this.preload == "function")
			await this.preload(this);

		//this function will start the http/https/ws listening.
		let init = (emit = true) => {
			//Start up either an HTTP server or HTTPS server.
			var server = this["_" + this.server.protocol]();
			//Start up a Websocket server using the http(s) server created.
			if (this.server.ws !== false)
				this._ws(server);

			if (emit)
				this._welcomeMessage();
		}

		//If clustering is enabled, run as a cluster, with the HTTP/WS services running on worker processes. Otherwise, run them as usual.
		if (this.cluster.enabled === true) {
			if (cluster.isMaster) {
				//if cluster is enabled, the master thread does not listen on any ports, and is reserved for managing the forked processes.
				this._log.info(`${this.name} is running as a cluster. Master process '${process.pid}' has started.`);

				for (let i = 0; i < this.cluster.forks; i++) {
					//Slightly delay forking the cluster so logging doesn't get all fucked from async stream writes.
					setTimeout(cluster.fork, i * 150);
				}

				cluster.on('exit', (worker, code, signal) => {
					this._log.info(`${this.name} cluster fork '${worker.process.pid}' has died.`);
					cluster.fork();
				});

				this._welcomeMessage();
			}
			else {
				this._log.info(`${this.name} cluster fork '${process.pid}' has started.`);
				init(false);
			}
		}
		else
			init(true);

		//For some extra debugging of promises and exceptions.
		if (this.debug) {
			process.on("unhandledRejection", (err) => {
				this._log.error(err);
				console.error(err);
				if (this.newrelic)
					this.newrelic.noticeError(err);
				process.exit();
			});

			process.on('uncaughtException', (err) => {
				this._log.error(err);
				console.log(err);
				if (this.newrelic)
					this.newrelic.noticeError(err);
				process.exit();
			})
		}
	}

	_welcomeMessage () {
		//var port = (typeof this.server.redirect !== "undefined") ? this.server.https_port : this.server.http_port;
		var host = "http://localhost:" + this.server.port + "/";// + this.api.route;
		var border = ""; for (var i = 0; i < host.length; i++) border += "=";
		var space = ""; for (var i = 0; i < host.length; i++) space += " ";
		console.log("\n     Started " + this.name + "! Your API is available at this URL: -\n");
		console.log("     O==" + border + "==O");
		console.log("     |  " + space  + "  |");
		console.log("     |  " + host   + "  |");
		console.log("     |  " + space  + "  |");
		console.log("     O==" + border + "==O");
		console.log(`\n Machine hostname: ${os.hostname()}: CPU#: ${os.cpus().length}`)
		//console.log("\n   API Routes: - \n");
		//console.log(this.routes);
		console.log("\n");
	}

	//This method determines what the root directory of the application is, since Azure App Service runs node.js in a weird way.
	_appDir () {
		if (process.env.APP_DIR)
			return process.env.APP_DIR;
		var iisPaths = ["D:\\Program Files\\iisnode", "D:\\Program Files (x86)\\iisnode"];
		var app_dir = path.dirname(require.main.filename);

		if (app_dir in iisPaths)
			return "D:\\home\\site\\wwwroot";
		else
			return app_dir;
	}

	//This method autoloads files from a directory. The modules they export are assigned as properties of this[propertyname]. E.g. this.domains.users == module;
	_autoload (directory, propertyName) {
		this[propertyName] = this[propertyName] || {};

		var iterate = (directory) => {
			if (fs.existsSync(directory)) {
				var files = fs.readdirSync(directory);
				for (var f = 0 ; f < files.length; f++) {
					if (files[f].indexOf("_") === -1) {
						if (fs.lstatSync(directory + "/" + files[f]).isDirectory()) {
							iterate(directory + "/" + files[f])
						}
						else {
							var file = path.normalize(this._appDir() + "/" + directory.substring(1) + "/" + files[f]);
							this[propertyName][files[f].split(".")[0]] = require(file);
							var thing = this[propertyName][files[f].split(".")[0]];
							try {
								thing();
							}
							catch (err) {
								try {
									this[propertyName][files[f].split(".")[0]] = new thing(this);
								}
								catch (err) {
									if (err.message != "thing is not a constructor")
										throw err;
								}
							}
						}
					}
				}
			}
		}

		iterate(directory);
	}


	//This method accepts a directory to search for files within. Each file will be parsed for API routes.
	_createAPIRoutes (directory) {
		this.routes = this.routes || {};

		var iterate = (directory) => {
			if (fs.existsSync(directory)) {
				var routes = fs.readdirSync(directory);
				for (var e = 0; e < routes.length; e++) {
					if (routes[e].indexOf("_") === -1) {
						if (fs.lstatSync(directory + "/" + routes[e]).isDirectory()) {
							iterate(directory + "/" + routes[e]);
						}
						else {
							var file = path.normalize(this._appDir() + "/" + directory.substring(1) + "/" + routes[e]);
							var apiRoutes = require(file).routes;
							for (var api in apiRoutes)
								this.routes[this.api.route + api] = apiRoutes[api];
						}
					}
				}
			}
			else 
				throw new Error("Cannot create API routes from directory '" + directory + "'; directory doesn't exist.");
		}

		iterate(directory);
		
	}
	
	//Sets up the HTTP server for serving HTTP requests. (HTTP protocol)
	_http () {
		this.server.protocol = "http";
		this.server.http = http.createServer(async (req, res) => {
			let call = await this._request("http", req, res);
			this._defaultCompleteHandler(call);
		}).listen(this.server.port);
		return this.server.http;
	}

	//Sets up the HTTP server for serving HTTP requests. (HTTPS protocol), this method supports TLS. Creates HTTP instead of certificate/key not available.
	_https () {
		try {
			var credentials = {key: fs.readFileSync('key.pem'), cert:  fs.readFileSync('cert.pem')};
		}
		catch (err) {
			return this._http();
		}

		this.server.http = https.createServer(credentials, async (req, res) => {
			let call = await this._request("http", req, res);
			this._defaultCompleteHandler(call);
		}).listen(this.server.port);

		//redirect all HTTP traffic to HTTPS.
		//this.server.redirect = http.createServer('*', (req, res) => {
		//	res.redirect('https://' + req.headers.host + req.url);
		//}).listen(this.server.http_port);

		return this.server.https;
	}

	//Sets up the WebSocket server for sending and receiving websocket messages.
	_ws (httpServer) {
		this.server.ws = new CeleritasWS(this, httpServer);
		return this.server.ws;
	}

	//This method handles incoming requests from either HTTP or WebSocket, wraps it all up as a class Call, which processes the API request.
	/*
		arguments {
			type: "http" or "ws", indicates if the message came from an HTTP request or a Websocket message.
			request: if type=="http", it is the Http Request object. If type=="ws", it is the websocket JSON message.
			response: If type=="http", it is the response object. If type=="ws", it is the websocket client.
		}
	*/
	async _request (type, request, response) {
		try {
			switch (type) {
				case "http": var args = {type: type, request: request, response: response, util: CallUtilities}; break;
				case "ws": var args = {type: type, message: request, client: response, util: CallUtilities}; break;
			}

			var trackInNewrelic = true;

			try {
				var bannedEndings = [
					'.php',
					'.aspx',
					'.ico',
					'.html',
					'.js',
					'.cfg',
					'phpmyadmin',
					'dbadmin'
				];
	
				if (type != "ws") {
					var route = request.url.split("?").shift().substring(1).toLowerCase();
		
					for (let f of bannedEndings) {
						if (route.endsWith(f) == true)
							trackInNewrelic = false;
					}
				}
			}
			catch (err) {
				console.warn(err);
			}

			if (this.newrelic && trackInNewrelic === true) {
				return new Promise((resolve, reject) => {
					this.newrelic.startWebTransaction("", async () => {
						let transaction = this.newrelic.getTransaction();
						let call = await new Call(this, args)
						this.newrelic.endTransaction(transaction);
						return resolve(call);
					})
				});
			}
			else {
				let call = await new Call(this, args);
				return call;		
			}
		}
		catch (err) {
			console.warn(err);
		}
	}

	//wehn a call completes, this will run by default. This occurs after the new relic web transaction has completed and the api response has been emitted.
	async _defaultCompleteHandler (call) {
		if (typeof this.api.completeHandler == "function") {
			//if we're using new relic, track this as a background job.
			if (this.newrelic) {
				return new Promise((resolve, reject) => {
					this.newrelic.startBackgroundTransaction("", async () => {
						let transaction = this.newrelic.getTransaction();
						try {
							var job = await this.api.completeHandler(call);
						}
						catch (err) {
							this.newrelic.noticeError(err);
						}
						this.newrelic.endTransaction(transaction);
						return job;
					})
				});
			}
			else {
				try {
					return await this.api.completeHandler(call);
				}
				catch (err) {
					console.error(err);
				}
			}
		}
	}

	async trace (name, handler) {
		return new Promise((resolve, reject) => {
			try {
				if (this.newrelic) {
					this.newrelic.startSegment(name, false, async () => {
						return resolve(await handler());
					});
				}
				else 
					return resolve(handler());
			}
			catch (err) {
				return reject(err);
			}
		})
		
	}

	//Cache an API call response, to be used by others requesting this data.
	cache (call) {
		var t = Date.now();
		var key = this._cacheKey(call);
		var toCache = {
			from: t,
			til: t + (call.api.cache*1000),
			output: call.output
		};

		if (this.redis && this.redis.constructor.name == "RedisClient") {
			this.redis.set(key, JSON.stringify(toCache));
			this.redis.expireAsync(key, call.api.cache*1000);
		}
		else {
			this._cache = this._cache || {};
			this._cache[key] = toCache;
	
			//Once per second, check if cached data should be cleared.
			if (typeof this._cacheTIL == "undefined") {
				this._cacheTIL = setInterval(() => {
					for (var i in this._cache) {
						if (this._cache[i].til < Date.now())
							delete this._cache[i];
					}
				}, 1000);
			}
		}		
	}

	//Retrievs a cached API response.
	retrieve (call, maxAge) {
		this._cache = this._cache || {};

		var adjustCachedResponse = function (call, key, cachedData) {
			cachedData.output.data.meta.requestId = uuid();
			cachedData.output.data.meta.time = {
				emittedAtUtc: new Date().toISOString(),
				receivedAtUtc: call.timestamp,
				totalTimeInMS: (Date.now() - Date.parse(call.timestamp))
			};
			//cachedData.output.data.meta.cacheKey = key;
			return cachedData;
		}

		var key = this._cacheKey(call);
		if (this.redis && this.redis.constructor.name == "RedisClient") {
			return new Promise((resolve, reject) => {
				this.redis.get(key, (err, cachedData) => {
					if (err || cachedData === null)
						return resolve(null);

					cachedData = JSON.parse(cachedData);
					cachedData = adjustCachedResponse(call, key, cachedData);
					call.output = cachedData.output;
					return resolve(cachedData);
				});
			});
		}
		else if (typeof this._cache[key] !== "undefined" && this._cache[key].til > Date.now()) {
			var cachedData = adjustCachedResponse(call, key, this._cache[key]);
			call.output = cachedData.output;
			return Promise.resolve(cachedData);
		}

		return Promise.resolve(null);
	}

	//Returns a key which uniquely identifies a cache-able call, used by the "cache" and "retrieve" methods.
	_cacheKey (call) {
		var key = JSON.stringify({
			method: call.method,
			route: call.route,
			get: call.get,
			post: call.post,
			auth: crypto.createHash('sha512').update(JSON.stringify(call.auth)).digest('utf-8')
		});

		return new Buffer(key).toString("base64");
	}

	//This is an asynchronous method which returns a promise. This method is called to authenticate the user, and either allow the request to continue, or reject it based on how the method proceeds.
	//It takes in the request Call, and should resolve to a User object, which describes the currently authenticated user, or resolve to false if the user is not authenticated.
	//In this method, you should describe how the permissions you pass to _initializeAPIEndpoint affects who can access that API service.
	authenticate (call) {
		if (call.api.permissions === true)
			return Promise.resolve(true);
		else
			return Promise.resolve(false);
	}

	//the 'replay' function triggers when a websocket client connects, authenticates or subcribes. This function is intended as a way to "replay" events or messages that a websocket client might have missed.
	//for example, you could have DB "events". When a event occurs, it could be broadcast to clients subscribed to that type of event.
	//if any clients that are subscribed to that event "miss" that event (because the connection is closed), you could write this function to "replay" the event to the client.
	replay (client) {
		//client.user contains the details of the user object.
		//use this method to get cross-reference your events, and the client's subscriptions to determine if any events were missed. Then use 'client.event' to send the event through.
		//client.event returns a promise, if it resolves to true, then the message was delivered. Otherwise the message was not delivered. Update the event to reflect if the client got it.
	}

	broadcast (subscription, event) {
		if (!this.server.ws)
			throw new Error("Cannot broadcast events to websocket clients if there is no websocket server initialized.");
		return this.server.ws.broadcast(subscription, event);
	}

	//Celeritas runs natively as a cluster. Use this function if you want to specify code that should only be run by a single process.
	single (func) {
		if (cluster.isMaster && typeof func == "function")
			func();
	}
}

module.exports = {
	Celeritas,
	Call,
	Router,
	Webhook: require("./lib/webhook.js"),
	Error: require('./lib/error.js'),
	validate: require("./lib/validate.js")
};