rutilus-analytics-node-js
Version:
Provides a GUI web app that allows users to examine their data in detail. Includes CSV export functionality.
285 lines (228 loc) • 8.47 kB
JavaScript
/**
* Rutilus
*
* @homepage https://gmrutilus.github.io
* @license Apache-2.0
*/
const fs = require('fs');
const path = require('path');
const arraySizeLimit = 100000;
/** @module */
module.exports = /** @param {KoaCtx} ctx */ async (ctx) => {
const {
errorHandler,
errorFiles,
db,
} = ctx.res;
ctx.body = {};
let firstErrorDate = 0;
/**
* Data structure to be returned by API call
* Schema:
* [Category]
* See other functions called for Category schema
*/
const response = {
categories: []
};
await getFirstErrorDate();
await getHitsNoSinceFirstError();
await getClientErrorSummaries();
/**
* Gets the date of the first error in the client
*/
function getFirstErrorDate() {
return new Promise((resolve) => {
db.collection('errors').aggregate([
{ $group: {
_id: 'date',
date: { $min: '$date' }
}}
], (err, docs) => {
if (errorHandler.dbErrorCatcher(err)) {
return;
}
firstErrorDate = (docs && docs[0] && docs[0].date) || 0;
resolve();
});
});
}
/**
* Gets how many visits we had since the first error happened in the client
*/
function getHitsNoSinceFirstError() {
return new Promise((resolve) => {
db.collection('visits').find({
date: { $gte: firstErrorDate }
}).count((err, count) => {
if (errorHandler.dbErrorCatcher(err)) {
return;
}
response.visitsNo = count;
resolve();
});
});
}
/**
* Gets the error summaries for the errors that happened in the client side code.
* Mutates the state of the categories object in the enclosing scope.
*/
function getClientErrorSummaries() {
const errors = [];
/**
* Data structure to be added to data structure returned by API call
* Schema:
* errorGroups: {
* name: String,
* numberOfErrors: Number,
* errorGroupSummaries: [
* {
* name: String,
* numberOfErrors: Number
* }
* ]
* }
*/
const category = {};
category.name = 'Observer';
category.numberOfErrors = 0; // Will increment
category.errorGroupSummaries = []; // Will build
// Get all the errors, grouped together.
const aggPipe = [
{
// Group by the details field of the details object in each document
$group: {
_id: "$details.details",
numberOfErrors: { $sum: 1 }
}
},
];
return new Promise((resolve) => {
db.collection('errors').aggregate(aggPipe, (err, docs) => {
if (errorHandler.dbErrorCatcher(err)) {
return;
}
// Loop through each errorGroups returned from ctx query
docs.forEach(({ _id, numberOfErrors }) => {
category.errorGroupSummaries.push({
name: _id,
numberOfErrors
});
category.numberOfErrors += numberOfErrors;
});
// Add to enclosing data structure
response.categories.push(category);
errorFiles.forEach((file) => {
if (!file) { return; }
try {
getFileSystemErrorSummaries(file);
}
catch (e) {}
});
ctx.body = response;
resolve();
}); // Finished looping through error groups returned from db
});
}
/**
* Gets the errors that happened server side in the file system
* Mutates the state of the categories object in the enclosing scope.
* @param {String} file The specific file
*/
function getFileSystemErrorSummaries(file) {
/**
* Data structure to be added to data structure returned by API call
* Schema:
* errorGroups: {
* name: String,
* numberOfErrors: Number,
* errorGroupSummaries: [
* {
* name: String,
* numberOfErrors: Number,
* loaded: true,
* loadedAll: true,
* numLoaded: Number,
* }
* ]
* }
*/
const category = {};
category.name = file;
category.numberOfErrors = 0; // Will increment
category.errorGroupSummaries = []; // Will build
const errors = []; // Will build
const rawFileContent = fs.readFileSync(file).toString();
let splitFileContent = rawFileContent.split(/\n/);
if (splitFileContent.length > 0) {
splitFileContent = splitFileContent.splice(
splitFileContent.length - arraySizeLimit,
arraySizeLimit
);
splitFileContent.forEach(raw => {
if ((raw + '').length < 1) {
return;
}
try {
const error = JSON.parse(raw);
// Raise "error.message" property of the error object up to "message" if it exists. ctx is
// needed because some file system errors don't have a top level "message" property and we need
// that in order to group and display them.
if (error.error && error.error.message) {
error.message = error.error.message;
}
errors.push(error);
}
catch (e) {
// Analyzing structure of error is impossible if it cannot be parsed to an object.
errors.push(raw);
}
});
// Errors parsed and in errors array, but not yet grouped
const groupedErrors = {};
errors.forEach(error => {
// Errors that are not objects are a special case
if (typeof error !== 'object') {
if (!groupedErrors['-']) {
groupedErrors['-'] = {
name: '-',
errors: []
};
}
groupedErrors['-'].errors.push(error);
return;
}
// If there is no property with the errorGroups's name already in the groupedErrors object, make a errorGroups
// object for it and start its array, which will be built later
if (groupedErrors[error.message] === undefined) {
groupedErrors[error.message] = {
name: error.message,
errors: []
};
}
groupedErrors[error.message].errors.push(error);
});
// Loop through groupedErrors, building an error errorGroups summary and pushing it to the errorGroupSummaries
// array
for (let groupName in groupedErrors) {
if (groupedErrors.hasOwnProperty(groupName)) {
const errors = groupedErrors[groupName].errors;
const groupNumberOfErrors = errors.length;
// Increment total too.
category.numberOfErrors += groupNumberOfErrors;
// Now we have everything we need for ctx error errorGroups summary, add it to the data
category.errorGroupSummaries.push({
name: groupName,
numberOfErrors: groupNumberOfErrors,
numLoaded: groupNumberOfErrors,
errors,
loaded: true,
loadedAll: true,
});
}
}
}
// Add to enclosing data structure
response.categories.push(category);
}
};