gpii-universal
Version:
Cross platform, core components of the GPII personalization infrastructure.
418 lines (390 loc) • 17.6 kB
JavaScript
/*!
Copyright 2018 OCAD University
Licensed under the New BSD license. You may not use this file except in
compliance with this License.
You may obtain a copy of the License at
https://github.com/GPII/universal/blob/master/LICENSE.txt
*/
// This script modifies the preferences database:
// 1. Update the views records for accessing Prefs Safes and GPII Keys,
// 2. Retrieves all the Prefs Safes of type "snapset" (prefsSafesType = "snapset") from the databsse.
// 3. Retrieves all the GPII Keys associated with each snapset Prefs Safe so found,
// 4. Deletes these Prefs Safes and their associated GPII Keys from the database,
// 5. Uploads the new Prefs Safes and their GPII Keys to the database,
// A sample command that runs this script:
// node deleteAndLoadSnapsets.js $COUCHDBURL $STATIC_DATA_DIR $BUILD_DATA_DIR
//
// There is also an optional final [--justDelete] argument for testing/debugging.
// If present, the script exits with a zero exit status after deleting all the
// snapset PrefsSafes and their GPII keys. That is, the script does only the
// first four steps listed above.
;
var url = require("url"),
fs = require("fs"),
fluid = require("infusion");
fluid.require("%gpii-universal/scripts/shared/dbRequestUtils.js");
var gpii = fluid.registerNamespace("gpii");
fluid.registerNamespace("gpii.dataLoader");
fluid.setLogging(fluid.logLevel.INFO);
// Handle command line
if (process.argv.length < 5) {
fluid.log("Usage: node deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR [--justDelete]");
process.exit(1);
}
/**
* Create a set of options for data loader and a function to retreive them.
* The options are based on the command line parameters and a set of database
* constants.
* @param {Array} processArgv - The command line arguments.
* @return {Object} - The options.
*/
gpii.dataLoader.initOptions = function (processArgv) {
var dbOptions = {};
dbOptions.couchDbUrl = processArgv[2];
dbOptions.staticDataDir = processArgv[3];
dbOptions.buildDataDir = processArgv[4];
// for debugging
dbOptions.justDelete = fluid.contains(processArgv, "--justDelete");
// Set up database specific options
dbOptions.viewsUrl = dbOptions.couchDbUrl + "/_design/views";
dbOptions.prefsSafesViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findSnapsetPrefsSafes";
dbOptions.gpiiKeysViewUrl = dbOptions.couchDbUrl + "/_design/views/_view/findAllGpiiKeys";
dbOptions.parsedCouchDbUrl = url.parse(dbOptions.couchDbUrl);
dbOptions.staticData = [];
/* dbOptions.newViews; */
/* dbOptions.oldViews; */
dbOptions.snapsetPrefsSafes = [];
dbOptions.gpiiKeys = [];
dbOptions.postOptions = {
hostname: dbOptions.parsedCouchDbUrl.hostname,
port: dbOptions.parsedCouchDbUrl.port,
path: "/gpii/_bulk_docs",
auth: dbOptions.parsedCouchDbUrl.auth,
method: "POST",
headers: {
"Accept": "application/json",
"Content-Length": 0, // IMPORTANT: FILL IN PER REQUEST
"Content-Type": "application/json"
}
};
fluid.log("COUCHDB_URL: '" +
dbOptions.parsedCouchDbUrl.protocol + "//" +
dbOptions.parsedCouchDbUrl.hostname + ":" +
dbOptions.parsedCouchDbUrl.port +
dbOptions.parsedCouchDbUrl.pathname + "'"
);
fluid.log("STATIC_DATA_DIR: '" + dbOptions.staticDataDir + "'");
fluid.log("BUILD_DATA_DIR: '" + dbOptions.buildDataDir + "'");
return dbOptions;
};
/**
* Reads the static data from disk, and creates a separate reference to the
* views document. Two fields are added to the `options` parameter on return:
* options.staticData {Array} - Array of Objects to be put in the database.
* options.newViews {Object} - The data used to update the `_design/views`
* record.
* @param {Object} options - The source of the static data:
* @param {String} options.staticDataDir - The path to the directory containing
* the static data.
*/
gpii.dataLoader.loadStaticDataFromDisk = function (options) {
var data = gpii.dataLoader.getDataFromDirectory(options.staticDataDir);
var views = fluid.find(data, function (anElement) {
if (anElement._id && anElement._id === "_design/views") {
return anElement;
} else {
return undefined;
}
});
options.staticData = data;
options.newViews = views;
fluid.log("Retrieved static data from: '" + options.staticDataDir + "'");
fluid.log("\tViews data " + ( views ? "read." : "missing." ));
};
/**
* Create the step that loads the static data into the database.
* @param {Object} options - The static data:
* @param {Array} options.staticData - The static data to load.
* @return {Promise} - A promise that resolves loading the static data.
*/
gpii.dataLoader.createStaticDataStep = function (options) {
var details = {
dataToPost: options.staticData,
responseDataHandler: function (responseString, options) {
fluid.log("Loading static data from '" + options.staticDataDir + "'");
return "Uploaded static data.";
},
responseErrMsg: "Error loading static data into database: "
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Create the step that retrieves the current views from the database.
* @param {Object} options - The URL to query the database with:
* @param {String} options.viewsUrl - The `_design/views` URL.
* @return {Promise} - A promise that resolves retrieving the old views.
*/
gpii.dataLoader.createFetchOldViewsStep = function (options) {
var details = {
requestUrl: options.viewsUrl,
requestErrMsg: "Error requesting old views from database: ",
responseDataHandler: function (responseString, options) {
fluid.log("Retrieving old views from database.");
var oldViews = JSON.parse(responseString);
options.oldViews = oldViews;
return oldViews;
},
responseErrMsg: "Error retrieving old views from database: "
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Create the step that updates the views in the database. If the new views
* are the same as the old views, the `_design/views` record is not updated.
* @param {Object} options - New and old `_design/views` data:
* @param {Array} options.oldViews - the old views currently in the database.
* Its `id` and `rev` fields are needed for
* any update.
* @param {Object} options.newViews - the new views data to update with.
* @return {Promise} - A promise that resolves updating the views.
*/
gpii.dataLoader.createUpdateViewsStep = function (options) {
// Check to see if the views need updating.
// JS: Not sure how useful this is.
var oldViews = JSON.stringify(options.oldViews.views);
var newViews = JSON.stringify(options.newViews.views);
if (newViews === oldViews) {
var togo = fluid.promise();
fluid.log("New views match old views, no change.");
togo.resolve("Updated views: no change");
return togo;
}
else {
var viewsDataToPost = options.oldViews; // id and rev
viewsDataToPost.views = options.newViews.views; // new data.
var details = {
dataToPost: [viewsDataToPost],
responseDataHandler: function (responseString) {
var result = JSON.parse(responseString)[0];
fluid.log("Updated views: '" + JSON.stringify(result) + "'");
return JSON.stringify(result);
},
responseErrMsg: "Error updating views: "
};
return gpii.dbRequest.configureStep(details, options);
}
};
/**
* Find the Prefs Safes of type "snapset", mark them to be deleted, and add
* them to an array of records to remove.
* @param {String} responseString - The response from the database query -- the
* retrieved snapset PrefsSafes records.
* @param {Object} options - Used to store the snapsets:
* @param {Array} options.snapsetPrefsSafes - On output, contains the "snapset"
* PrefsSafes marked for deletion.
* @return {Array} - The snapset PrefsSafes records marked for deletion.
*/
gpii.dataLoader.processSnapsets = function (responseString, options) {
fluid.log("Processing the snapset Prefs Safes records...");
var snapSetRecords = JSON.parse(responseString);
fluid.each(snapSetRecords.rows, function (aSnapset) {
aSnapset.value._deleted = true;
options.snapsetPrefsSafes.push(aSnapset.value);
});
fluid.log("\tSnapset Prefs Safes marked for deletion.");
return options.snapsetPrefsSafes;
};
/**
* Find the GPII Key records that are associated with a snapset PrefsSafe, mark
* them for deletion, and add them to array of records to delete.
* @param {String} responseString - The response from the database query -- all
* of the GPII Keys in the database.
* @param {Object} options - Used to find and store the snapset GPII Keys:
* @param {Array} options.snapsetPrefsSafes - Contains the relevant PrefsSafes
* to use to find their associated
* GPII Keys.
* @param {Array} options.gpiiKeys - On output, contains the snapset GPII Key
* records marked for deletion.
* @return {Array} - The GPII Key records marked for deletion.
*/
gpii.dataLoader.processGpiiKeys = function (responseString, options) {
fluid.log("Processing the GPII Keys...");
var gpiiKeyRecords = JSON.parse(responseString);
options.gpiiKeys = gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion(
gpiiKeyRecords, options.snapsetPrefsSafes
);
fluid.log("\tGPII Keys associated with snapset Prefs Safes marked for deletion.");
return options.gpiiKeys;
};
/**
* Given all the GPII Keys records in the database, find the ones that reference
* a snapset PrefsSafe. As each GPII Key is found it is marked for
* deletion.
* @param {Array} gpiiKeyRecords - Array of GPII Key records from the database.
* @param {Array} snapSets - Array of snapset Prefs Safes whose id references
* its associated GPII Key record.
* @return {Array} - the values from the gpiiKeyRecords that are snapset GPII
* Keys, marked for deletion.
*/
gpii.dataLoader.markPrefsSafesGpiiKeysForDeletion = function (gpiiKeyRecords, snapSets) {
var gpiiKeysToDelete = [];
fluid.each(gpiiKeyRecords.rows, function (gpiiKeyRecord) {
var gpiiKey = fluid.find(snapSets, function (aSnapSet) {
if (gpiiKeyRecord.value.prefsSafeId === aSnapSet._id) {
return gpiiKeyRecord.value;
}
}, null);
if (gpiiKey !== null) {
gpiiKey._deleted = true;
gpiiKeysToDelete.push(gpiiKey);
}
});
return gpiiKeysToDelete;
};
/**
* Read all the json files from the given directory, then loop to put their
* contents into an array of Objects.
* @param {String} dataDir - Directory containing the files to load.
* @return {Array} - Each element of the array is an Object based on the
* contents of each file loaded.
*/
gpii.dataLoader.getDataFromDirectory = function (dataDir) {
var contentArray = [];
var files = fs.readdirSync(dataDir);
files.forEach(function (aFile) {
if (aFile.endsWith(".json")) {
var fileContent = fs.readFileSync(dataDir + "/" + aFile, "utf-8");
contentArray = contentArray.concat(JSON.parse(fileContent));
}
});
return contentArray;
};
/**
* Create the step that fetches the current "snapset" Prefs Safes from the
* database.
* @param {Object} options - Object for querying the database:
* @param {String} options.prefsSafesViewUrl - Views URL for finding all the
* "snapset" PrefsSafes records.
* @return {Promise} - A promise that resolves to the set of "snapset" PrefsSafes
* currently in the database.
*/
gpii.dataLoader.createFetchSnapsetsStep = function (options) {
var details = {
requestUrl: options.prefsSafesViewUrl,
requestErrMsg: "Error requesting snapsets Prefs Safes: ",
responseDataHandler: gpii.dataLoader.processSnapsets,
responseErrMsg: "Error retrieving snapsets Prefs Safes: "
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Create the step that fetches the current GPII keys associate with the snapset
* Prefs Safes.
* @param {Object} options - Object for querying the database:
* @param {String} options.gpiiKeysViewUrl - Views URL for finding all GPII Key
* records in the database.
* @return {Promise} - A promise that resolves to the set of GPII keys in the
* database that correspond to "snapset" PrefsSafes.
*/
gpii.dataLoader.createFetchGpiiKeysStep = function (options) {
var details = {
requestUrl: options.gpiiKeysViewUrl,
requestErrMsg: "Error requesting GPII Keys: ",
responseDataHandler: gpii.dataLoader.processGpiiKeys,
responseErrMsg: "Error finding snapset Prefs Safes associated GPII Keys: "
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Log how many snapset Prefs Safes and GPII Keys were deleted.
* @param {String} responseString - Response from the database (ignored)
* @param {Object} options - Object containing the sets of Prefs Safes and
* their GPII keys:
* @param {Array} options.snapsetPrefsSafes - The set of Prefs Safes.
* @param {Array} options.gpiiKeys - The set of associated GPII Keys.
* @return {Object} - An object with properties "snapsets" and "gpiiKeys" that
* are the number of snapsets and gpiiKeys deleted.
*/
gpii.dataLoader.logSnapsetDeletion = function (responseString, options) {
fluid.log( "Deleted " +
options.snapsetPrefsSafes.length + " Prefs Safes and " +
options.gpiiKeys.length + " associated GPII Keys, "
);
return {
snapsets: options.snapsetPrefsSafes.length,
gpiiKeys: options.gpiiKeys.length
};
};
/**
* Create the step that deletes, in batch, the current snapset Prefs Safes and
* their associated GPII keys.
* @param {Object} options - The records to be deleted:
* @param {Array} options.snapsetPrefsSafes - The "snapset" PrefsSafe records to
* delete.
* @param {Array} options.gpiiKeys - The GPII Key records to delete.
* @return {Promise} - The promise that resolves the deletion.
*/
gpii.dataLoader.createBatchDeleteStep = function (options) {
var details = {
dataToPost: options.snapsetPrefsSafes.concat(options.gpiiKeys),
responseDataHandler: gpii.dataLoader.logSnapsetDeletion
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Log the uploading of all the "snapset" Prefs Safes and their GPII Keys.
* @param {String} responseString - Response from the database (ignored)
* @param {Object} options - The directory containing the data:
* @param {String} options.buildDataDir - The directory from which the Prefs
* Safes and GPII keys were loaded.
* @return {String} - A message to indicate that the upload is complete.
*/
gpii.dataLoader.logSnapsetsUpload = function (responseString, options) {
fluid.log("Bulk loading of build data from '" + options.buildDataDir + "'");
return "Uploaded latest snapsets preferences";
};
/**
* Create the step that uploads, in batch, the new "snapset" Prefs Safes, and
* their associated GPII keys.
* @param {Object} options - The directory containing the data:
* @param {String} options.buildDataDir - The directory from which to load the
* Prefs Safes and GPII keys data.
* @return {Promise} - A promise that resolves the upload.
*/
gpii.dataLoader.createBatchUploadStep = function (options) {
var newSnapsetsData = gpii.dataLoader.getDataFromDirectory(options.buildDataDir);
var details = {
dataToPost: newSnapsetsData,
responseDataHandler: gpii.dataLoader.logSnapsetsUpload
};
return gpii.dbRequest.configureStep(details, options);
};
/**
* Create and execute the steps to update the database.
*/
gpii.dataLoader.orchestrate = function () {
var options = gpii.dataLoader.initOptions(process.argv);
gpii.dataLoader.loadStaticDataFromDisk(options);
var sequence = [
gpii.dataLoader.createStaticDataStep,
gpii.dataLoader.createFetchOldViewsStep,
gpii.dataLoader.createUpdateViewsStep,
gpii.dataLoader.createFetchSnapsetsStep,
gpii.dataLoader.createFetchGpiiKeysStep,
gpii.dataLoader.createBatchDeleteStep
];
if (!options.justDelete) {
sequence.push(gpii.dataLoader.createBatchUploadStep);
}
fluid.promise.sequence(sequence, options).then(
function (/*result*/) {
fluid.log("Done.");
process.exit(0);
},
function (error) {
fluid.log(error);
process.exit(1);
}
);
};
gpii.dataLoader.orchestrate();