survey_stack_data_tools
Version:
Tools meant to help users retrieve and analyze data obtained in survey stack.
320 lines (278 loc) • 14.8 kB
JavaScript
// we will parse the survey source code in order to obtain data paths, question phrasings, etc.
// this is useful to control the places from which the scoring script obtains information, see if they are current, etc.
const fs = require("fs");
// let surveySource = JSON.parse( fs.readFileSync(`${__dirname}/../test/test_data/example_survey_description.json`) );
// let currentVersion = surveySource.revisions.find( revision => revision.version == surveySource.latestVersion );
// // let allSections = surveySource.revisions[3].controls.map(d => d.label);
// // let equity = currentVersion.controls.find( section => section.label == 'Employee Equity' );
// // review the types of all groups, to ignore those that contain no questinos
// let groupTypes = currentVersion.controls.map(d => d.type);
// let sectionTypes = currentVersion.controls.flatMap( d => d.children?.map( c => { return { children_type: c.type, parent_type: d.type, answers: c.options.source?.content?.length }; } ) );
// let oneLevel = flattenAnswersLevel(currentVersion.controls, 0, surveySource.resources);
// // let parsedQuestions = flattenAnswersTable(currentVersion.controls, surveySource.resources);
// let parsedQuestions = flattenAnswersTable(surveySource);
// parsedQuestions.filter( d => d.level_3_type )[1];
// parsedQuestions.filter( d => d.level_3_answers );
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_answers`] ).length;
// parsedQuestions.filter( d => !d[`level_${d.finalIndex}_answers`] ).length;
// parsedQuestions.filter( d => !d[`level_${d.finalIndex}_answers`] );
// let allFinalLevelTypes = new Set( parsedQuestions.map( d => d[`level_${d.finalIndex}_type`] ) );
// fs.writeFileSync(`${__dirname}/../documentation/questions_parsed_from_survey.json`, JSON.stringify( parsedQuestions ), console.error);
// let answerPairingTable = parsedQuestions
// .filter( d => d.answer_details )
// .flatMap( question => {
// let output = question.answer_details?.map( answer => {
// let answerOutput = {
// path: question.path,
// type: question[`level_${question.finalIndex}_type`],
// section_name: question.level_0_section_name,
// section_label: question.level_0_section_label,
// question_label: question[`level_${question.finalIndex}_label`],
// question_name: question[`level_${question.finalIndex}_name`],
// answer_value: answer.value,
// answer_label: answer.label,
// answer_type: answer.type,
// full_body: answer,
// all_labels: question.allLabels,
// };
// return answerOutput;
// } );
// return output;
// } );
// fs.writeFileSync(`${__dirname}/../documentation/answers_parsed_from_survey.json`, JSON.stringify( answerPairingTable ), console.error);
// // obtaining answers for a matrix
// parsedQuestions.find( d => d.level_1_type == "matrix" );
// surveySource.resources.find(d => d.id == '63f8ccb5fb583400016e6838' );
// // all types without answer
// let typesWithoutAnswer = ["instructions", "file", "farmOsFarm", "geoJSON", "location", "script"];
// let typesWithAnswer = ["selectSingle", "selectMultiple", "matrix"];
// new Set( parsedQuestions.filter( d => !d.anwsers ).map( d => d[`level_${d.finalIndex}_type`] ).filter( d => !typesWithoutAnswer.includes(d) ) );
// // it seems we can ignore the 'string' answers
// parsedQuestions.filter( d => d.level_1_type == "string" );
// //as well as number
// parsedQuestions.filter( d => d.level_1_type == "number" );
// // check "ontology" type
// parsedQuestions.filter( d => d[ `level_${d.finalIndex}_type` ] == "ontology" );
// // chosen example 'data.common_onboarding.common.management_plans.current.detail'
// let ontologyExample = currentVersion.controls
// .find( d => d.name == "common_onboarding" )
// .children
// .find( d => d.name == "common" )
// .children
// .find( d => d.name == "management_plans" )
// .children
// .find( d => d.name == "current" )
// .children
// .find( d => d.name == "detail" )
// ;
// // ontologyExample
// surveySource.resources.find(d => d.id == '64c93f16cfe61f00019299e4' );
// let ontologyExampleValues = surveySource.resources.find(d => d.id == '64c93f16cfe61f00019299e4' ).content;
// // ontologyExampleValues
// // another ontologyExample
// let ontologyOther = currentVersion.controls
// .find( d => d.name == "common_onboarding" )
// .children
// .find( d => d.name == "common" )
// .children
// .find( d => d.name == "products" )
// .children
// .find( d => d.name == "other" )
// ;
// // ontologyExample
// surveySource.resources.find(d => d.id == ontologyOther.options.source );
// let ontologyOtherValues = surveySource.resources.find(d => d.id == '64c93f16cfe61f00019299e4' ).content;
// // ontology fields with answers
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_type`] == "ontology" && Array.isArray( d[`level_${d.finalIndex}_answers`] ) ).length;
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_type`] == "ontology" && Array.isArray( d[`level_${d.finalIndex}_answers`] ) ).map( d => d.path );
// // ontology fields without answers
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_type`] == "ontology" && !Array.isArray( d[`level_${d.finalIndex}_answers`] ) ).length;
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_type`] == "ontology" && !Array.isArray( d[`level_${d.finalIndex}_answers`] ) );
// // ontology example
// answerPairingTable.filter( d => d.path == "data.engagement.hubs_networks" );
// // example parsed matrix value
// parsedQuestions.find( d => d.path.match(/data.croplands.tillage/) );
// answerPairingTable.filter( d => d.path.match(/data.croplands.tillage/) );
// // search for missing answer
// parsedQuestions.filter( d => d.allLabels.find( l => l.match(/women/i) ) );
// // searching for management plans
// parsedQuestions.filter( d => d.path.match(/data.common_onboarding.common.management_plans.current/) );
// currentVersion.controls
// .find( d => d.name == "common_onboarding" )
// .children
// .find( d => d.name == "common" )
// .children
// .find( d => d.name == "management_plans" )
// .children
// .find( d => d.name == "current" )
// ;
// //other matrices with an "is_true" name
// parsedQuestions.filter( d => d[`level_${d.finalIndex}_name`] == "is_true" ).length;
/**
* Shapes a matrix object so that we can list its answers. This requires reshaping, as matrix values are packaged together. We will shape them as a list of fields and search for the answers for each field.
* @param {Object} question -- Provided by our recursive survey simplifaction function.
* @param {integer} index -- The index indicating the current level of recursion for the simplification.
* @param {Object[]} resources -- Obtained from a first level attribute in the survey definition.
*/
function processMatrixObject(question, index, resources) {
let fields = question[`level_${question.finalIndex}_answers`]
.content
.map( cont => {
let output = structuredClone(question);
output[`level_${index}_answers`] = undefined;
output[`level_${index + 1}_name`] = cont.value;
output[`level_${index + 1}_label`] = question[`level_${index}_label`];
output[`level_${index + 1}_field_label`] = cont.label;
output[`level_${index + 1}_type`] = cont.type;
output[ `is_matrix_type_level${index}` ] = true;
output.finalIndex = index + 1;
if (cont.resource && !cont[`level_${index}_missing_ontology`]) {
let ontologyEntry = resources
.find( res => res.id == cont.resource )
;
let ontology = ontologyEntry ? ontologyEntry.content : undefined;
;
if ( Array.isArray(ontology) ) {
let answers = ontology.map( item => {
let ansObj = {
label: item.label,
value: item.value
};
return ansObj;
} )
;
output[`level_${index+1}_answers`] = answers;
}
;
};
return output;
} );
return fields;
};
/**
* Retrieves answers for ontology type questions. This involves sourcing the ontology that feeds it and placing its contents as the answer values.
* @param {Object} question -- Provided by our recursive survey simplifaction function.
* @param {integer} index -- The index indicating the current level of recursion for the simplification.
* @param {Object[]} resources -- Obtained from a first level attribute in the survey definition.
* @returns {}
*/
function processOntologyQuestionObject(question, index,resources) {
let ontologyEntry = resources
.find( res => res.id == question[`level_${index}_answers`] )
;
let ontology = ontologyEntry ? ontologyEntry.content : undefined;
if (Array.isArray(ontology)) {
let answers = ontology.map(item => {
let output = {
label: item.label,
value: item.value
};
return output;
})
;
question[`level_${index}_answers`] = answers;
} else {
question.errors = [{ index: index, error: "missing_ontology", extra: { id: question[`level_${index}_answers`] } }];
question[`level_${index}_answers`] = [];
question[`level_${index}_missing_ontology`] = question[`level_${index}_answers`];
};
return question;
};
/**
* Recursive step for `flattenAnswersTable`. It will transform each children into a new object, still containing relevant data from ancestors. Nested attributes are transformed into flat, indexed ones.
* @param {} questions
* @param {} index
* @returns {}
*/
function flattenAnswersLevel(questions, index, resources) {
let output = [];
questions.forEach( section => {
section.children?.forEach( question => {
let questionDescription = {};
if (index == 0) {
questionDescription.level_0_section_label = section.label;
questionDescription.level_0_section_name = section.name;
}
questionDescription[`level_${index}_label`] = question.label;
questionDescription[`level_${index}_name`] = question.name;
questionDescription[`level_${index}_answers`] = question.options?.source;
questionDescription[`level_${index}_type`] = question.type;
questionDescription[`level_${index}_hint`] = question.hint;
questionDescription.finalIndex = index;
questionDescription.children = question.children;
let previousDataNames = Object.keys(section).filter( q => q.match(/level_[0-9]*_/) );
previousDataNames.forEach( dataName => { questionDescription[dataName] = section[dataName]; } );
if (question.type == "ontology") {
questionDescription = processOntologyQuestionObject(questionDescription, index, resources);
} else if (question.type == "matrix") {
questionDescription = processMatrixObject(questionDescription, index, resources);
};
if (Array.isArray(questionDescription)) {
output.push(... questionDescription);
} else {
output.push(questionDescription);
};
} );
} );
return output;
};
/**
* This function is recursive. It transform each object into simpler objects, keepìng the information we will need to read a submission and understand its meaning: the data path is built using the succesive 'data' attributes, and is needed to retrieve the answer. The stopping criteria is reaching levels in which there are no children.
* @param {} questions
* @returns {}
*/
function flattenAnswersTable(surveySource) {
let currentVersion = surveySource.revisions.find( revision => revision.version == surveySource.latestVersion );
let questions = currentVersion.controls;
let resources = surveySource.resources;
let sectionsWithChildren = questions.filter( d => d.children ).length;
// console.log(`Original length is ${sectionsWithChildren}`);
let index = 0;
let output = [];
while( sectionsWithChildren > 0 ) {
// console.log(`Index: ${index}. Current output length: ${output.length}.`);
questions = flattenAnswersLevel(questions,index, resources);
output.push( ... questions.filter( d => !d.children ) );
questions = questions.filter(d => d.children);
index ++;
sectionsWithChildren = questions.length;
};
// console.log(`findal index is ${index}`);
output.forEach( q => {
let pathIndex = 0;
while(pathIndex <= q.finalIndex) {
q.path = q.path ? q.path.concat(".",q[`level_${pathIndex}_name`]) : "data.".concat(q.level_0_section_name,".",q[`level_${pathIndex}_name`]);
q.allLabels = Array.isArray( q.allLabels ) ? [ ... q.allLabels, q[`level_${pathIndex}_label`] ] : [ q[`level_${pathIndex}_label`] ];
pathIndex ++;
};
if (q[`level_${q.finalIndex}_answers`]) {
switch (q[`level_${q.finalIndex}_type`]) {
case "selectMultiple":
q.answer_details = q[`level_${q.finalIndex}_answers`];
q.answers = q.answer_details.map( d => d.value );
break;
case "selectSingle":
q.answer_details = q[`level_${q.finalIndex}_answers`];
q.answers = q.answer_details.map( d => d.value );
break;
case "dropdown":
q.answer_details = q[`level_${q.finalIndex}_answers`];
q.answers = q.answer_details.map( d => d.value );
break;
case "ontology":
q.answer_details = q[`level_${q.finalIndex}_answers`];
q.answers = q.answer_details.map( d => d.value );
break;
// case "matrix":
// q.answer_details = q[`level_${q.finalIndex}_answers`].content.map( row => { return { label: row.label, value: row.value, type: row.type, resource: row.resource }; } );
// q.answers = q.answer_details.map( d => d.value );
break;
}
};
} );
return output;
};
exports.processMatrixObject = processMatrixObject;
exports.processOntologyQuestionObject = processOntologyQuestionObject;
exports.flattenAnswersLevel = flattenAnswersLevel;
exports.flattenAnswersTable = flattenAnswersTable;