@syntest/javascript
Version:
SynTest JavaScript is a tool for automatically generating test cases for the JavaScript language
466 lines • 26.1 kB
JavaScript
"use strict";
/*
* Copyright 2020-2023 Delft University of Technology and SynTest contributors
*
* This file is part of SynTest Framework - SynTest Javascript.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.JavaScriptLauncher = void 0;
const path = require("node:path");
const analysis_javascript_1 = require("@syntest/analysis-javascript");
const base_language_1 = require("@syntest/base-language");
const instrumentation_javascript_1 = require("@syntest/instrumentation-javascript");
const logging_1 = require("@syntest/logging");
const search_1 = require("@syntest/search");
const search_javascript_1 = require("@syntest/search-javascript");
class JavaScriptLauncher extends base_language_1.Launcher {
constructor(arguments_, moduleManager, metricManager, storageManager, userInterface) {
super(arguments_, moduleManager, metricManager, storageManager, userInterface);
this.coveredInPath = new Map();
JavaScriptLauncher.LOGGER = (0, logging_1.getLogger)("JavaScriptLauncher");
}
// eslint-disable-next-line @typescript-eslint/require-await
async initialize() {
JavaScriptLauncher.LOGGER.info("Initialization started");
const start = Date.now();
this.metricManager.recordProperty(base_language_1.PropertyName.CONSTANT_POOL_ENABLED, `${this.arguments_.constantPool.toString()}`);
this.metricManager.recordProperty(base_language_1.PropertyName.CONSTANT_POOL_PROBABILITY, `${(this.arguments_).constantPoolProbability.toString()}`);
this.storageManager.deleteTemporaryDirectories([
[this.arguments_.testDirectory],
[this.arguments_.logDirectory],
[this.arguments_.instrumentedDirectory],
]);
JavaScriptLauncher.LOGGER.info("Creating directories");
this.storageManager.createDirectories([
[this.arguments_.testDirectory],
[this.arguments_.statisticsDirectory],
[this.arguments_.logDirectory],
]);
JavaScriptLauncher.LOGGER.info("Creating temp directories");
this.storageManager.createDirectories([
[this.arguments_.testDirectory],
[this.arguments_.logDirectory],
[this.arguments_.instrumentedDirectory],
], true);
const abstractSyntaxTreeFactory = new analysis_javascript_1.AbstractSyntaxTreeFactory();
const targetFactory = new analysis_javascript_1.TargetFactory(this.arguments_.syntaxForgiving);
const controlFlowGraphFactory = new analysis_javascript_1.ControlFlowGraphFactory(this.arguments_.syntaxForgiving);
const dependencyFactory = new analysis_javascript_1.DependencyFactory(this.arguments_.syntaxForgiving);
const exportFactory = new analysis_javascript_1.ExportFactory(this.arguments_.syntaxForgiving);
const typeExtractor = new analysis_javascript_1.TypeExtractor(this.arguments_.syntaxForgiving);
const typeResolver = new analysis_javascript_1.InferenceTypeModelFactory();
const constantPoolFactory = new analysis_javascript_1.ConstantPoolFactory(this.arguments_.syntaxForgiving);
const fileSelector = new base_language_1.FileSelector();
const targetFiles = fileSelector.loadFilePaths(this.arguments_.targetInclude, this.arguments_.targetExclude);
if (this.arguments_.analysisInclude.length === 0) {
JavaScriptLauncher.LOGGER.warn("'analysis-include' config parameter is empty so we only use the target files for analysis");
}
for (const target of targetFiles) {
if (this.arguments_.analysisExclude.includes(target)) {
throw new Error(`Target files cannot be excluded from analysis. Target file: ${target}`);
}
}
const analysisFiles = fileSelector.loadFilePaths([...targetFiles, ...this.arguments_.analysisInclude], this.arguments_.analysisExclude);
this.rootContext = new analysis_javascript_1.RootContext(this.arguments_.targetRootDirectory, targetFiles, analysisFiles, abstractSyntaxTreeFactory, controlFlowGraphFactory, targetFactory, dependencyFactory, exportFactory, typeExtractor, typeResolver, constantPoolFactory);
this.userInterface.printHeader("GENERAL INFO");
const timeInMs = (Date.now() - start) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.INITIALIZATION_TIME, `${timeInMs}`);
JavaScriptLauncher.LOGGER.info("Initialization done");
}
async preprocess() {
JavaScriptLauncher.LOGGER.info("Preprocessing started");
const startPreProcessing = Date.now();
const startTargetSelection = Date.now();
const targetSelector = new base_language_1.TargetSelector(this.rootContext);
this.targets = targetSelector.loadTargets(this.arguments_.targetInclude, this.arguments_.targetExclude);
let timeInMs = (Date.now() - startTargetSelection) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.TARGET_LOAD_TIME, `${timeInMs}`);
if (this.targets.length === 0) {
// Shut server down
this.userInterface.printError(`No targets where selected! Try changing the 'target-include' parameter`);
await this.exit();
// eslint-disable-next-line unicorn/no-process-exit
process.exit();
}
const itemization = [];
for (const target of this.targets) {
itemization.push({
text: `${target.path}: ${target.name} #${target.subTargets.length}`,
subItems: target.subTargets.map((subtarget) => {
return {
text: `${subtarget.type} ${subtarget.id}`,
};
}),
});
}
this.userInterface.printItemization("TARGETS", itemization);
const selectionSettings = {
headers: ["Setting", "Value"],
rows: [
["Target Root Directory", this.arguments_.targetRootDirectory],
["Target Include", `${this.arguments_.targetInclude.join(", ")}`],
["Target Exclude", `${this.arguments_.targetExclude.join(", ")}`],
["Analysis Include", `${this.arguments_.analysisInclude.join(", ")}`],
["Analysis Exclude", `${this.arguments_.analysisExclude.join(", ")}`],
],
footers: ["", ""],
};
this.userInterface.printTable("SELECTION SETTINGS", selectionSettings);
const settings = {
headers: ["Setting", "Value"],
rows: [
["Preset", this.arguments_.preset],
["Search Algorithm", this.arguments_.searchAlgorithm],
["Population Size", `${this.arguments_.populationSize}`],
["Objective Manager", `${this.arguments_.objectiveManager}`],
[
"Secondary Objectives",
`[${this.arguments_.secondaryObjectives.join(", ")}]`,
],
["Procreation Operator", `${this.arguments_.procreation}`],
["Crossover Operator", `${this.arguments_.crossover}`],
["Sampling Operator", `${this.arguments_.sampler}`],
[
"Termination Triggers",
`[${this.arguments_.terminationTriggers.join(", ")}]`,
],
["Test Minimization Enabled", String(this.arguments_.testMinimization)],
["Seed", `${this.arguments_.randomSeed.toString()}`],
],
footers: ["", ""],
};
this.userInterface.printTable("SETTINGS", settings);
const budgetSettings = {
headers: ["Setting", "Value"],
rows: [
["Iteration Budget", `${this.arguments_.iterations} iterations`],
["Evaluation Budget", `${this.arguments_.evaluations} evaluations`],
["Search Time Budget", `${this.arguments_.searchTime} seconds`],
["Total Time Budget", `${this.arguments_.totalTime} seconds`],
],
footers: ["", ""],
};
this.userInterface.printTable("BUDGET SETTINGS", budgetSettings);
const mutationSettings = {
headers: ["Setting", "Value"],
rows: [
[
"Delta Mutation Probability",
`${this.arguments_.deltaMutationProbability}`,
],
["Crossover Probability", `${this.arguments_.crossoverProbability}`],
[
"Multi-point Crossover Probability",
`${this.arguments_.multiPointCrossoverProbability}`,
],
// sampling
["Max Depth", `${this.arguments_.maxDepth}`],
["Max Action Statements", `${this.arguments_.maxActionStatements}`],
[
"Explore Illegal Values",
String(this.arguments_.exploreIllegalValues),
],
[
"Use Constant Pool Values",
String(this.arguments_.constantPool),
],
[
"Use Constant Pool Probability",
`${this.arguments_.constantPoolProbability}`,
],
[
"Use Type Pool Values",
String(this.arguments_.typePool),
],
[
"Use Type Pool Probability",
`${this.arguments_.typePoolProbability}`,
],
[
"Use Statement Pool Values",
String(this.arguments_.statementPool),
],
[
"Use Statement Pool Probability",
`${this.arguments_.statementPoolProbability}`,
],
],
footers: ["", ""],
};
this.userInterface.printTable("MUTATION SETTINGS", mutationSettings);
const typeSettings = {
headers: ["Setting", "Value"],
rows: [
[
"Type Inference Mode",
`${this.arguments_.typeInferenceMode}`,
],
[
"Incorporate Execution Information",
String(this.arguments_
.incorporateExecutionInformation),
],
[
"Random Type Probability",
`${this.arguments_.randomTypeProbability}`,
],
],
footers: ["", ""],
};
this.userInterface.printTable("Type SETTINGS", typeSettings);
const directorySettings = {
headers: ["Setting", "Value"],
rows: [
["Syntest Directory", `${this.arguments_.syntestDirectory}`],
["Temporary Directory", `${this.arguments_.tempSyntestDirectory}`],
["Target Root Directory", `${this.arguments_.targetRootDirectory}`],
],
footers: ["", ""],
};
this.userInterface.printTable("DIRECTORY SETTINGS", directorySettings);
JavaScriptLauncher.LOGGER.info("Instrumenting targets");
const startInstrumentation = Date.now();
const instrumenter = new instrumentation_javascript_1.Instrumenter();
await instrumenter.instrumentAll(this.storageManager, this.rootContext, this.targets, this.arguments_.instrumentedDirectory);
timeInMs = (Date.now() - startInstrumentation) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.INSTRUMENTATION_TIME, `${timeInMs}`);
const startTypeResolving = Date.now();
JavaScriptLauncher.LOGGER.info("Extracting types");
this.rootContext.getAllElements();
this.rootContext.getAllRelations();
this.rootContext.getAllObjectTypes();
JavaScriptLauncher.LOGGER.info("Resolving types");
this.rootContext.resolveTypes();
timeInMs = (Date.now() - startTypeResolving) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.TYPE_RESOLVE_TIME, `${timeInMs}`);
timeInMs = (Date.now() - startPreProcessing) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.PREPROCESS_TIME, `${timeInMs}`);
this.decoder = new search_javascript_1.JavaScriptDecoder(this.arguments_.targetRootDirectory);
const executionInformationIntegrator = new search_javascript_1.ExecutionInformationIntegrator(this.rootContext.getTypeModel());
this.runner = new search_javascript_1.JavaScriptRunner(this.storageManager, this.decoder, executionInformationIntegrator, this.arguments_.testDirectory, this.arguments_.executionTimeout, this.arguments_.testTimeout, this.arguments_.silenceTestOutput);
JavaScriptLauncher.LOGGER.info("Preprocessing done");
}
async process() {
JavaScriptLauncher.LOGGER.info("Processing started");
const start = Date.now();
this.archive = new search_1.Archive();
for (const target of this.targets) {
JavaScriptLauncher.LOGGER.info(`Processing ${target.name}`);
const archive = await this.testTarget(this.rootContext, target);
this.archive.merge(archive);
}
JavaScriptLauncher.LOGGER.info("Processing done");
const timeInMs = (Date.now() - start) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.PROCESS_TIME, `${timeInMs}`);
}
async postprocess() {
JavaScriptLauncher.LOGGER.info("Postprocessing started");
const start = Date.now();
const suiteBuilder = new search_javascript_1.JavaScriptSuiteBuilder(this.storageManager, this.decoder, this.runner);
const reducedArchive = suiteBuilder.reduceArchive(this.archive);
if (this.archive.size === 0) {
throw new Error("Zero tests were created");
}
// TODO fix hardcoded paths
await suiteBuilder.runSuite(reducedArchive, "../instrumented", this.arguments_.testDirectory, true, false);
// reset states
this.storageManager.clearTemporaryDirectory([
this.arguments_.testDirectory,
]);
const { stats, instrumentationData } = await suiteBuilder.runSuite(reducedArchive, "../instrumented", this.arguments_.testDirectory, false, true);
if (stats.failures > 0) {
this.userInterface.printError("Test case has failed!");
}
this.userInterface.printHeader("SEARCH RESULTS");
const table = {
headers: ["Target", "Statement", "Branch", "Function", "File"],
rows: [],
footers: ["Average"],
};
const overall = {
branch: 0,
statement: 0,
function: 0,
};
let totalBranches = 0;
let totalStatements = 0;
let totalFunctions = 0;
for (const file of Object.keys(instrumentationData)) {
const target = this.targets.find((target) => target.path === file);
if (!target) {
continue;
}
const data = instrumentationData[file];
const summary = {
branch: 0,
statement: 0,
function: 0,
};
for (const statementKey of Object.keys(data.s)) {
summary["statement"] += data.s[statementKey] ? 1 : 0;
overall["statement"] += data.s[statementKey] ? 1 : 0;
}
for (const branchKey of Object.keys(data.b)) {
summary["branch"] += data.b[branchKey][0] ? 1 : 0;
overall["branch"] += data.b[branchKey][0] ? 1 : 0;
summary["branch"] += data.b[branchKey][1] ? 1 : 0;
overall["branch"] += data.b[branchKey][1] ? 1 : 0;
}
for (const functionKey of Object.keys(data.f)) {
summary["function"] += data.f[functionKey] ? 1 : 0;
overall["function"] += data.f[functionKey] ? 1 : 0;
}
totalStatements += Object.keys(data.s).length;
totalBranches += Object.keys(data.b).length * 2;
totalFunctions += Object.keys(data.f).length;
table.rows.push([
`${path.basename(target.path)}: ${target.name}`,
`${summary["statement"]} / ${Object.keys(data.s).length}`,
`${summary["branch"]} / ${Object.keys(data.b).length * 2}`,
`${summary["function"]} / ${Object.keys(data.f).length}`,
target.path,
]);
}
this.metricManager.recordProperty(base_language_1.PropertyName.BRANCHES_COVERED, `${overall["branch"]}`);
this.metricManager.recordProperty(base_language_1.PropertyName.STATEMENTS_COVERED, `${overall["statement"]}`);
this.metricManager.recordProperty(base_language_1.PropertyName.FUNCTIONS_COVERED, `${overall["function"]}`);
this.metricManager.recordProperty(base_language_1.PropertyName.BRANCHES_TOTAL, `${totalBranches}`);
this.metricManager.recordProperty(base_language_1.PropertyName.STATEMENTS_TOTAL, `${totalStatements}`);
this.metricManager.recordProperty(base_language_1.PropertyName.FUNCTIONS_TOTAL, `${totalFunctions}`);
// other results
this.metricManager.recordProperty(base_language_1.PropertyName.ARCHIVE_SIZE, `${this.archive.size}`);
this.metricManager.recordProperty(base_language_1.PropertyName.MINIMIZED_ARCHIVE_SIZE, `${this.archive.size}`);
overall["statement"] /= totalStatements;
if (totalStatements === 0)
overall["statement"] = 1;
overall["branch"] /= totalBranches;
if (totalBranches === 0)
overall["branch"] = 1;
overall["function"] /= totalFunctions;
if (totalFunctions === 0)
overall["function"] = 1;
table.footers.push(`${overall["statement"] * 100} %`, `${overall["branch"] * 100} %`, `${overall["function"] * 100} %`, "");
const originalSourceDirectory = path
.join("../../", path.relative(process.cwd(), this.arguments_.targetRootDirectory))
.replace(path.basename(this.arguments_.targetRootDirectory), "");
this.userInterface.printTable("Coverage", table);
// create final suite
await suiteBuilder.runSuite(reducedArchive, originalSourceDirectory, this.arguments_.testDirectory, false, true, true);
JavaScriptLauncher.LOGGER.info("Postprocessing done");
const timeInMs = (Date.now() - start) / 1000;
this.metricManager.recordProperty(base_language_1.PropertyName.POSTPROCESS_TIME, `${timeInMs}`);
}
async testTarget(rootContext, target) {
JavaScriptLauncher.LOGGER.info(`Testing target ${target.name} in ${target.path}`);
const currentSubject = new search_javascript_1.JavaScriptSubject(target, this.rootContext, this.arguments_.syntaxForgiving, this.arguments_.stringAlphabet);
const rootTargets = currentSubject
.getActionableTargets()
.filter((target) => (0, analysis_javascript_1.isExported)(target));
if (rootTargets.length === 0) {
JavaScriptLauncher.LOGGER.info(`No actionable exported root targets found for ${target.name} in ${target.path}`);
// report skipped
return new search_1.Archive();
}
const constantPoolManager = rootContext.getConstantPoolManager(target.path);
const sampler = new search_javascript_1.JavaScriptRandomSampler(currentSubject, constantPoolManager, this.arguments_.constantPool, this.arguments_.constantPoolProbability, this.arguments_.typePool, this.arguments_.typePoolProbability, this.arguments_.statementPool, this.arguments_.statementPoolProbability, this.arguments_.typeInferenceMode, this.arguments_.randomTypeProbability, this.arguments_.incorporateExecutionInformation, this.arguments_.maxActionStatements, this.arguments_.stringAlphabet, this.arguments_.stringMaxLength, this.arguments_.deltaMutationProbability, this.arguments_.exploreIllegalValues);
sampler.rootContext = rootContext;
const secondaryObjectives = new Set(this.arguments_.secondaryObjectives.map((secondaryObjective) => {
return (this.moduleManager.getPlugin(base_language_1.PluginType.SecondaryObjective, secondaryObjective)).createSecondaryObjective();
}));
const objectiveManager = (this.moduleManager.getPlugin(base_language_1.PluginType.ObjectiveManager, this.arguments_.objectiveManager)).createObjectiveManager({
runner: this.runner,
secondaryObjectives: secondaryObjectives,
});
const crossover = (this.moduleManager.getPlugin(base_language_1.PluginType.Crossover, this.arguments_.crossover)).createCrossoverOperator({
crossoverEncodingProbability: this.arguments_.crossoverProbability,
crossoverStatementProbability: this.arguments_.multiPointCrossoverProbability,
});
const procreation = (this.moduleManager.getPlugin(base_language_1.PluginType.Procreation, this.arguments_.procreation)).createProcreationOperator({
crossover: crossover,
mutateFunction: (sampler, encoding) => {
return encoding.mutate(sampler);
},
sampler: sampler,
});
const algorithm = (this.moduleManager.getPlugin(base_language_1.PluginType.SearchAlgorithm, this.arguments_.searchAlgorithm)).createSearchAlgorithm({
objectiveManager: objectiveManager,
encodingSampler: sampler,
procreation: procreation,
populationSize: this.arguments_.populationSize,
});
this.storageManager.clearTemporaryDirectory([
this.arguments_.testDirectory,
]);
// allocate budget manager
const iterationBudget = new search_1.IterationBudget(this.arguments_.iterations);
const evaluationBudget = new search_1.EvaluationBudget(this.arguments_.evaluations);
const searchBudget = new search_1.SearchTimeBudget(this.arguments_.searchTime);
const totalTimeBudget = new search_1.TotalTimeBudget(this.arguments_.totalTime);
const budgetManager = new search_1.BudgetManager();
budgetManager.addBudget(search_1.BudgetType.ITERATION, iterationBudget);
budgetManager.addBudget(search_1.BudgetType.EVALUATION, evaluationBudget);
budgetManager.addBudget(search_1.BudgetType.SEARCH_TIME, searchBudget);
budgetManager.addBudget(search_1.BudgetType.TOTAL_TIME, totalTimeBudget);
// Termination
const terminationManager = new search_1.TerminationManager();
for (const trigger of this.arguments_.terminationTriggers) {
terminationManager.addTrigger((this.moduleManager.getPlugin(base_language_1.PluginType.TerminationTrigger, trigger)).createTerminationTrigger({
objectiveManager: objectiveManager,
encodingSampler: sampler,
runner: this.runner,
crossover: crossover,
populationSize: this.arguments_.populationSize,
}));
}
// This searches for a covering population
const archive = await algorithm.search(currentSubject, budgetManager, terminationManager);
if (this.coveredInPath.has(target.path)) {
archive.merge(this.coveredInPath.get(target.path));
this.coveredInPath.set(target.path, archive);
}
else {
this.coveredInPath.set(target.path, archive);
}
this.storageManager.clearTemporaryDirectory([this.arguments_.logDirectory]);
this.storageManager.clearTemporaryDirectory([
this.arguments_.testDirectory,
]);
// timing and iterations/evaluations
this.metricManager.recordProperty(base_language_1.PropertyName.TOTAL_TIME, `${budgetManager.getBudgetObject(search_1.BudgetType.TOTAL_TIME).getUsedBudget()}`);
this.metricManager.recordProperty(base_language_1.PropertyName.SEARCH_TIME, `${budgetManager.getBudgetObject(search_1.BudgetType.SEARCH_TIME).getUsedBudget()}`);
this.metricManager.recordProperty(base_language_1.PropertyName.EVALUATIONS, `${budgetManager.getBudgetObject(search_1.BudgetType.EVALUATION).getUsedBudget()}`);
this.metricManager.recordProperty(base_language_1.PropertyName.ITERATIONS, `${budgetManager.getBudgetObject(search_1.BudgetType.ITERATION).getUsedBudget()}`);
JavaScriptLauncher.LOGGER.info(`Finished testing target ${target.name} in ${target.path}`);
return archive;
}
// eslint-disable-next-line @typescript-eslint/require-await
async exit() {
JavaScriptLauncher.LOGGER.info("Exiting");
if (this.runner && this.runner.process) {
this.runner.process.kill();
}
// TODO should be cleanup step in tool
// Finish
JavaScriptLauncher.LOGGER.info("Deleting temporary directories");
this.storageManager.deleteTemporaryDirectories([
[this.arguments_.testDirectory],
[this.arguments_.logDirectory],
[this.arguments_.instrumentedDirectory],
]);
this.storageManager.deleteMainTemporary();
}
}
exports.JavaScriptLauncher = JavaScriptLauncher;
//# sourceMappingURL=JavaScriptLauncher.js.map