@augment-vir/node
Version:
A collection of augments, helpers types, functions, and classes only for Node.js (backend) JavaScript environments.
622 lines (621 loc) • 20.3 kB
JavaScript
import { assert, check } from '@augment-vir/assert';
import { awaitedBlockingMap, getObjectTypedKeys, log, typedObjectFromEntries, } from '@augment-vir/common';
import { spawn } from 'node:child_process';
import { lstat, readdir, stat } from 'node:fs/promises';
import { isAbsolute, join, resolve } from 'node:path';
import { isOperatingSystem, OperatingSystem } from '../os/operating-system.js';
const grepBinPath = '/usr/bin/grep';
function shellQuote(input) {
return [
"'",
input.replaceAll("'", String.raw `'\''`),
"'",
].join('');
}
function recursiveFlag({ recursive, followSymLinks, }) {
if (!recursive) {
return '';
}
else if (!followSymLinks) {
return '--recursive';
}
/**
* BSD `grep` (macOS) requires `-S` to follow symlinks while recursing, but GNU `grep` (Linux)
* has no `-S` flag and instead follows all symlinks with `-R`. Only one of these branches can
* run on a given operating system.
*/
/* node:coverage ignore next */
return isOperatingSystem(OperatingSystem.Mac) ? '-RS' : '-R';
}
function isValidMaxCount(maxCount) {
return (maxCount == undefined ||
(check.isNumber(maxCount) && Number.isInteger(maxCount) && maxCount >= -1));
}
function isOptionalBoolean(input) {
return input == undefined || check.isBoolean(input);
}
function isValidTrueOnlyOptionGroup({ input, values, }) {
return (input == undefined ||
(check.isObject(input) &&
values.every((value) => value == undefined || value === true) &&
values.filter((value) => value === true).length === 1));
}
function isValidPatternSyntax(input) {
return isValidTrueOnlyOptionGroup({
input,
values: check.isObject(input)
? [
input.basicRegExp,
input.extendedRegExp,
input.fixedStrings,
]
: [],
});
}
function isValidMatchType(input) {
return isValidTrueOnlyOptionGroup({
input,
values: check.isObject(input)
? [
input.lineRegExp,
input.wordRegExp,
]
: [],
});
}
function isValidOutput(input) {
return isValidTrueOnlyOptionGroup({
input,
values: check.isObject(input)
? [
input.countOnly,
input.filesOnly,
]
: [],
});
}
function didGrepFail({ exitCode }) {
return exitCode == undefined || exitCode > 1;
}
function spawnGrepProcess({ args, cwd, }) {
try {
return spawn(grepBinPath, args, {
cwd,
stdio: [
'ignore',
'pipe',
'pipe',
],
});
/* node:coverage ignore next 3 */
}
catch {
return undefined;
}
}
async function runGrepCommand({ args, cwd, }) {
return new Promise((resolveOutput) => {
const stdoutChunks = [];
const grepProcess = spawnGrepProcess({
args,
cwd,
});
if (!grepProcess) {
resolveOutput({
exitCode: undefined,
stdout: '',
});
return;
}
assert.isDefined(grepProcess.stdout, 'stdout emitter was not created for grep.');
assert.isDefined(grepProcess.stderr, 'stderr emitter was not created for grep.');
grepProcess.stdout.on('data', (chunk) => {
stdoutChunks.push(Buffer.from(chunk));
});
grepProcess.stderr.on('data', () => { });
/* node:coverage ignore next 5 */
grepProcess.on('error', () => {
resolveOutput({
exitCode: undefined,
stdout: Buffer.concat(stdoutChunks).toString(),
});
});
grepProcess.on('close', (rawExitCode) => {
resolveOutput({
/* node:coverage ignore next */
exitCode: rawExitCode ?? undefined,
stdout: Buffer.concat(stdoutChunks).toString(),
});
});
});
}
function redactGrepArgForLogging({ arg, index, operandDelimiterIndex, previousArg, }) {
if (previousArg === '-e') {
return '<pattern>';
}
else if (operandDelimiterIndex >= 0 && index > operandDelimiterIndex) {
return '<path>';
}
else if (arg.startsWith('--exclude-dir=')) {
return '--exclude-dir=<glob>';
}
else if (arg.startsWith('--exclude=')) {
return '--exclude=<glob>';
}
else if (arg.startsWith('--include=')) {
return '--include=<glob>';
}
else if (arg.startsWith('--max-count=')) {
return '--max-count=<count>';
}
else {
return arg;
}
}
function formatGrepCommand(args) {
const operandDelimiterIndex = args.indexOf('--');
return [
'grep',
...args.map((arg, index) => shellQuote(redactGrepArgForLogging({
arg,
index,
operandDelimiterIndex,
previousArg: args[index - 1],
}))),
].join(' ');
}
function replaceGrepCountOutputArg({ args, replacement, }) {
const countArgIndex = args.indexOf('--count');
/* node:coverage ignore next */
return countArgIndex < 0 ? [...args] : args.toSpliced(countArgIndex, 1, replacement);
}
function replaceGrepSearchOperands({ args, searchParts, }) {
const operandDelimiterIndex = args.indexOf('--');
return [
...args.slice(0, operandDelimiterIndex + 1),
...searchParts,
];
}
function extractStringArray(input) {
return check.isArray(input) ? input.filter(check.isString).filter(check.isTruthy) : [];
}
function extractOptionalStringArray(input) {
if (input == undefined) {
return [];
}
else if (!check.isArray(input) || !input.every(check.isString)) {
return undefined;
}
return input.filter(check.isTruthy);
}
function extractString(input) {
return check.isString(input) && input ? input : undefined;
}
function extractGrepOptionArrays({ excludeDirs, excludePatterns, includeFiles, }) {
const extractedExcludeDirs = extractOptionalStringArray(excludeDirs);
const extractedExcludePatterns = extractOptionalStringArray(excludePatterns);
const extractedIncludeFiles = extractOptionalStringArray(includeFiles);
return extractedExcludeDirs && extractedExcludePatterns && extractedIncludeFiles
? {
excludeDirs: extractedExcludeDirs,
excludePatterns: extractedExcludePatterns,
includeFiles: extractedIncludeFiles,
}
: undefined;
}
function areGrepOptionsValid({ grepOptions, }) {
return (isValidMaxCount(grepOptions.maxCount) &&
(grepOptions.cwd == undefined || !!extractString(grepOptions.cwd)) &&
[
grepOptions.binary,
grepOptions.followSymLinks,
grepOptions.ignoreCase,
grepOptions.invertMatch,
grepOptions.printCommand,
grepOptions.recursive,
].every(isOptionalBoolean) &&
isValidPatternSyntax(grepOptions.patternSyntax) &&
isValidMatchType(grepOptions.matchType) &&
isValidOutput(grepOptions.output));
}
function createSearchPatterns(grepSearchPattern) {
if (!check.isObject(grepSearchPattern)) {
return [];
}
const rawPatterns = check.isArray(grepSearchPattern.patterns)
? grepSearchPattern.patterns
: [
grepSearchPattern.pattern,
];
return extractStringArray(rawPatterns);
}
function createSearchLocation(grepSearchLocation) {
if (!check.isObject(grepSearchLocation)) {
return undefined;
}
const file = extractString(grepSearchLocation.file);
const dir = extractString(grepSearchLocation.dir);
if (grepSearchLocation.files != undefined) {
return {
files: extractStringArray(grepSearchLocation.files),
};
}
else if (file) {
return {
files: [
file,
],
};
}
else if (grepSearchLocation.dirs != undefined) {
return {
dirs: extractStringArray(grepSearchLocation.dirs),
};
}
return dir
? {
dirs: [
dir,
],
}
: undefined;
}
function resolveSearchPart({ cwd, searchPart, }) {
return cwd && !isAbsolute(searchPart) ? resolve(cwd, searchPart) : searchPart;
}
async function shouldSearchPart({ cwd, followSymLinks, includeDirectories, searchPart, }) {
try {
const fileStats = await (followSymLinks ? stat : lstat)(resolveSearchPart({
cwd,
searchPart,
}));
return fileStats.isFile() || (includeDirectories && fileStats.isDirectory());
}
catch {
return false;
}
}
async function filterSearchParts({ cwd, followSymLinks, includeDirectories, searchParts, }) {
return (await awaitedBlockingMap(searchParts, async (searchPart) => {
return (await shouldSearchPart({
cwd,
followSymLinks,
includeDirectories,
searchPart,
}))
? searchPart
: undefined;
})).filter(check.isTruthy);
}
async function readDirectDirSearchParts({ cwd, dir, followSymLinks, }) {
try {
const readDirPath = resolveSearchPart({
cwd,
searchPart: dir,
});
return (await awaitedBlockingMap((await readdir(readDirPath)).toSorted().filter((entry) => !entry.startsWith('.')), async (entry) => {
const searchPart = join(dir, entry);
return (await shouldSearchPart({
cwd: readDirPath,
followSymLinks,
includeDirectories: false,
searchPart: entry,
}))
? searchPart
: undefined;
})).filter(check.isTruthy);
/* node:coverage ignore next 3 */
}
catch {
return [];
}
}
async function createSearchParts({ cwd, followSymLinks, recursive, searchLocation, }) {
const searchParts = searchLocation.dirs || searchLocation.files;
assert.isDefined(searchParts, 'Grep search location was not resolved.');
const filteredSearchParts = await filterSearchParts({
cwd,
followSymLinks,
includeDirectories: !!searchLocation.dirs,
searchParts,
});
return searchLocation.dirs
? recursive
? filteredSearchParts
: (await awaitedBlockingMap(filteredSearchParts, (dir) => readDirectDirSearchParts({
cwd,
dir,
followSymLinks,
}))).flat()
: filteredSearchParts;
}
function createGrepCountEntry({ countString, fileName }) {
assert.isDefined(fileName, 'Failed parse grep file name.');
const count = Number(countString);
assert.isNumber(count, `Failed to parse grep number from: '${countString}'`);
/* node:coverage ignore next 3 */
if (!count) {
return undefined;
}
return {
key: fileName,
value: count,
};
}
function parseNullDelimitedGrepRecords({ stdout }) {
const records = [];
let recordStartIndex = 0;
while (recordStartIndex < stdout.length) {
const delimiterIndex = stdout.indexOf('\0', recordStartIndex);
/* node:coverage ignore next 3 */
if (delimiterIndex < 0) {
break;
}
const valueStartIndex = delimiterIndex + 1;
const newlineIndex = stdout.indexOf('\n', valueStartIndex);
/* node:coverage ignore next */
const valueEndIndex = newlineIndex < 0 ? stdout.length : newlineIndex;
records.push({
fileName: stdout.slice(recordStartIndex, delimiterIndex),
value: stdout.slice(valueStartIndex, valueEndIndex),
});
/* node:coverage ignore next */
recordStartIndex = newlineIndex < 0 ? stdout.length : newlineIndex + 1;
}
return records;
}
/* node:coverage ignore next 26 */
function parseColonDelimitedGrepCountOutput(stdout) {
return typedObjectFromEntries(stdout
.trimEnd()
.split('\n')
.map((entry) => {
if (!entry) {
return undefined;
}
const countDelimiterIndex = entry.lastIndexOf(':');
return createGrepCountEntry({
countString: countDelimiterIndex < 0 ? undefined : entry.slice(countDelimiterIndex + 1),
fileName: countDelimiterIndex < 0 ? undefined : entry.slice(0, countDelimiterIndex),
});
})
.filter(check.isTruthy)
.map((entry) => [
entry.key,
entry.value,
]));
}
/* node:coverage ignore next 20 */
function parseGrepCountOutput(stdout) {
return stdout.includes('\0')
? typedObjectFromEntries(parseNullDelimitedGrepRecords({
stdout,
})
.map((record) => {
return createGrepCountEntry({
countString: record.value,
fileName: record.fileName,
});
})
.filter(check.isTruthy)
.map((entry) => [
entry.key,
entry.value,
]))
: parseColonDelimitedGrepCountOutput(stdout);
}
/* node:coverage ignore next 7 */
function tryParseGrepCountOutput(stdout) {
try {
return parseGrepCountOutput(stdout);
}
catch {
return undefined;
}
}
function parseKnownFileCountOutput({ filePath, stdout, }) {
/* node:coverage ignore next 3 */
if (stdout.includes('\0')) {
return parseGrepCountOutput(stdout)[filePath];
}
const countPrefix = `${filePath}:`;
/* node:coverage ignore next */
const outputLine = stdout.endsWith('\n') ? stdout.slice(0, -1) : stdout;
/* node:coverage ignore next 3 */
if (!outputLine.startsWith(countPrefix)) {
return undefined;
}
/* node:coverage ignore next */
return createGrepCountEntry({
countString: outputLine.slice(countPrefix.length),
fileName: filePath,
})?.value;
}
async function runKnownFileGrepCount({ cwd, filePath, grepArgs, }) {
const result = await runGrepCommand({
args: replaceGrepSearchOperands({
args: grepArgs,
searchParts: [
filePath,
],
}),
cwd,
});
/* node:coverage ignore next 3 */
if (didGrepFail(result)) {
return undefined;
}
const count = parseKnownFileCountOutput({
filePath,
stdout: result.stdout,
});
/* node:coverage ignore next 3 */
if (!count) {
return undefined;
}
return {
key: filePath,
value: count,
};
}
async function runGrepCountFallback({ cwd, grepArgs, }) {
const filesOnlyResult = await runGrepCommand({
args: replaceGrepCountOutputArg({
args: grepArgs,
replacement: '--files-with-matches',
}),
cwd,
});
/* node:coverage ignore next 3 */
if (didGrepFail(filesOnlyResult) || filesOnlyResult.exitCode === 1 || !filesOnlyResult.stdout) {
return {};
}
return typedObjectFromEntries((await awaitedBlockingMap(getObjectTypedKeys(parseGrepFilesOnlyOutput(filesOnlyResult.stdout)), (filePath) => {
return runKnownFileGrepCount({
cwd,
filePath,
grepArgs,
});
}))
.filter(check.isTruthy)
.map((entry) => [
entry.key,
entry.value,
]));
}
function parseGrepFilesOnlyOutput(stdout) {
return typedObjectFromEntries(
/* node:coverage ignore next */
(stdout.includes('\0') ? stdout.split('\0') : stdout.trimEnd().split('\n'))
.filter(check.isTruthy)
.map((entry) => [
entry,
[],
]));
}
function parseGrepNormalOutput(stdout) {
const fileMatches = new Map();
parseNullDelimitedGrepRecords({
stdout,
}).forEach((record) => {
fileMatches.set(record.fileName, [
...(fileMatches.get(record.fileName) || []),
record.value,
]);
});
return typedObjectFromEntries([...fileMatches.entries()]);
}
/**
* Run `grep`, matching patterns to specific lines in files or directories.
*
* @category Node : File
* @category Package : @augment-vir/node
* @package [`@augment-vir/node`](https://www.npmjs.com/package/@augment-vir/node)
*/
export async function grep(grepSearchPattern, grepSearchLocation, options = {}) {
const grepOptions = check.isObject(options)
? options
: {};
const searchPatterns = createSearchPatterns(grepSearchPattern);
const grepOptionArrays = extractGrepOptionArrays(grepOptions);
const cwd = extractString(grepOptions.cwd);
if (!searchPatterns.length ||
!grepOptionArrays ||
!areGrepOptionsValid({
grepOptions,
})) {
return {};
}
const searchLocation = createSearchLocation(grepSearchLocation);
if (!searchLocation ||
(searchLocation.dirs && !searchLocation.dirs.length) ||
(searchLocation.files && !searchLocation.files.length)) {
return {};
}
const searchParts = await createSearchParts({
cwd,
followSymLinks: grepOptions.followSymLinks,
recursive: grepOptions.recursive,
searchLocation,
});
if (!searchParts.length) {
return {};
}
const grepArgs = [
grepOptions.patternSyntax?.basicRegExp
? '--basic-regexp'
: grepOptions.patternSyntax?.extendedRegExp
? '--extended-regexp'
: grepOptions.patternSyntax?.fixedStrings
? '--fixed-strings'
: '',
grepOptions.ignoreCase ? '--ignore-case' : '',
grepOptions.invertMatch && !grepOptions.output?.filesOnly ? '--invert-match' : '',
grepOptions.matchType?.wordRegExp
? '--word-regexp'
: grepOptions.matchType?.lineRegExp
? '--line-regexp'
: '',
grepOptions.output?.countOnly
? '--count'
: grepOptions.output?.filesOnly
? grepOptions.invertMatch
? '--files-without-match'
: '--files-with-matches'
: '',
'--color=never',
grepOptions.maxCount == undefined ? '' : `--max-count=${grepOptions.maxCount}`,
'--no-messages',
'--devices=skip',
'--with-filename',
'--null',
...(grepOptionArrays.excludePatterns.length
? grepOptionArrays.excludePatterns.map((excludePattern) => `--exclude=${excludePattern}`)
: []),
recursiveFlag(grepOptions),
...(grepOptionArrays.excludeDirs.length
? grepOptionArrays.excludeDirs.map((excludeDir) => `--exclude-dir=${excludeDir}`)
: []),
...(grepOptionArrays.includeFiles.length
? grepOptionArrays.includeFiles.map((includeFile) => `--include=${includeFile}`)
: []),
grepOptions.binary ? '--binary' : '',
...searchPatterns.flatMap((searchPattern) => [
'-e',
searchPattern,
]),
'--',
...searchParts,
].filter(check.isTruthy);
if (grepOptions.printCommand) {
log.faint(`> ${formatGrepCommand(grepArgs)}`);
}
const result = await runGrepCommand({
args: grepArgs,
cwd,
});
if (didGrepFail(result) || result.exitCode === 1 || !result.stdout) {
/** No matches. */
return {};
}
else if (grepOptions.output?.countOnly) {
/* node:coverage ignore next 4 */
const parsedCountOutput = isOperatingSystem(OperatingSystem.Mac) && !result.stdout.includes('\0')
? undefined
: tryParseGrepCountOutput(result.stdout);
/* node:coverage ignore next 3 */
if (parsedCountOutput) {
return parsedCountOutput;
}
return (await runGrepCountFallback({
cwd,
grepArgs,
}));
}
else if (grepOptions.output?.filesOnly) {
return parseGrepFilesOnlyOutput(result.stdout);
}
else {
return parseGrepNormalOutput(result.stdout);
}
}