taraskevizer
Version:
Канвэртацыя акадэмічнага правапісу ў клясычны
187 lines (186 loc) • 8.82 kB
JavaScript
import { cpus } from 'node:os';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
import { Worker } from 'node:worker_threads';
import { pipelines } from '../index.js';
import { parseArgs } from './parse-args.js';
const getPrint = (stream) => (msg) => {
stream.write("[34m[taraskevizer][0m" + ' ' + msg);
};
const getPrintLn = (printFn) => (msg) => {
printFn(msg + '\n');
};
const print = getPrint(process.stdout);
const printErr = getPrint(process.stderr);
const printLn = getPrintLn(print);
const printErrLn = getPrintLn(printErr);
// AI-written, may need improvements
const splitIntoChunks = (text, n) => {
const size = Math.ceil(text.length / n);
const chunks = [];
let start = 0;
for (let i = 0; i < n; i++) {
let end = start + size;
// Adjust end to the nearest newline character
if (end < text.length) {
const forward = text.indexOf('\n', end);
const backward = text.lastIndexOf('\n', end);
if (forward === -1 && backward === -1) {
// no-op
}
else if (forward === -1) {
end = backward;
}
else if (backward === -1) {
end = forward;
}
else {
end = forward - end < end - backward ? forward : backward;
}
}
// Ensure we don't split inside HTML tags or special syntax tags
if (end < text.length) {
const lastOpen = text.lastIndexOf('<', end);
const lastClose = text.lastIndexOf('>', end);
if (lastOpen > lastClose) {
const nextClose = text.indexOf('>', end);
if (nextClose !== -1) {
end = nextClose + 1;
}
}
}
if (end > text.length)
end = text.length;
chunks.push(text.slice(start, end));
start = end;
if (start >= text.length)
break;
}
return chunks;
};
const getPrettyByteSize = (n) => n < 1024
? `${n} B`
: n < 1024 * 1024
? `${(n / 1024).toFixed(2)} KB`
: `${(n / (1024 * 1024)).toFixed(2)} MB`;
process.argv.splice(0, 2);
const firstArg = process.argv[0];
if (firstArg) {
if (firstArg === '-v' || firstArg === '--version') {
printLn("10.4.20");
process.exit(0);
}
if (firstArg === '-h' || firstArg === '--help') {
printLn("Usage: \u001b[34mtarask\u001b[0m [options] text\nIf text is not passed, interactive mode is enabled\n\nEXAMPLES\n\nConvert and latinize a word\n\t\u001b[34mtarask\u001b[0m \u001b[35m--latin\u001b[0m 'планета'\nPrints \"p\u001b[32ml\u001b[0ma\u001b[32mne\u001b[0mta\"\n\nRead from one file and write converted text to another\n\t\u001b[34mtarask\u001b[0m < ./cyr-text.txt > ./lat-text.txt\n\nEnter interactive mode\n\t\u001b[34mtarask\u001b[0m\nPrints \"\u001b[34m[taraskevizer]\u001b[0m Enter the text:\" and waits until you enter a new line\n\nOPTIONS\n\n\u001b[33mGeneral\u001b[0m:\n \u001b[35m-h\u001b[0m \u001b[35m--help\u001b[0m\n \u001b[35m-v\u001b[0m \u001b[35m--version\u001b[0m\n\n\u001b[33mAlphabet\u001b[0m:\n \u001b[35m-l\u001b[0m \u001b[35m--latin\u001b[0m\n \u001b[35m-lj\u001b[0m \u001b[35m--latin-ji\u001b[0m\n \u001b[35m-a\u001b[0m \u001b[35m--arabic\u001b[0m\n\n\u001b[33mWhen to replace і(i) by й(j) after vowels\u001b[0m:\n \u001b[35m-jr\u001b[0m \u001b[35m--jrandom\u001b[0m\n \u001b[35m-ja\u001b[0m \u001b[35m--jalways\u001b[0m\n\n\u001b[33mReplace ґ(g) by г(h) in cyrillic alphabet\u001b[0m:\n \u001b[35m--h\u001b[0m\n\n\u001b[33mVariations\u001b[0m:\n \u001b[35m-nv\u001b[0m \u001b[35m--no-variations\u001b[0m\n \u001b[35m-fv\u001b[0m \u001b[35m--first-variation\u001b[0m\n\n\u001b[33mMode (only one can be used)\u001b[0m:\n \u001b[35m-html\u001b[0m \u001b[35m--html\u001b[0m\n \u001b[35m-abc\u001b[0m \u001b[35m--alphabet-only\u001b[0m\n\n\u001b[33mOther\u001b[0m:\n \u001b[35m-nec\u001b[0m \u001b[35m--not-escape-caps\u001b[0m\n \u001b[35m-nc\u001b[0m \u001b[35m--no-color\u001b[0m\n \u001b[35m-st\u001b[0m \u001b[35m--single-thread\u001b[0m\n");
process.exit(0);
}
}
const argv = process.argv.slice();
const { mode, cfg, doForceSingleThread } = parseArgs(process.argv);
const workers = {
size: cpus()?.length || 1,
workers: null,
init() {
if (this.workers)
return;
process.stderr.write(`(Initializing ${this.size} workers... `);
const dirname = path.dirname(fileURLToPath(import.meta.url));
this.workers = Array.from({ length: this.size }, () => new Worker("var {\n\tparentPort,\n\tworkerData: { argv, dirname },\n} = require('node:worker_threads');\nvar { resolve } = require('node:path');\nvar { pipelines } = require(resolve(dirname, '..'));\nvar { parseArgs } = require(resolve(dirname, 'parse-args'));\n\nvar { mode, cfg } = parseArgs(argv);\n\nparentPort.on('message', (chunk) => {\n\tparentPort.postMessage(pipelines[mode](chunk, cfg));\n});\n", {
eval: true,
workerData: { argv, dirname },
}));
process.stderr.write('done.) ');
},
process(chunks) {
return Promise.all(chunks.map((chunk, i) => new Promise((resolve, reject) => {
const worker = this.workers[i % this.size];
worker.postMessage(chunk);
worker.once('message', resolve);
worker.once('error', reject);
})));
},
};
const convert = (text) => pipelines[mode](text, cfg);
const processText = async (text) => {
let result = '';
if (!doForceSingleThread && workers.size > 1 && text.length > 50_000) {
workers.init();
const chunks = splitIntoChunks(text, workers.size);
const results = await workers.process(chunks);
result = results.join('');
}
else {
result = convert(text);
}
if (!process.stdout.write(result)) {
process.stdout.once('drain', () => {
printErrLn('Drain event fired, exiting.');
process.exit(0);
});
}
};
if (process.argv.length) {
process.stdout.write(convert(process.argv.reverse().join(' ')) + '\n');
}
else {
const chunks = [];
let value = '';
let length = 0;
let byteLength = 0;
const MAX_BYTE_LENGTH = 64 * 1024 * 1024;
const getChunksString = () => Buffer.concat(chunks, length).toString();
if (process.stdin.isTTY) {
printErrLn('Enter the text');
for await (const chunk of process.stdin) {
chunks.push(chunk);
length += chunk.length;
if (chunk.includes('\n'))
break;
}
process.stdout.write(convert(getChunksString()));
}
else {
printErrLn('Reading from stdin...');
const processTextWithLogs = async (value) => {
printErr(`Processing ${value.length
.toString()
.replace(/\B(?=(\d{3})+(?!\d))/g, ' ')} characters (${getPrettyByteSize(Buffer.byteLength(value))}) chunk... `);
const startTime = performance.now();
await processText(value);
process.stderr.write(`done in ${((performance.now() - startTime) / 1000).toFixed(2)} seconds.\n`);
};
for await (const chunk of process.stdin) {
byteLength += chunk.byteLength;
if (byteLength >= MAX_BYTE_LENGTH) {
value += getChunksString();
const lastNewlineIndex = value.lastIndexOf('\n');
if (lastNewlineIndex === -1) {
printErrLn('\nInput exceeded maximum size of ' +
MAX_BYTE_LENGTH +
' bytes without a newline. Stopping.');
process.exit(1);
}
let valueForNextBatch = value.slice(lastNewlineIndex + 1);
value = value.slice(0, lastNewlineIndex + 1);
const lastOpeningTagIndex = value.lastIndexOf('<');
const lastClosingTagIndex = value.lastIndexOf('>');
if (lastOpeningTagIndex !== -1 &&
lastOpeningTagIndex > lastClosingTagIndex) {
const incompleteTag = value.slice(lastOpeningTagIndex);
value = value.slice(0, lastOpeningTagIndex);
valueForNextBatch = incompleteTag + valueForNextBatch;
}
await processTextWithLogs(value);
value = valueForNextBatch;
byteLength = chunk.byteLength;
length = 0;
chunks.length = 0;
}
length += chunk.length;
chunks.push(chunk);
}
await processTextWithLogs(value + getChunksString());
}
}
process.exit(0);