indexdump
Version:
Manticore Search index dump utility
251 lines (248 loc) • 10.5 kB
JavaScript
import * as fs from 'fs';
import * as path from 'path';
import margv from 'margv';
import mariadb from 'mariadb';
import { stdout } from 'process';
import { performance } from 'perf_hooks';
import chalk from 'chalk';
import archiver from 'archiver';
import { Writable, PassThrough } from 'stream';
(async () => {
const start = performance.now();
const argv = margv();
const showVersion = argv.v || argv.version;
if (showVersion) {
stdout.write("indexdump: 1.2.4\n");
process.exit(0);
}
if (argv.$.length < 3) {
console.error("Error parse arguments. Use: -h<host> -P<port> <index> > dump.sql");
process.exit(0);
}
const showHelp = !!argv.help;
if (showHelp) {
const TAB3 = " ";
const TAB4 = " ";
const TAB5 = " ";
stdout.write(chalk.green("List of settings\n"));
stdout.write(chalk.bold("-h/--host") + TAB4 + "host (default: 127.0.0.1)\n");
stdout.write(chalk.bold("-P/--port") + TAB4 + "port (default: 9306)\n");
stdout.write(chalk.bold("--dry-run") + TAB4 + "run in dry mode\n");
stdout.write(chalk.bold("-ch/--chunk") + TAB4 + "chunk size for bulk inserts\n");
stdout.write(chalk.bold("--add-drop-index\n--add-drop-table") + " add DROP TABLE\n");
stdout.write(chalk.bold("--add-locks") + TAB4 + "add LOCK " + chalk.red("not currently implemented in Manticore Search\n"));
stdout.write(chalk.bold("--add-freeze") + TAB4 + "add FREEZE\n");
stdout.write(chalk.bold("--to-index") + TAB4 + "rename to index in backup " + chalk.red("only for single index\n"));
stdout.write(chalk.bold("--prefix") + TAB4 + "add prefix for all indexes\n");
stdout.write(chalk.bold("--indexes test1\n--indexes test2\n--indexes=test1,test2") + " indexes list for dump\n");
stdout.write(chalk.bold("--all") + TAB5 + "all indexes\n");
stdout.write(chalk.bold("--limit") + TAB5 + "add limit for all indexes, --limit=0 dump only structure\n");
stdout.write(chalk.bold("--path") + TAB5 + "path from which the index will be restored (default: current)\n");
stdout.write(chalk.bold("--data-dir") + TAB4 + "allow to set manticore data path\n");
stdout.write(chalk.bold("--add-config") + TAB3 + "add manticore.json to dump\n");
process.exit(0);
}
const host = (argv.$.find((v) => v.indexOf("-h") === 0) || "").replace("-h", "") || argv["h"] || argv["host"] || "127.0.01";
const port = (argv.$.find((v) => v.indexOf("-P") === 0) || "").replace("-P", "") || argv["P"] || argv["port"] || 9306;
const chunk = (argv.$.find((v) => v.indexOf("-ch") === 0) || "").replace("-ch", "") || argv["ch"] || argv["chunk"] || 1e3;
const dropTable = !!(argv["add-drop-index"] || argv["add-drop-table"]);
const addLocks = !!argv["add-locks"];
const addFreeze = !!argv["add-freeze"];
const toTable = argv["to-index"] || argv["to-table"];
const toPrefix = argv["prefix"] || "";
const tables = argv["indexes"] || argv["tables"];
const isAll = argv["all"];
const limit = argv["limit"] ? parseInt(argv["limit"]) : argv["limit"];
const filesPath = argv["path"] || path.resolve();
const index = tables || argv.$.pop();
const dryRun = !!argv["dry-run"];
const dataDir = argv["data-dir"] || "";
const addConfig = !!argv["add-config"];
const filesToCopy = [];
if (toTable && tables) {
console.error("Error parse arguments. You can`t use to-table and tables together");
process.exit(0);
}
dryRun && stdout.write(chalk.yellow("\n----- Start dry run -----\n"));
const devNull = new Writable({
write(chunk2, encoding, callback) {
setImmediate(callback);
}
});
const arch = archiver("tar", { gzip: true });
!dryRun && arch.pipe(stdout);
dryRun && arch.pipe(devNull);
const promisifyStream = (file, name) => {
if (!dryRun) {
try {
fs.accessSync(file, fs.constants.R_OK);
} catch (e) {
return;
}
}
dryRun && stdout.write(`${file} => ./${name}`);
return new Promise((resolve) => {
const entryListener = () => {
arch.off("entry", entryListener);
arch.off("error", errorListener);
dryRun && stdout.write(chalk.green(" done\n"));
resolve(true);
};
const errorListener = () => {
arch.off("entry", entryListener);
arch.off("error", errorListener);
dryRun && stdout.write(chalk.red(" not accessible\n"));
resolve(true);
};
arch.append(fs.createReadStream(file), { name }).on("entry", entryListener).on("error", errorListener);
});
};
const createStream = (name) => {
dryRun && stdout.write(`./${name}
`);
const stream2 = new PassThrough();
arch.append(stream2, { name });
return stream2;
};
const pathReplacer = async (strIndexCreate, pathToFile, index2) => {
for (const label of ["exceptions", "stopwords", "wordforms"]) {
const matches = strIndexCreate.match(RegExp(`${label}='([^']*)'`));
const files = (matches?.[1]?.split(/\s+/) || []).filter((file) => /^([A-Z]:)?\//.test(file));
if (files.length) {
const newFiles = files.map((file) => path.resolve(pathToFile, `./${path.basename(file)}`));
const newFilesString = newFiles.join(" ");
strIndexCreate = strIndexCreate.replace(matches[0], `${label}='${newFilesString}'`);
files.map((file) => filesToCopy.push({ file, path: path.join(index2, path.basename(file)) }));
}
}
return strIndexCreate;
};
const conn = await mariadb.createConnection({ host, port });
const dumpTable = async (index2, output) => {
const toIndex = toTable ? `${toPrefix}${toTable}` : `${toPrefix}${index2}`;
const startChunk = (cols) => output.write(`INSERT INTO ${toIndex} (${cols.join(",")}) VALUES`);
const endChunk = (rows) => output.write(`${rows.join(",\n")};
`);
output.write(`-- START DUMP ${index2} --
`);
toIndex !== (index2 || toPrefix) && output.write(`-- WITH TABLE NAME CHANGE TO ${toIndex} --
`);
if (dropTable) {
output.write(`DROP TABLE IF EXISTS ${toIndex};
`);
}
const describe = await conn.query(`DESCRIBE ${index2};`);
const types = describe.reduce((ac, row) => {
ac[row["Field"]] = row["Type"];
if (row["Type"] === "text" && row["Properties"].indexOf("stored") === -1) {
output.write(`-- WARNING: field ${row["Field"]} NOT stored, so not included in output --
`);
}
return ac;
}, {});
const allowsFields = describe.filter((row) => !["tokencount"].includes(row["Type"])).map((row) => row["Field"]);
const allFields = describe.map((row) => row["Field"]);
const createTableString = (await conn.query(`SHOW CREATE TABLE ${index2};`))[0]["Create Table"].replace(RegExp(`CREATE[\\s\\t]+TABLE[\\s\\t]+${index2}`, "i"), `CREATE TABLE ${toIndex}`);
let createTableFiltered = createTableString.split(/\r?\n/).filter((row) => !allFields.includes(row.split(/\s+/)[0]) || allowsFields.includes(row.split(/\s+/)[0])).join("\n");
createTableFiltered = await pathReplacer(createTableFiltered, path.resolve(filesPath, `./${toIndex}`), toIndex);
output.write(createTableFiltered + ";\n");
if (limit !== 0) {
let count = 0, lastId = 0, next = true;
while (next) {
let i = 0, rows = [];
addLocks && await conn.query(`LOCK TABLES ${index2} WRITE;`);
addFreeze && await conn.query(`FREEZE ${index2}`);
await new Promise((resolve) => {
const stream2 = conn.queryStream(`SELECT *
FROM ${index2}
WHERE id > ${lastId}
ORDER BY id ASC
LIMIT ${chunk}
OPTION max_matches =
${chunk};`);
stream2.on("error", (err) => {
console.error(err);
process.exit(0);
}).on("data", (row) => {
Object.keys(row).forEach((key) => {
if (!allowsFields.includes(key)) {
delete row[key];
}
});
i === 0 && startChunk(Object.keys(row));
if (!limit || count < limit) {
rows.push(`(${Object.entries(row).map(([key, value]) => types[key] === "mva" ? `(${value})` : conn.escape(value)).join(",")})`);
lastId = row["id"];
i++;
count++;
}
if (limit && limit === count) {
next = false;
}
}).on("end", async () => {
if (!rows.length) {
next = false;
} else {
endChunk(rows);
}
addFreeze && await conn.query(`UNFREEZE ${index2}`);
addLocks && await conn.query(`UNLOCK TABLES;`);
resolve(true);
});
});
}
output.write(`-- COUNT: ${count} --
`);
}
output.write(`-- END DUMP ${index2} --
`);
};
let tablesList = [index];
if (tables) {
if (Array.isArray(tables)) {
tablesList = tables;
} else {
tablesList = tables.split(",");
}
}
if (isAll) {
tablesList = (await conn.query(`SHOW TABLES;`)).map((row) => row["Index"]);
}
const stream = createStream("dump.sql");
for (const current of tablesList) {
dryRun && stdout.write(`${current}`);
await dumpTable(current, stream);
dryRun && stdout.write(chalk.green(" done\n"));
}
stream.write(`
`);
const interval = Math.round((performance.now() - start) / 10) / 100;
stream.write(`-- TIME: ${interval}s --
`);
stream.end();
await new Promise((resolve) => setTimeout(resolve, 100));
for (const { file, path: path2 } of filesToCopy) {
await promisifyStream(file, path2);
}
if ((isAll || addConfig) && filesToCopy.length) {
const base = dataDir || path.dirname(filesToCopy[0].file);
if (base) {
const manticoreJson = path.join(base, "..", "manticore.json");
try {
dryRun && stdout.write(chalk.green(`manticore.json
`));
await promisifyStream(manticoreJson, path.basename(manticoreJson));
} catch (e) {
}
}
}
await new Promise((resolve) => setTimeout(resolve, 100));
dryRun && stdout.write(`
`);
dryRun && stdout.write(`TIME: ${interval}s
`);
await arch.finalize();
await conn.end();
dryRun && stdout.write(chalk.yellow("----- End dry run -----\n\n"));
})();