@mieweb/wikigdrive
Version:
Google Drive to MarkDown synchronization
571 lines (570 loc) • 25.6 kB
JavaScript
import Transport from 'winston-transport';
import { Container } from '../../ContainerEngine.js';
import { appendConflict, DirectoryScanner, stripConflict } from './DirectoryScanner.js';
import { GoogleFilesScanner } from './GoogleFilesScanner.js';
import { convertToRelativeMarkDownPath, convertToRelativeSvgPath } from '../../LinkTranslator.js';
import { LocalFilesGenerator } from './LocalFilesGenerator.js';
import { QueueTransformer } from './QueueTransformer.js';
import { TaskLocalFileTransform } from './TaskLocalFileTransform.js';
import { MimeTypes } from '../../model/GoogleFile.js';
import { generateDirectoryYaml, parseDirectoryYaml } from './frontmatters/generateDirectoryYaml.js';
import { getContentFileService, removeMarkDownsAndImages } from './utils.js';
import { LocalLog } from './LocalLog.js';
import { LocalLinks } from './LocalLinks.js';
import { TaskRedirFileTransform } from './TaskRedirFileTransform.js';
import { TocGenerator } from './frontmatters/TocGenerator.js';
import { MarkdownTreeProcessor } from './MarkdownTreeProcessor.js';
import { LunrIndexer } from '../search/LunrIndexer.js';
import { UserConfigService } from '../google_folder/UserConfigService.js';
import { getUrlHash } from '../../utils/idParsers.js';
import { TaskGoogleMarkdownTransform } from './TaskGoogleMarkdownTransform.js';
import { frontmatter } from './frontmatters/frontmatter.js';
const __filename = globalThis[Symbol.for("import-meta-ponyfill-esmodule")](import.meta).filename;
function doesExistIn(googleFolderFiles, localFile) {
return !!googleFolderFiles.find(file => file.id === localFile.id);
}
export function solveConflicts(filesToGenerate, destinationFiles) {
const nameToConflictGroups = {};
for (const file of filesToGenerate) {
if (!nameToConflictGroups[file.fileName]) {
nameToConflictGroups[file.fileName] = [];
}
nameToConflictGroups[file.fileName].push(file);
}
const realFileNameToGenerated = {};
for (const fileName in nameToConflictGroups) {
const group = nameToConflictGroups[fileName];
if (group.length === 1) {
realFileNameToGenerated[fileName] = group[0];
}
else {
const conflictFile = group[0].type === 'md' ? {
conflicting: [],
fileName: fileName,
id: 'conflict:' + fileName,
mimeType: MimeTypes.MARKDOWN,
modifiedTime: new Date().toISOString(),
title: 'Conflict: ' + group[0].title,
type: 'conflict'
} : null;
if (conflictFile) {
realFileNameToGenerated[fileName] = conflictFile;
}
const conflictsToAssign = [];
for (const fileToGenerate of group) {
const destinationEntry = Object.entries(destinationFiles).find(f => f[1].id === fileToGenerate.id);
if (destinationEntry) {
const realFileName = destinationEntry[0];
const destinationFile = destinationEntry[1];
if (stripConflict(realFileName) === fileName) {
realFileNameToGenerated[realFileName] = destinationFile;
if (conflictFile) {
conflictFile.conflicting.push({
realFileName,
id: destinationFile.id,
title: destinationFile.title
});
}
continue;
}
}
conflictsToAssign.push(fileToGenerate);
}
let counter = 1;
for (const destinationFile of conflictsToAssign) {
let realFileName = appendConflict(destinationFile.fileName, counter++);
while (realFileNameToGenerated[realFileName]) {
realFileName = appendConflict(destinationFile.fileName, counter++);
}
realFileNameToGenerated[realFileName] = destinationFile;
if (conflictFile) {
conflictFile.conflicting.push({
realFileName,
id: destinationFile.id,
title: destinationFile.title
});
}
}
}
}
return realFileNameToGenerated;
}
function processLogExisting(realFileName, fileToGenerate, destinationFiles, localLog, prefix) {
const destinationEntry = Object.entries(destinationFiles).find(item => item[1].id === fileToGenerate.id);
if (destinationEntry) {
if (destinationEntry[0] !== realFileName) {
localLog.append({
filePath: prefix + realFileName,
id: fileToGenerate.id,
type: fileToGenerate.type,
event: 'renamed',
});
}
else {
localLog.append({
filePath: prefix + realFileName,
id: fileToGenerate.id,
type: fileToGenerate.type,
event: 'touched',
});
}
}
else {
localLog.append({
filePath: prefix + realFileName,
id: fileToGenerate.id,
type: fileToGenerate.type,
event: 'created',
});
}
}
function processLogRemoved(realFileName, destinationFiles, localLog, prefix) {
const destinationFile = destinationFiles[realFileName];
const entryToGenerate = Object.entries(destinationFiles).find(item => item[1].id === destinationFile.id);
if (!entryToGenerate) {
localLog.append({
filePath: prefix + realFileName,
id: destinationFile.id,
type: destinationFile.type,
event: 'removed',
});
}
}
async function addBinaryMetaData(destinationFiles, destinationDirectory) {
const yamlContent = await destinationDirectory.exists('.wgd-directory.yaml') ?
await destinationDirectory.readFile('.wgd-directory.yaml') : '';
const props = parseDirectoryYaml(yamlContent);
const map = props?.fileMap || {};
for (const realFileName in destinationFiles) {
const destinationFile = destinationFiles[realFileName];
if (destinationFile.id !== 'TO_FILL') {
continue;
}
const mapData = map[realFileName];
if (!mapData) {
continue;
}
destinationFile.fileName = mapData.fileName;
destinationFile.id = mapData.id;
destinationFile.modifiedTime = mapData.modifiedTime;
}
}
export class TransformLog extends Transport {
constructor(options = {}) {
super(options);
Object.defineProperty(this, "errors", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
}
log(info, next) {
switch (info.level) {
case 'error':
case 'warn':
if (info.errorMdFile) {
if (!this.errors[info.errorMdFile]) {
this.errors[info.errorMdFile] = [];
}
if (info.errorMdMsg) {
this.errors[info.errorMdFile].push(info.errorMdMsg);
}
}
}
if (next) {
next();
}
}
}
export class TransformContainer extends Container {
constructor(params, paramsArr = {}) {
super(params, paramsArr);
Object.defineProperty(this, "params", {
enumerable: true,
configurable: true,
writable: true,
value: params
});
Object.defineProperty(this, "paramsArr", {
enumerable: true,
configurable: true,
writable: true,
value: paramsArr
});
Object.defineProperty(this, "logger", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "generatedFileService", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "localLog", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "localLinks", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "filterFilesIds", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "userConfigService", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "progressNotifyCallback", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "transformLog", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "isFailed", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "useGoogleMarkdowns", {
enumerable: true,
configurable: true,
writable: true,
value: false
});
Object.defineProperty(this, "globalHeadersMap", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
Object.defineProperty(this, "globalInvisibleBookmarks", {
enumerable: true,
configurable: true,
writable: true,
value: {}
});
this.filterFilesIds = paramsArr['filesIds'] || [];
}
async mount2(fileService, destFileService) {
this.filesService = fileService;
this.generatedFileService = destFileService;
this.userConfigService = new UserConfigService(this.filesService);
await this.userConfigService.load();
}
async init(engine) {
await super.init(engine);
this.logger = engine.logger.child({ filename: __filename, driveId: this.params.folderId, jobId: this.params.jobId });
this.transformLog = new TransformLog();
this.logger.add(this.transformLog);
}
async syncDir(googleFolder, destinationDirectory, queueTransformer) {
const googleScanner = new GoogleFilesScanner();
if (!await googleFolder.exists('.folder.json')) {
return;
}
const googleFolderData = await googleFolder.readJson('.folder.json') || {};
const googleFolderFiles = await googleScanner.scan(googleFolder);
const destinationScanner = new DirectoryScanner();
const destinationFiles = await destinationScanner.scan(destinationDirectory);
await addBinaryMetaData(destinationFiles, destinationDirectory);
const localFilesGenerator = new LocalFilesGenerator();
const filesToGenerate = await localFilesGenerator.generateLocalFiles(googleFolderFiles);
const realFileNameToGenerated = solveConflicts(filesToGenerate, destinationFiles);
for (const realFileName in destinationFiles) {
if (realFileName.startsWith('.')) {
continue;
}
processLogRemoved(realFileName, destinationFiles, this.localLog, destinationDirectory.getVirtualPath());
const fileInDirectory = destinationFiles[realFileName];
if (!doesExistIn(filesToGenerate, fileInDirectory)) {
await removeMarkDownsAndImages(realFileName, destinationDirectory);
}
if (fileInDirectory.type === 'redir' || fileInDirectory.type === 'conflict') {
await removeMarkDownsAndImages(realFileName, destinationDirectory);
}
if (!realFileNameToGenerated[realFileName]) {
await removeMarkDownsAndImages(realFileName, destinationDirectory);
}
}
for (const realFileName in realFileNameToGenerated) {
const localFile = realFileNameToGenerated[realFileName];
processLogExisting(realFileName, localFile, destinationFiles, this.localLog, destinationDirectory.getVirtualPath());
if (localFile.type === 'directory') {
await destinationDirectory.mkdir(realFileName);
const googleFolderFile = googleFolderFiles.find(f => f.id === localFile.id);
if (googleFolderFile) {
const googleSubFolder = await googleFolder.getSubFileService(googleFolderFile.id);
await this.syncDir(googleSubFolder, await destinationDirectory.getSubFileService(realFileName), queueTransformer);
}
continue;
}
const googleFile = googleFolderFiles.find(f => f.id === localFile.id);
if (this.filterFilesIds.length > 0 && -1 === this.filterFilesIds.indexOf(localFile.id)) {
continue;
}
const jobManagerContainer = this.engine.getContainer('job_manager');
if (!this.useGoogleMarkdowns) {
const task = new TaskLocalFileTransform(this.logger, jobManagerContainer, realFileName, googleFolder, googleFile, destinationDirectory, localFile, this.localLinks, this.userConfigService.config, this.globalHeadersMap, this.globalInvisibleBookmarks);
queueTransformer.addTask(task);
}
else {
const task = new TaskGoogleMarkdownTransform(this.logger, jobManagerContainer, realFileName, googleFolder, googleFile, destinationDirectory, localFile, this.localLinks, this.userConfigService.config);
queueTransformer.addTask(task);
}
}
const dirNames = destinationDirectory.getVirtualPath().replace(/\/$/, '').split('/');
const yaml = generateDirectoryYaml(stripConflict(dirNames[dirNames.length - 1]), googleFolderData, realFileNameToGenerated);
await destinationDirectory.writeFile('.wgd-directory.yaml', yaml);
}
async run(rootFolderId) {
if (!(this.userConfigService.config.transform_subdir || '').startsWith('/')) {
this.logger.warn('Content subdirectory must be set and start with /');
return;
}
const contentFileService = await getContentFileService(this.generatedFileService, this.userConfigService);
const queueTransformer = new QueueTransformer(this.logger);
queueTransformer.onProgressNotify(({ total, completed, warnings, failed }) => {
if (failed > 0) {
this.isFailed = true;
}
if (this.progressNotifyCallback) {
this.progressNotifyCallback({ total, completed, warnings, failed });
}
});
this.logger.info('Start transforming: ' + rootFolderId);
this.localLog = new LocalLog(contentFileService);
await this.localLog.load();
this.localLinks = new LocalLinks(contentFileService);
await this.localLinks.load();
const processed = new Set();
const previouslyFailed = new Set();
let retry = true;
while (retry) {
retry = false;
await this.syncDir(this.filesService, contentFileService, queueTransformer);
await queueTransformer.finished();
if (this.filterFilesIds.length > 0) {
const filterFilesIds = new Set();
for (const fileId of this.filterFilesIds) {
processed.add(fileId);
const backLinks = this.localLinks.getBackLinks(fileId);
for (const backLink of backLinks) {
if (processed.has(backLink.fileId)) {
continue;
}
filterFilesIds.add(backLink.fileId);
}
}
if (filterFilesIds.size > 0) {
if (previouslyFailed.size === filterFilesIds.size) {
let shouldBreak = true;
for (const fileId of previouslyFailed) {
if (filterFilesIds.has(fileId)) {
shouldBreak = false;
break;
}
}
if (shouldBreak) {
break;
}
}
this.filterFilesIds = Array.from(filterFilesIds);
previouslyFailed.clear();
for (const fileId of filterFilesIds) {
previouslyFailed.add(fileId);
}
retry = true;
}
}
}
await queueTransformer.finished();
await contentFileService.remove('_errors.md');
if (Object.keys(this.transformLog.errors).length > 0) {
let errorLog = '';
errorLog += '---\n';
errorLog += 'type: \'page\'\n';
errorLog += '---\n';
for (const mdFile in this.transformLog.errors) {
errorLog += `\n* [${mdFile}](${mdFile})\n`;
for (const mdMsg of this.transformLog.errors[mdFile]) {
errorLog += ` ${mdMsg}\n`;
}
}
await contentFileService.writeFile('_errors.md', errorLog);
}
await this.createRedirs(contentFileService);
await this.writeToc(contentFileService);
await this.rewriteLinks(contentFileService);
await this.localLog.save();
await this.localLinks.save();
this.logger.info('Regenerate tree: ' + rootFolderId + ` to: ${contentFileService.getRealPath()}/.tree.json`);
const markdownTreeProcessor = new MarkdownTreeProcessor(contentFileService);
await markdownTreeProcessor.regenerateTree(rootFolderId);
await markdownTreeProcessor.save();
const indexer = new LunrIndexer();
await markdownTreeProcessor.walkTree((page) => {
indexer.addPage(page);
return false;
});
await this.generatedFileService.mkdir('/.private');
await this.generatedFileService.writeJson('/.private/lunr.json', indexer.getJson());
}
failed() {
return this.isFailed;
}
async rewriteLinks(destinationDirectory) {
const files = await destinationDirectory.list();
for (const fileName of files) {
if (await destinationDirectory.isDirectory(fileName)) {
await this.rewriteLinks(await destinationDirectory.getSubFileService(fileName));
continue;
}
if (fileName.endsWith('.md') || fileName.endsWith('.svg')) {
const content = await destinationDirectory.readFile(fileName);
const parsed = frontmatter(content);
const props = parsed.data;
let newContent = content;
if (props?.id) {
newContent = newContent.replace(/\n? ?<a id="([^"]*)"><\/a>\n?/igm, (str, hash) => {
const fullLink = 'gdoc:' + props.id + '#' + hash;
if (this.globalInvisibleBookmarks[fullLink]) {
const retVal = str.replace(`<a id="${hash}"></a>`, '');
if (retVal === '\n \n') {
return '\n';
}
if (retVal === '\n\n') {
return '\n';
}
if (retVal.endsWith(' \n')) {
return retVal.substring(0, retVal.length - 2) + '\n';
}
if (retVal.startsWith('\n ')) {
return '\n' + retVal.substring(1);
}
if (retVal === ' ') {
return '';
}
return retVal;
}
else {
this.logger.warn(`In ${fileName} there is a link to ${fullLink} which can't be translated into bookmark link`);
}
return str;
});
}
newContent = newContent.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str) => {
let fileId = str.substring('gdoc:'.length).replace(/#.*/, '');
let hash = getUrlHash(str) || '';
if (hash) {
if (this.globalHeadersMap[str]) {
const idx = this.globalHeadersMap[str].indexOf('#');
if (idx >= 0) {
fileId = this.globalHeadersMap[str].substring('gdoc:'.length, idx);
hash = this.globalHeadersMap[str].substring(idx);
}
}
else {
const fullLink = str;
this.logger.warn(`In ${fileName} there is a link to ${fullLink} which can't be translated into bookmark link`);
}
}
const lastLog = this.localLog.findLastFile(fileId);
if (lastLog && lastLog.event !== 'removed') {
if (fileName.endsWith('.svg')) {
return convertToRelativeSvgPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName);
}
else {
return convertToRelativeMarkDownPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName) + hash;
}
}
else {
return 'https://drive.google.com/open?id=' + fileId + hash.replace('#_', '#heading=h.');
}
});
if (content !== newContent) {
await destinationDirectory.writeFile(fileName, newContent);
}
}
}
}
async createRedirs(contentFileService) {
const rows = this.localLog.getLogs();
const markDownScanner = new DirectoryScanner();
const transformerQueue = new QueueTransformer(this.logger);
transformerQueue.onProgressNotify(({ total, completed, warnings, failed }) => {
if (this.progressNotifyCallback) {
this.progressNotifyCallback({ total, completed, warnings, failed });
}
});
for (let rowNo = rows.length - 1; rowNo >= 0; rowNo--) {
const row = rows[rowNo];
if (row.type === 'md' && !await contentFileService.exists(row.filePath)) {
const lastLog = this.localLog.findLastFile(row.id);
if (lastLog) {
const parts = row.filePath.split('/');
const fileName = parts.pop();
const dirName = parts.join('/');
if (!await contentFileService.exists(lastLog.filePath)) {
continue;
}
const localFileContent = await contentFileService.readFile(lastLog.filePath);
const localFile = markDownScanner.parseMarkdown(localFileContent, lastLog.filePath);
if (!localFile) {
continue;
}
const lastLogRedir = this.localLog.findLastFileByPath(dirName ? dirName + '/' + fileName : fileName);
if (lastLogRedir?.event === 'removed') {
continue;
}
const redirFile = {
type: 'redir',
fileName,
id: row.id,
mimeType: MimeTypes.MARKDOWN,
modifiedTime: new Date(row.mtime).toISOString(),
redirectTo: lastLog.id,
title: 'Redirect to: ' + localFile.title,
};
const task = new TaskRedirFileTransform(this.logger, fileName, dirName ? await contentFileService.getSubFileService(dirName) : contentFileService, redirFile, localFile);
transformerQueue.addTask(task);
}
}
}
await transformerQueue.finished();
}
async writeToc(contentFileService) {
const tocGenerator = new TocGenerator();
const md = await tocGenerator.generate(contentFileService);
await contentFileService.writeFile('toc.md', md);
}
// eslint-disable-next-line @typescript-eslint/no-empty-function
async destroy() {
}
onProgressNotify(callback) {
this.progressNotifyCallback = callback;
}
setUseGoogleMarkdowns(value) {
this.useGoogleMarkdowns = value;
}
}