awesome-gadgets
Version:
Storage, management, compilation, and automatic deployment of MediaWiki gadgets.
853 lines (754 loc) • 23.9 kB
text/typescript
import type {
Api,
Credentials,
CredentialsOnlyPassword,
DeploymentDirectTargets,
DeploymentTargets,
GlobalSourceFiles,
} from '../types';
import {CONVERT_VARIANT, DEFINITION_SECTION_MAP, VARIANTS} from '../../constant';
import {
__rootDir,
exec,
generateBannerAndFooter,
generateDefinition,
prompt,
readFileContent,
sortObject,
trim,
writeFileContent,
} from './general-util';
import {env, exit, stdout} from 'node:process';
import {existsSync, open} from 'node:fs';
import {ESLint} from 'eslint';
import {MwnError} from 'mwn/build/error';
import {Window} from 'happy-dom';
import alphaSort from 'alpha-sort';
import {apiQueue} from '../deploy';
import chalk from 'chalk';
import {globSync} from 'glob';
import path from 'node:path';
import {setTimeout} from 'node:timers/promises';
/**
* @private
*/
const deployPages: Record<string, string[]> = {};
/**
* @private
*/
const eslintOnlyAllowES5 = new ESLint({
overrideConfig: {
plugins: ['eslint-plugin-es5'],
extends: ['plugin:es5/no-es2015', 'plugin:es5/no-es2016'],
env: {
browser: true,
},
parserOptions: {
ecmaVersion: 5,
},
},
useEslintrc: false,
});
/**
* Read `dist/definition.txt`
*
* @return {string|undefined} Gadget definitions (in the format of MediaWiki:Gadgets-definition item)
* @throws If `dist/definition.txt` is not found
*/
const readDefinition = () => {
const definitionPath = path.join(__rootDir, 'dist/definition.txt');
if (!existsSync(definitionPath)) {
console.log(chalk.red(`Failed to read ${chalk.italic('definition.txt')}, please try build again.`));
exit(1);
}
const fileContent = readFileContent(definitionPath);
return fileContent;
};
/**
* Generate deployment targets based on the definitions
*
* @return {DeploymentTargets} Deployment targets
*/
const generateTargets = () => {
const targets: DeploymentTargets = {};
const definitionText = readDefinition();
const definitions = definitionText.split('\n').filter((lineContent) => {
return /^\*\s\S+?\[ResourceLoader[|\]]/.test(lineContent);
});
const gadgetNames = definitions.map<string>((definition) => {
const regExpMatchArray = definition.match(/^\*\s(\S+?)\[/) as [string, string];
return regExpMatchArray[1];
});
for (const gadgetName of gadgetNames.toSorted(
alphaSort({
caseInsensitive: true,
natural: true,
})
)) {
const definition = generateDefinition(gadgetName, false);
const {description, enable, excludeSites} = definition;
if (enable === false) {
continue;
}
targets[gadgetName] = {
excludeSites,
description:
trim(description, {
addNewline: false,
}) || gadgetName,
files: [],
};
const distFiles = globSync([`${gadgetName}/*.{css,js}`], {
cwd: path.join(__rootDir, 'dist'),
withFileTypes: true,
});
for (const file of distFiles) {
const {name: fileName} = file;
const fileExt: string = path.extname(fileName);
const fileBaseName: string = path.basename(fileName, fileExt);
targets[gadgetName].files.push([
fileName,
fileBaseName === gadgetName ? fileName : `${gadgetName}-${fileName}`,
]);
}
}
for (const [gadgetName, {files}] of Object.entries(targets)) {
if (!files.length) {
delete targets[gadgetName];
}
}
if (!Object.keys(targets).length) {
console.log(chalk.yellow('━ No gadget need to deploy'));
return {};
}
return targets;
};
/**
* Generate direct deployment targets based on the `global.json`
*
* @param {Api['site']} site The target site name
* @return {Promise<DeploymentDirectTargets>} Direct deployment targets
*/
const generateDirectTargets = async (site: Api['site']) => {
const targets: DeploymentDirectTargets = [];
interface GlobalJsonObject {
[key: string]: {
[K in keyof GlobalSourceFiles]: Omit<GlobalSourceFiles[K], 'contentType'>;
};
}
const globalJsonFilePath = path.join(__rootDir, 'src/global.json');
if (!existsSync(globalJsonFilePath)) {
console.log(chalk.white(`━ No ${chalk.bold('global.json')} found`));
return [];
}
const globalJsonText = readFileContent(globalJsonFilePath);
let globalJsonObject: GlobalJsonObject = {};
try {
globalJsonObject = JSON.parse(globalJsonText) as GlobalJsonObject;
} catch {
console.log(chalk.red(`✘ Failed to parse ${chalk.bold('global.json')}`));
return [];
}
const currentSiteGlobalTargets = globalJsonObject[site];
if (!currentSiteGlobalTargets) {
return [];
}
for (const [file, {enable, sourceCode, licenseText}] of Object.entries(currentSiteGlobalTargets)) {
if (enable === false) {
continue;
}
const contentType = file.endsWith('.css')
? 'text/css'
: file.endsWith('.js')
? 'application/javascript'
: undefined;
if (!/^(?!Gadget-).+\.(css|js)$/i.test(file)) {
if (!contentType) {
console.log(chalk.yellow(`━ Skip ${chalk.bold(file)}, only allowed to deploy CSS or JavaScript pages`));
continue;
}
console.log(chalk.yellow(`━ Skip invalid target page ${chalk.bold(file)}`));
continue;
}
const {banner, footer} = generateBannerAndFooter({
licenseText,
isDirectly: true,
});
switch (contentType) {
case 'application/javascript': {
const lintResult = await eslintOnlyAllowES5.lintText(sourceCode);
const formatter = await eslintOnlyAllowES5.loadFormatter('stylish');
const resultText = await formatter.format(lintResult);
if (resultText) {
console.log(resultText);
console.log(chalk.red(`✘ Failed to lint ${chalk.bold(file)}, skip it`));
continue;
}
targets.push([
`MediaWiki:${file}`,
`${banner.js}\n"use strict";\n\n${sourceCode.trim()}\n${footer.js}`,
]);
break;
}
case 'text/css':
targets.push([`MediaWiki:${file}`, `${banner.css}\n${sourceCode.trim()}\n${footer.css}`]);
break;
}
}
if (!targets.length) {
console.log(chalk.yellow('━ No page need to deploy directly'));
return [];
}
return targets;
};
/**
* Check the integrity of configuration items
*
* @param {Object} config To be completed configuration
* @param {Object} [object]
* @param {boolean} [object.isCheckApiUrlOnly] Only check `config[site].apiUrl` is empty or not
* @param {boolean} [object.isSkipAsk] Run the deploy process directly or not
* @return {Promise<void>}
*/
async function checkConfig(
config: {
[key: string]: Partial<CredentialsOnlyPassword>;
},
{
isCheckApiUrlOnly,
isSkipAsk,
}: {
isCheckApiUrlOnly: true;
isSkipAsk?: boolean;
}
): Promise<void>;
async function checkConfig(config: ReturnType<typeof loadConfig>): Promise<void>;
async function checkConfig(
config: ReturnType<typeof loadConfig>,
{
isCheckApiUrlOnly,
isSkipAsk,
}: {
isCheckApiUrlOnly?: boolean;
isSkipAsk?: boolean;
}
): Promise<void>;
// eslint-disable-next-line func-style
async function checkConfig(
config: ReturnType<typeof loadConfig>,
{
isCheckApiUrlOnly,
isSkipAsk,
}: {
isCheckApiUrlOnly?: boolean;
isSkipAsk?: boolean;
} = {}
): Promise<void> {
const exitWithError = (reason: string): void => {
console.log(chalk.red(`✘ ${reason} in ${chalk.italic('credentials.json')}`));
exit(1);
};
if (!Object.keys(config).length) {
exitWithError('No site found');
}
for (const site of Object.keys(config)) {
if (!site) {
exitWithError('Site Name should be a non-empty string');
}
if (site !== site.trim()) {
exitWithError('Site name should not start or end with whitespace characters');
}
}
for (const [site, credentials] of Object.entries(config)) {
if (isCheckApiUrlOnly) {
if (credentials.apiUrl || isSkipAsk) {
continue;
}
credentials.apiUrl = await prompt(`> [${site}] Enter api url (eg. https://your.wiki/api.php)`);
} else {
if (isSkipAsk) {
continue;
}
const _credentials = credentials as Partial<CredentialsOnlyPassword>;
_credentials.username ||= await prompt(`> [${site}] Enter username}`);
_credentials.password ||= await prompt(`> [${site}] Enter bot password`, 'password');
}
}
}
/**
* Load credentials.json
*
* @return {Object} The result of parsing the credentials.json file
*/
const loadConfig = () => {
const logError = (reason: string) => {
console.log(
chalk.yellow(`${chalk.italic('credentials.json')} is ${reason}, credentials must be provided manually.`)
);
};
let isMissing = false;
const {CREDENTIALS_JSON} = env;
const credentialsFilePath = path.join(__rootDir, 'scripts/credentials.json');
if (!CREDENTIALS_JSON && !existsSync(credentialsFilePath)) {
isMissing = true;
logError('missing');
}
let credentialsJsonText = CREDENTIALS_JSON || '{}';
if (!CREDENTIALS_JSON && !isMissing) {
credentialsJsonText = readFileContent(credentialsFilePath);
}
let credentials: {
[key: string]: Partial<Credentials>;
} = {};
try {
credentials = JSON.parse(credentialsJsonText) as typeof credentials;
} catch {
logError('broken');
}
const sites = Object.keys(credentials);
if (!isMissing && (!sites.length || !Object.keys(credentials[sites[0] as string] as Partial<Credentials>).length)) {
logError('empty');
}
return credentials;
};
/**
* Make editing summary
*
* @param {boolean} [isSkipAsk] Run the deploy process directly or not
* @param {Object} [object] The target file path and the fallback editing summary
* @return {Promise<string>} The editing summary
*/
async function makeEditSummary(isSkipAsk?: boolean): Promise<string>;
async function makeEditSummary(
isSkipAsk: boolean,
{
filePath,
fallbackEditSummary,
}: {
filePath?: string;
fallbackEditSummary?: string;
}
): Promise<string>;
// eslint-disable-next-line func-style
async function makeEditSummary(
isSkipAsk?: boolean,
{
filePath,
fallbackEditSummary,
}: {
filePath?: string;
fallbackEditSummary?: string;
} = {}
): Promise<string> {
const execLog = async (currentPath: string) => {
try {
const {stdout: _log} = await exec(`git log --pretty=format:"%H %s" -1 -- ${currentPath}`);
const log = _log.trim();
if (!log) {
return '';
}
const logSplit = log.split(' ');
const {stdout: _sha} = await exec(`git rev-parse --short ${logSplit.shift()}`);
return `Git commit ${_sha.trim()}: ${logSplit.join(' ')}`;
} catch {
return '';
}
};
const getLog = async (currentPath: string) => {
if (!existsSync(currentPath)) {
return '';
}
const log: string = await execLog(currentPath);
if (!log) {
return '';
}
return log;
};
if (filePath && fallbackEditSummary !== undefined) {
if (!/^Git\scommit\s\S+?:\s/.test(fallbackEditSummary)) {
return fallbackEditSummary;
}
const log: string = await getLog(filePath);
if (!log) {
return fallbackEditSummary;
}
return log;
}
let sha = '';
let summary = '';
try {
const {stdout: _sha} = await exec('git rev-parse --short HEAD');
sha = _sha.trim();
const {stdout: _summary} = await exec('git log --pretty=format:"%s" HEAD -1');
summary = _summary.trim();
} catch {}
const customSummary = isSkipAsk ? '' : await prompt('> Custom editing summary message (optional):');
const customSummaryTrimmed = trim(customSummary, {
addNewline: false,
});
// If trimmed input is an empty string, use the message from the last git commit
const editSummary = customSummaryTrimmed || `${sha ? `Git commit ${sha}: ` : ''}${summary}`;
console.log(
chalk.white(`${customSummaryTrimmed ? 'Editing' : 'Fallback editing'} summary is: ${chalk.bold(editSummary)}`)
);
if (!isSkipAsk) {
await prompt('> Confirm to continue deployment?', 'confirm', true);
}
console.log(chalk.yellow('--- deployment will continue in three seconds ---'));
await setTimeout(3 * 1000);
return editSummary;
}
/**
* Convert language variants of a page
*
* @param {string} pageTitle The title of this page
* @param {string} content The content of this page
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by the api instance
*/
const convertVariant = (pageTitle: string, content: string, api: Api, editSummary: string) => {
const {apiInstance, site} = api;
/**
* @see {@link https://zh.wikipedia.org/wiki/User:Xiplus/js/TranslateVariants}
* @license CC-BY-SA-4.0
*/
content = content
.replace(/[\s#&*_[\]{}|:'<>]/gim, (substring) => {
return `&#${substring.codePointAt(0)};`;
})
.replace(/([[)((?:(?!|)(?!]).)+?)(|(?:(?!]).)+?]])/g, '$1-{$2}-$3')
.replace(/-{(.+?)}-/g, (substring) => {
return substring
.replace('-{', '-{')
.replace('}-', '}-')
.replace(/|/g, '|')
.replace(/ /g, ' ')
.replace(/:/g, ':')
.replace(/=/g, '=')
.replace(/>/g, '>');
});
const convert = async (variant: string) => {
const parsedHtml = await apiInstance.parseWikitext(
`{{NoteTA|G1=IT|G2=MediaWiki}}<div class="convertVariant">${content}</div>`,
{
action: 'parse',
prop: ['text'],
uselang: variant,
}
);
const {document} = new Window({
url: apiInstance.options.apiUrl as string,
});
document.body.innerHTML = `<div>${parsedHtml}</div>`;
const convertedDescription = document.querySelector('.convertVariant')!.textContent.replace(/-{}-/g, '');
const convertPageTitle = `${pageTitle}/${variant}`;
deployPages[site] ??= [];
deployPages[site].push(convertPageTitle);
const {nochange} = await apiInstance.save(convertPageTitle, convertedDescription, editSummary);
return Boolean(nochange);
};
apiQueue
.addAll(
VARIANTS.map((variant) => {
return async () => {
return await convert(variant);
};
})
)
.then((noChanges) => {
const isNoChange = noChanges.every(Boolean);
if (isNoChange) {
console.log(chalk.yellow(`━ No change converting ${chalk.bold(pageTitle)}`));
} else {
console.log(chalk.green(`✔ Successfully converted ${chalk.bold(pageTitle)}`));
}
})
.catch((error) => {
console.log(chalk.red(`✘ Failed to convert ${chalk.bold(pageTitle)}`));
console.error(error);
});
};
/**
* Save gadget definition
*
* @param {string} definitionText The MediaWiki:Gadgets-definition content
* @param {string[]} enabledGadgets The enabled gadgets
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by this api instance
* @return {string} The `MediaWiki:Gadgets-definition` content of the current site
*/
const saveDefinition = (definitionText: string, enabledGadgets: string[], api: Api, editSummary: string) => {
const {apiInstance, site} = api;
const pageTitle = 'MediaWiki:Gadgets-definition';
deployPages[site] ??= [];
deployPages[site].push(pageTitle);
definitionText = definitionText
.split('\n')
.filter((lineContent) => {
const regex = /^\*\s(\S+?)\[ResourceLoader[|\]]/;
const regExpMatchArray = lineContent.match(regex);
if (regExpMatchArray && regExpMatchArray[1]) {
return enabledGadgets.includes(regExpMatchArray[1]);
}
return true;
})
.join('\n');
const removeEmptySection = (string: string) => {
const firstSectionIndex = string.search(/[=]=[\S\s]+?==/);
if (firstSectionIndex === -1) {
return string;
}
const keepString = string.slice(0, Math.max(0, firstSectionIndex));
string = string.slice(Math.max(0, firstSectionIndex));
string = string.replace(/\n{3,}/g, '\n\n');
const blocks = string.split(/([=]=[\S\s]+?==\n)/);
const newBlocks = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i] as string;
if (block.startsWith('==') && blocks[i + 1] && !(blocks[i + 1] as string).startsWith('*')) {
i++;
continue;
}
newBlocks.push(block);
}
return keepString + newBlocks.join('');
};
definitionText = removeEmptySection(definitionText);
definitionText = trim(definitionText);
void apiQueue.add(async () => {
try {
const {nochange} = await apiInstance.save(pageTitle, definitionText, editSummary);
if (nochange) {
console.log(chalk.yellow(`━ No change saving ${chalk.bold('gadget definitions')}`));
} else {
console.log(chalk.green(`✔ Successfully saved ${chalk.bold('gadget definitions')}`));
}
} catch (error) {
console.log(chalk.red(`✘ Failed to save ${chalk.bold('gadget definitions')}`));
console.error(error);
}
});
return definitionText;
};
/**
* Save gadget definition section pages
*
* @param {string} definitionText The MediaWiki:Gadgets-definition content
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by this api instance
*/
const saveDefinitionSectionPage = (definitionText: string, api: Api, editSummary: string) => {
const {apiInstance, site} = api;
const sections = (definitionText.match(/^==([\S\s]+?)==$/gm) as RegExpMatchArray).map<string>((sectionHeader) => {
return sectionHeader.replace(/[=]=/g, '').trim();
});
const pageTitles = sections.map<string>((section) => {
return `MediaWiki:Gadget-section-${section}`;
});
for (const [index, section] of sections.entries()) {
const sectionText = DEFINITION_SECTION_MAP[section as keyof typeof DEFINITION_SECTION_MAP] || section;
const pageTitle = pageTitles[index] as string;
deployPages[site] ??= [];
deployPages[site].push(pageTitle);
void apiQueue.add(async () => {
try {
const {nochange} = await apiInstance.save(pageTitle, sectionText, editSummary);
if (nochange) {
console.log(chalk.yellow(`━ No change saving ${chalk.bold(pageTitle)}`));
} else {
console.log(chalk.green(`✔ Successfully saved ${chalk.bold(pageTitle)}`));
}
} catch (error) {
console.log(chalk.red(`✘ Failed to save ${chalk.bold(pageTitle)}`));
console.error(error);
}
});
if (CONVERT_VARIANT) {
convertVariant(pageTitle, sectionText, api, editSummary);
}
}
};
/**
* Save gadget description
*
* @param {string} gadgetName The gadget name
* @param {string} description The definition of this gadget
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by the api instance
*/
const saveDescription = (gadgetName: string, description: string, api: Api, editSummary: string) => {
const {apiInstance, site} = api;
const pageTitle = `MediaWiki:Gadget-${gadgetName}`;
deployPages[site] ??= [];
deployPages[site].push(pageTitle);
void apiQueue.add(async () => {
try {
const {nochange} = await apiInstance.save(pageTitle, description, editSummary);
if (nochange) {
console.log(chalk.yellow(`━ No change saving ${chalk.bold(`${gadgetName} description`)}`));
} else {
console.log(chalk.green(`✔ Successfully saved ${chalk.bold(`${gadgetName} description`)}`));
}
} catch (error) {
console.log(chalk.red(`✘ Failed to save ${chalk.bold(`${gadgetName} description`)}`));
console.error(error);
}
});
if (CONVERT_VARIANT) {
convertVariant(pageTitle, description, api, editSummary);
}
};
/**
* Save gadget file
*
* @param {string} gadgetName The gadget name
* @param {string} fileName The target file name
* @param {string} fileContent The target file content
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by the api instance
*/
const saveFiles = (gadgetName: string, fileName: string, fileContent: string, api: Api, editSummary: string) => {
const {apiInstance, site} = api;
const pageTitle = `MediaWiki:Gadget-${fileName}`;
deployPages[site] ??= [];
deployPages[site].push(pageTitle);
void apiQueue.add(async () => {
try {
const {nochange} = await apiInstance.save(pageTitle, fileContent, editSummary);
if (nochange) {
console.log(
chalk.yellow(`━ No change saving ${chalk.underline(fileName)} to ${chalk.bold(gadgetName)}`)
);
} else {
console.log(
chalk.green(`✔ Successfully saved ${chalk.underline(fileName)} to ${chalk.bold(gadgetName)}`)
);
}
} catch (error) {
console.log(chalk.red(`✘ Failed to save ${chalk.underline(fileName)} to ${chalk.bold(gadgetName)}`));
console.error(error);
}
});
};
/**
* Save page directly
*
* @param {string} pageTitle The target page title
* @param {string} pageContent The target page content
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by the api instance
*/
const savePages = (pageTitle: string, pageContent: string, api: Api, editSummary: string) => {
const {apiInstance, site} = api;
deployPages[site] ??= [];
deployPages[site].push(pageTitle);
void apiQueue.add(async () => {
try {
const {nochange} = await apiInstance.save(pageTitle, pageContent, editSummary);
if (nochange) {
console.log(chalk.yellow(`━ No change saving ${chalk.underline(pageTitle)}`));
} else {
console.log(chalk.green(`✔ Successfully saved ${chalk.underline(pageTitle)}`));
}
} catch (error) {
console.log(chalk.red(`✘ Failed to save ${chalk.underline(pageTitle)}`));
console.error(error);
}
});
};
/**
* Delete unused pages from the target MediaWiki site
*
* @param {Api} api The api instance and the site name
* @param {string} editSummary The editing summary used by this api instance
* @param {boolean} [isSkipAsk] Run the deploy process directly or not
*/
const deleteUnusedPages = async (api: Api, editSummary: string, isSkipAsk?: boolean) => {
const {apiInstance, site} = api;
const storeFilePath = path.join(__rootDir, 'dist/store.txt');
let lastDeployPages: typeof deployPages = {};
try {
const fileContent = readFileContent(storeFilePath);
lastDeployPages = JSON.parse(fileContent) as typeof deployPages;
} catch {}
const fullDeployPages = sortObject(deployPages, true);
for (const [siteName, pages] of Object.entries(lastDeployPages)) {
if (fullDeployPages[siteName]) {
continue;
}
fullDeployPages[siteName] = pages.toSorted(
alphaSort({
caseInsensitive: true,
natural: true,
})
);
}
open(storeFilePath, 'w', (err, fd) => {
if (err) {
console.error(err);
return;
}
const fullDeployPagesSorted = sortObject(fullDeployPages);
writeFileContent(fd, `${JSON.stringify(fullDeployPagesSorted, null, '\t')}\n`);
});
const currentSiteLastDeployPages = lastDeployPages[site] ?? [];
if (!currentSiteLastDeployPages.length) {
console.log(chalk.yellow('━ No deployment log found'));
return;
}
const currentSiteDeployPages = fullDeployPages[site] ?? [];
const needToDeletePages = currentSiteLastDeployPages
.toSorted(
alphaSort({
caseInsensitive: true,
natural: true,
})
)
.filter((page) => {
return !currentSiteDeployPages.includes(page);
});
if (!needToDeletePages.length) {
console.log(chalk.yellow('━ No page need to delete'));
return;
}
stdout.write(`The following pages will be deleted:\n${needToDeletePages.join('\n')}\n`);
if (!isSkipAsk) {
await prompt('> Confirm to continue deleting?', 'confirm', true);
}
console.log(chalk.yellow(`--- [${chalk.bold(site)}] deleting will continue in three seconds ---`));
await setTimeout(3 * 1000);
const deletePage = async (pageTitle: string) => {
try {
await apiInstance.delete(pageTitle, editSummary);
console.log(chalk.green(`✔ Successfully deleted ${chalk.bold(pageTitle)}`));
} catch (error) {
if (error instanceof MwnError && error.code === 'missingtitle') {
console.log(chalk.yellow(`━ Page ${chalk.bold(pageTitle)} no need to delete`));
} else {
console.log(chalk.red(`✘ Failed to delete ${chalk.bold(pageTitle)}`));
console.error(error);
}
}
};
void apiQueue.addAll(
needToDeletePages.map((page) => {
return async (): Promise<void> => {
await deletePage(page);
};
})
);
};
export {
readDefinition,
generateTargets,
generateDirectTargets,
checkConfig,
loadConfig,
makeEditSummary,
saveDefinition,
saveDefinitionSectionPage,
saveDescription,
saveFiles,
savePages,
deleteUnusedPages,
};