mwn
Version:
JavaScript & TypeScript MediaWiki bot framework for Node.js
1,359 lines (1,358 loc) • 58.9 kB
JavaScript
"use strict";
/**
*
* mwn: a MediaWiki bot framework for Node.js
*
* Copyright (C) 2020 Siddharth VP
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.Mwn = void 0;
/**
* Attributions:
* Parts of the code are adapted from MWBot <https://github.com/Fannon/mwbot/src/index.js>
* released under the MIT license. Copyright (c) 2015-2018 Simon Heimler.
*
* Some parts are copied from the mediawiki.api module in mediawiki core
* <https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/core/+/master/resources/src/mediawiki.api>
* released under GNU GPL v2.
*
*/
// Node internal modules
const fs = require("node:fs");
const path = require("node:path");
const crypto = require("node:crypto");
const http = require("node:http");
const https = require("node:https");
// NPM modules
const axios_1 = require("axios");
const tough = require("tough-cookie");
const OAuth = require("oauth-1.0a");
// Nested classes of mwn
const date_1 = require("./date");
const title_1 = require("./title");
const page_1 = require("./page");
const wikitext_1 = require("./wikitext");
const user_1 = require("./user");
const category_1 = require("./category");
const file_1 = require("./file");
const core_1 = require("./core");
const log_1 = require("./log");
const error_1 = require("./error");
const static_utils_1 = require("./static_utils");
const utils_1 = require("./utils");
class Mwn {
/**
* Constructs a new bot instance. Recommended usage is one bot instance for every wiki and user.
* A bot instance has its own state (e.g. tokens) that is necessary for some operations.
*
* @param [customOptions] - Custom options
*/
constructor(customOptions) {
/**
* Bot instance Login State
* Is received from the MW Login API and contains token, userid, etc.
*/
this.state = {};
/**
* Bot instance is logged in or not
*/
this.loggedIn = false;
/**
* Bot instance's edit token. Initially set as an invalid token string
* so that the badtoken handling logic is invoked if the token is
* not set before a query is sent.
* @type {string}
*/
this.csrfToken = '%notoken%';
/**
* Default options.
* Should be immutable
*/
this.defaultOptions = {
silent: false,
apiUrl: null,
userAgent: 'mwn',
username: null,
password: null,
OAuthCredentials: {
consumerToken: null,
consumerSecret: null,
accessToken: null,
accessSecret: null,
},
OAuth2AccessToken: null,
maxRetries: 3,
retryPause: 5000,
shutoff: {
intervalDuration: 10000,
page: null,
condition: /^\s*$/,
onShutoff() { },
},
defaultParams: {
format: 'json',
formatversion: '2',
maxlag: 5,
},
suppressAPIWarnings: false,
editConfig: {
conflictRetries: 2,
suppressNochangeWarning: false,
exclusionRegex: null,
},
suppressInvalidDateWarning: false,
};
/**
* Cookie jar for the bot instance - holds session and login cookies
* @type {tough.CookieJar}
*/
this.cookieJar = new tough.CookieJar();
/** Axios instance for the bot instance. */
this.axiosInstance = axios_1.default.create();
/**
* Request options for the axios library.
* Change the defaults using setRequestOptions()
* @type {Object}
*/
this.requestOptions = (0, utils_1.mergeDeep1)({
responseType: 'json',
}, Mwn.requestDefaults);
/**
* Emergency shutoff config
* @type {{hook: NodeJS.Timeout, state: boolean}}
*/
this.shutoff = {
state: false,
hook: null,
};
this.hasApiHighLimit = false;
/**
* Title class associated with the bot instance.
* See {@link MwnTitle} interface for methods on title objects.
*/
this.Title = (0, title_1.default)();
/**
* Page class associated with the bot instance.
* See {@link MwnPage} interface for methods on page objects.
*/
this.Page = (0, page_1.default)(this);
/**
* Category class associated with the bot instance.
* See {@link MwnCategory} interface for methods on category objects.
*/
this.Category = (0, category_1.default)(this);
/**
* File class associated with the bot instance.
* See {@link MwnFile} interface for methods on file objects.
*/
this.File = (0, file_1.default)(this);
/**
* User class associated with the bot instance.
* See {@link MwnUser} interface for methods on user objects.
*/
this.User = (0, user_1.default)(this);
/**
* Wikitext class associated with the bot instance.
* See {@link MwnWikitext} interface for methods on wikitext objects.
*/
this.Wikitext = (0, wikitext_1.default)(this);
/**
* Date class associated with the bot instance.
* See {@link MwnDate} interface for methods on date objects.
*/
this.Date = (0, date_1.default)(this);
/************** CORE FUNCTIONS *******************/
this.loginInProgress = null;
/**
* Promisified version of setTimeout
* @param {number} duration - of sleep in milliseconds
*/
this.sleep = utils_1.sleep;
if (process.versions.node) {
let majorVersion = parseInt(process.versions.node);
if (majorVersion < 14) {
(0, log_1.log)(`[W] Detected node version v${process.versions.node}, but mwn is supported only on node v14.x and above`);
}
}
if (typeof customOptions === 'string') {
// Read options from file (JSON):
try {
customOptions = JSON.parse(fs.readFileSync(customOptions).toString());
}
catch (err) {
throw new Error(`Failed to read or parse JSON config file: ` + err);
}
}
this.options = (0, utils_1.mergeDeep1)(this.defaultOptions, customOptions);
// Wire up axios to use the cookie jar
this.axiosInstance.interceptors.request.use(async (config) => {
const cookieHeader = await this.cookieJar.getCookieString(config.url);
if (cookieHeader) {
config.headers.Cookie = cookieHeader;
}
return config;
});
this.axiosInstance.interceptors.response.use(async (response) => {
const setCookieHeaders = response.headers['set-cookie'];
if (setCookieHeaders) {
for (const cookie of setCookieHeaders) {
await this.cookieJar.setCookie(cookie, response.config.url);
}
}
return response;
});
}
/**
* Initialize a bot object. Login to the wiki and fetch editing tokens. If OAuth
* credentials are provided, they will be used over BotPassword credentials.
* Also fetches the site data needed for parsing and constructing title objects.
* @param {Object} config - Bot configurations, including apiUrl, and either the
* username and password or the OAuth credentials
* @returns {Promise<Mwn>} bot object
*/
static async init(config) {
const bot = new Mwn(config);
if (bot.options.OAuth2AccessToken || bot._usingOAuth()) {
bot.initOAuth();
await bot.getTokensAndSiteInfo();
}
else {
await bot.login();
}
return bot;
}
/**
* Set and overwrite mwn options
* @param {Object} customOptions
*/
setOptions(customOptions) {
this.options = (0, utils_1.mergeDeep1)(this.options, customOptions);
}
/**
* Sets the API URL for MediaWiki requests
* This can be uses instead of a login, if no actions are used that require login.
* @param {string} apiUrl - API url to MediaWiki, e.g. https://en.wikipedia.org/w/api.php
*/
setApiUrl(apiUrl) {
this.options.apiUrl = apiUrl;
}
/**
* Sets and overwrites the raw request options, used by the axios library
* See https://www.npmjs.com/package/axios
*/
setRequestOptions(customRequestOptions) {
(0, utils_1.mergeDeep1)(this.requestOptions, customRequestOptions);
}
/**
* Set the default parameters to be sent in API calls.
* @param {Object} params - default parameters
*/
setDefaultParams(params) {
this.options.defaultParams = (0, utils_1.merge)(this.options.defaultParams, params);
}
/**
* Set your API user agent. See https://meta.wikimedia.org/wiki/User-Agent_policy
* Required for WMF wikis.
* @param {string} userAgent
*/
setUserAgent(userAgent) {
this.options.userAgent = userAgent;
}
/**
* @private
* Determine if we're going to use OAuth for authentication
*/
_usingOAuth() {
const creds = this.options.OAuthCredentials;
if (typeof creds !== 'object') {
return false;
}
if (!creds.consumerToken || !creds.consumerSecret || !creds.accessToken || !creds.accessSecret) {
return false;
}
return true;
}
/**
* Initialize OAuth instance
*/
initOAuth() {
if (this.options.OAuth2AccessToken) {
this.usingOAuth2 = true;
return;
}
if (!this._usingOAuth()) {
// without this, the API would return a confusing
// mwoauth-invalid-authorization invalid consumer error
throw new Error('[mwn] Invalid OAuth config');
}
try {
this.oauth = new OAuth({
consumer: {
key: this.options.OAuthCredentials.consumerToken,
secret: this.options.OAuthCredentials.consumerSecret,
},
signature_method: 'HMAC-SHA1',
// based on example at https://www.npmjs.com/package/oauth-1.0a
hash_function(base_string, key) {
return crypto.createHmac('sha1', key).update(base_string).digest('base64');
},
});
this.usingOAuth = true;
}
catch (err) {
throw new Error('Failed to construct OAuth object. ' + err);
}
}
/************ CORE REQUESTS ***************/
/**
* Executes a raw request
* Uses the axios library
* @param {Object} requestOptions
* @returns {Promise}
*/
async rawRequest(requestOptions) {
if (!requestOptions.url) {
return (0, error_1.rejectWithError)({
code: 'mwn_nourl',
info: 'No URL provided for API request!',
disableRetry: true,
request: requestOptions,
});
}
const config = (0, utils_1.mergeDeep1)({}, Mwn.requestDefaults, {
method: 'get',
headers: {
'User-Agent': this.options.userAgent,
},
}, requestOptions);
return this.axiosInstance(config);
}
/**
* Executes a request with the ability to use custom parameters and custom
* request options
* @param {Object} params
* @param {Object} [customRequestOptions={}]
* @returns {Promise}
*/
async request(params, customRequestOptions = {}) {
if (this.shutoff.state) {
return (0, error_1.rejectWithError)({
code: 'bot-shutoff',
info: `Bot was shut off (check ${this.options.shutoff.page})`,
});
}
const req = new core_1.Request(this, params, customRequestOptions);
await req.process();
return this.rawRequest(req.requestParams).then((fullResponse) => new core_1.Response(this, req.apiParams, req.requestParams).process(fullResponse), (error) => new core_1.Response(this, req.apiParams, req.requestParams).handleRequestFailure(error));
}
async query(params, customRequestOptions = {}) {
return this.request(Object.assign({ action: 'query' }, params), customRequestOptions);
}
/**
* Executes a Login
* @see https://www.mediawiki.org/wiki/API:Login
* @returns {Promise}
*/
async login(loginOptions) {
Object.assign(this.options, loginOptions);
// Avoid multiple logins taking place concurrently, for instance when session loss occurs
// in the middle of a batch operation.
if (!this.loginInProgress) {
const loginPromise = this.loginInternal();
this.loginInProgress = [this.options.username, loginPromise];
loginPromise.finally(() => {
this.loginInProgress = null;
});
// Multiple logins with different usernames? Error out.
}
else if (this.loginInProgress[0] !== this.options.username) {
return (0, error_1.rejectWithError)({
code: 'mwn_invalidlogin',
info: 'Detected concurrent login with a different username',
});
}
// Return the response of the previous login call
return this.loginInProgress[1];
}
async loginInternal() {
if (!this.options.username || !this.options.password || !this.options.apiUrl) {
return (0, error_1.rejectWithError)({
code: 'mwn_nologincredentials',
info: 'Incomplete login credentials!',
});
}
let loginString = this.options.username + '@' + this.options.apiUrl;
// Step 1: Fetch login token
const loginTokenResponse = await this.request({
action: 'query',
meta: 'tokens',
type: 'login',
// Unset the assert parameter (in case it's given by the user as a default
// option), as it will invariably fail until login is performed.
assert: undefined,
// Also unset the maxlag parameter, not required for logins.
// This also avoids infinite recursion when assert and maxlag are both provided and replicas are lagged.
maxlag: undefined,
});
if (!loginTokenResponse?.query?.tokens?.logintoken) {
(0, log_1.log)('[E] [mwn] Login failed with invalid response: ' + loginString);
return (0, error_1.rejectWithError)({
code: 'mwn_notoken',
info: 'Failed to get login token',
response: loginTokenResponse,
});
}
Object.assign(this.state, loginTokenResponse.query.tokens);
// Step 2: Post login request
const loginResponse = await this.request({
action: 'login',
lgname: this.options.username,
lgpassword: this.options.password,
lgtoken: loginTokenResponse.query.tokens.logintoken,
assert: undefined, // as above, assert won't work till the user is logged in
maxlag: undefined, // as above, to avoid infinite recursions in error handling
});
let reason;
let data = loginResponse.login;
if (data) {
if (data.result === 'Success') {
Object.assign(this.state, data);
this.loggedIn = true;
if (!this.options.silent) {
(0, log_1.log)('[S] [mwn] Login successful: ' + loginString);
}
// Step 3: fetch tokens for editing, and info about namespaces for MwnTitle
await this.getTokensAndSiteInfo().catch((err) => {
(0, log_1.log)(`[W] Failed fetching tokens and siteinfo: ${err}`);
});
return data;
}
else if (data.result === 'Aborted') {
if (data.reason === 'Cannot log in when using MediaWiki\\Session\\BotPasswordSessionProvider sessions.') {
reason = `Already logged in as ${this.options.username}, logout first to re-login`;
}
else if (data.reason === 'Cannot log in when using MediaWiki\\Extension\\OAuth\\SessionProvider sessions.') {
reason = `Cannot use login/logout while using OAuth`;
}
else if (data.reason) {
reason = data.result + ': ' + data.reason;
}
}
else if (data.result && data.reason) {
reason = data.result + ': ' + data.reason;
}
}
return (0, error_1.rejectWithError)({
code: 'mwn_failedlogin',
info: reason || 'Login failed',
response: loginResponse,
});
}
/**
* Log out of the account. Flushes the cookie jar and clears the saved tokens.
* Should not be used if authenticating via OAuth.
* @returns {Promise<void>}
*/
async logout() {
if (this.usingOAuth) {
throw new Error("Can't use logout() while using OAuth");
}
return this.request({
action: 'logout',
token: this.csrfToken,
}).then(() => {
// returns an empty response ({}) if successful
this.loggedIn = false;
this.state = {};
this.csrfToken = '%notoken%';
return this.cookieJar.removeAllCookies();
});
}
/**
* Create an account. Only works on wikis without extensions like
* ConfirmEdit enabled (hence doesn't work on WMF wikis).
* @param username
* @param password
*/
async createAccount(username, password) {
if (!this.state.createaccounttoken) {
// not logged in
await this.getTokens();
}
return this.request({
action: 'createaccount',
createreturnurl: 'https://example.com',
createtoken: this.state.createaccounttoken,
username: username,
password: password,
retype: password,
}).then((json) => {
let data = json.createaccount;
if (data.status === 'FAIL') {
return (0, error_1.rejectWithError)({
code: data.messagecode,
info: data.message,
...data,
});
}
else {
// status === 'PASS' or other value
return data;
}
});
}
/**
* Get basic info about the logged-in user
* @param [options]
* @returns {Promise}
*/
async userinfo(options = {}) {
return this.request({
action: 'query',
meta: 'userinfo',
...options,
}).then((response) => response.query.userinfo);
}
/**
* Gets namespace-related information for use in title nested class.
* This need not be used if login() is being used. This is for cases
* where mwn needs to be used without logging in.
* @returns {Promise<void>}
*/
async getSiteInfo() {
return this.request({
action: 'query',
meta: 'siteinfo',
siprop: 'general|namespaces|namespacealiases',
}).then((result) => {
this.Title.processNamespaceData(result);
});
}
/**
* Get tokens and saves them in this.state
* @returns {Promise<void>}
*/
async getTokens() {
return this.request({
action: 'query',
meta: 'tokens',
type: 'csrf|createaccount|login|patrol|rollback|userrights|watch',
}).then((response) => {
if (response.query && response.query.tokens) {
this.csrfToken = response.query.tokens.csrftoken;
this.state = (0, utils_1.merge)(this.state, response.query.tokens);
}
else {
return (0, error_1.rejectWithError)({
code: 'mwn_notoken',
info: 'Could not get token',
response,
});
}
});
}
/**
* Gets an edit token (also used for most other actions
* such as moving and deleting)
* This is only compatible with MW >= 1.24
* @returns {Promise<string>}
*/
async getCsrfToken() {
return this.getTokens().then(() => this.csrfToken);
}
/**
* Get tokens and siteinfo (using a single API request) and save them in the bot state.
* @returns {Promise<void>}
*/
async getTokensAndSiteInfo() {
return this.request({
action: 'query',
meta: 'tokens|siteinfo|userinfo',
type: 'csrf|createaccount|login|patrol|rollback|userrights|watch',
siprop: 'general|namespaces|namespacealiases',
uiprop: 'rights',
maxlag: undefined,
}).then((response) => {
this.Title.processNamespaceData(response);
if (response.query.userinfo.rights.includes('apihighlimits')) {
this.hasApiHighLimit = true;
}
if (response.query && response.query.tokens) {
this.csrfToken = response.query.tokens.csrftoken;
this.state = (0, utils_1.merge)(this.state, response.query.tokens);
}
else {
return (0, error_1.rejectWithError)({
code: 'mwn_notoken',
info: 'Could not get token',
response,
});
}
});
}
/**
* Get type of token to be used with an API action
* @param {string} action - API action parameter
* @returns {Promise<string>}
*/
async getTokenType(action) {
return this.request({
action: 'paraminfo',
modules: action,
}).then((response) => {
return response.paraminfo.modules[0].parameters.find((p) => p.name === 'token')
.tokentype;
});
}
/**
* Get the wiki's server time
* @returns {Promise<string>}
*/
async getServerTime() {
return this.request({
action: 'query',
curtimestamp: true,
}).then((data) => {
return data.curtimestamp;
});
}
/**
* Fetch and parse a JSON wikipage
* @param {string} title - page title
* @returns {Promise<Object>} parsed JSON object
*/
async parseJsonPage(title) {
return this.read(title).then((data) => {
try {
return JSON.parse(data.revisions[0].content);
}
catch {
return (0, error_1.rejectWithErrorCode)('invalidjson');
}
});
}
/**
* Fetch MediaWiki messages
* @param messages
* @param options
*/
async getMessages(messages, options = {}) {
return this.request({
action: 'query',
meta: 'allmessages',
ammessages: messages,
...options,
}).then((data) => {
let result = {};
data.query.allmessages.forEach((obj) => {
if (!obj.missing) {
result[obj.name] = obj.content;
}
});
return result;
});
}
/**
* Enable bot emergency shutoff
*/
enableEmergencyShutoff(shutoffOptions) {
Object.assign(this.options.shutoff, shutoffOptions);
this.shutoff.hook = setInterval(async () => {
let text = await new this.Page(this.options.shutoff.page).text();
let cond = this.options.shutoff.condition;
if ((cond instanceof RegExp && !cond.test(text)) || (cond instanceof Function && !cond(text))) {
this.shutoff.state = true;
this.disableEmergencyShutoff();
// user callback executed last, so that an error thrown by
// it doesn't prevent the above from being run
this.options.shutoff.onShutoff(text);
}
}, this.options.shutoff.intervalDuration);
}
/**
* Disable emergency shutoff detection.
* Use this only if it was ever enabled.
*/
disableEmergencyShutoff() {
clearInterval(this.shutoff.hook);
}
read(titles, options) {
let pages = Array.isArray(titles) ? titles : [titles];
let batchFieldName = typeof pages[0] === 'number' ? 'pageids' : 'titles';
return this.massQuery({
action: 'query',
...(0, utils_1.makeTitles)(titles),
prop: 'revisions',
rvprop: 'content|timestamp',
rvslots: 'main',
redirects: true,
...options,
}, batchFieldName).then((jsons) => {
let data = jsons.reduce((data, json) => {
json.query.pages.forEach((pg) => {
if (pg.revisions) {
pg.revisions.forEach((rev) => {
Object.assign(rev, rev.slots.main);
});
}
});
return data.concat(json.query.pages);
}, []);
return data.length === 1 ? data[0] : data;
});
}
async *readGen(titles, options, batchSize) {
let massQueryResponses = this.massQueryGen({
action: 'query',
...(0, utils_1.makeTitles)(titles),
prop: 'revisions',
rvprop: 'content|timestamp',
rvslots: 'main',
redirects: true,
...options,
}, typeof titles[0] === 'number' ? 'pageids' : 'titles', batchSize);
for await (let response of massQueryResponses) {
if (response && response.query && response.query.pages) {
for (let pg of response.query.pages) {
if (pg.revisions) {
pg.revisions.forEach((rev) => {
Object.assign(rev, rev.slots.main);
});
}
yield pg;
}
}
}
}
// adapted from mw.Api().edit
/**
* @param {string|number|MwnTitle} title - Page title or page ID or MwnTitle object
* @param {Function} transform - Callback that prepares the edit. It takes one
* argument that is an { content: 'string: page content', timestamp: 'string:
* time of last edit' } object. This function should return an object with
* edit API parameters or just the updated text, or a promise providing one of
* those.
* @param {Object} [editConfig] - Overridden edit options. Available options:
* conflictRetries, suppressNochangeWarning, exclusionRegex
* @return {Promise<Object>} Edit API response
*/
async edit(title, transform, editConfig) {
editConfig = editConfig || this.options.editConfig;
// TODO: use baserevid instead of basetimestamp for conflict handling
let basetimestamp, curtimestamp;
return this.request({
action: 'query',
...(0, utils_1.makeTitles)(title),
prop: 'revisions',
rvprop: ['content', 'timestamp'],
rvslots: 'main',
formatversion: '2',
curtimestamp: true,
})
.then((data) => {
let page, revision, revisionContent;
if (!data.query || !data.query.pages) {
return (0, error_1.rejectWithErrorCode)('unknown');
}
page = data.query.pages[0];
if (!page || page.invalid) {
return (0, error_1.rejectWithErrorCode)('invalidtitle');
}
if (page.missing) {
return Promise.reject(new error_1.MwnMissingPageError());
}
revision = page.revisions[0];
try {
revisionContent = revision.slots.main.content;
}
catch {
return (0, error_1.rejectWithErrorCode)('unknown');
}
basetimestamp = revision.timestamp;
curtimestamp = data.curtimestamp;
if (editConfig.exclusionRegex && editConfig.exclusionRegex.test(revisionContent)) {
return (0, error_1.rejectWithErrorCode)('bot-denied');
}
return transform({
timestamp: revision.timestamp,
content: revisionContent,
});
})
.then((returnVal) => {
if (typeof returnVal !== 'string' && !returnVal) {
return { edit: { result: 'aborted' } };
}
const editParams = typeof returnVal === 'object'
? returnVal
: {
text: String(returnVal),
};
return this.request({
action: 'edit',
...(0, utils_1.makeTitle)(title),
formatversion: '2',
basetimestamp: basetimestamp,
starttimestamp: curtimestamp,
nocreate: true,
bot: true,
token: this.csrfToken,
...editParams,
});
})
.then((data) => {
if (data.edit && data.edit.nochange && !editConfig.suppressNochangeWarning) {
(0, log_1.log)(`[W] No change from edit to ${data.edit.title}`);
}
return data.edit;
}, (err) => {
if (err.code === 'editconflict' && editConfig.conflictRetries > 0) {
editConfig.conflictRetries--;
return this.edit(title, transform, editConfig);
}
else {
return (0, error_1.rejectWithError)(err);
}
});
}
/**
* Edit a page without loading it first. Straightforward version of `edit`.
* No edit conflict detection.
*
* @param {string|number} title - title or pageid (as number)
* @param {string} content
* @param {string} [summary]
* @param {object} [options]
* @returns {Promise}
*/
async save(title, content, summary, options) {
return this.request({
action: 'edit',
...(0, utils_1.makeTitle)(title),
text: content,
summary: summary,
bot: true,
token: this.csrfToken,
...options,
}).then((data) => data.edit);
}
/**
* Creates a new pages. Does not edit existing ones
*
* @param {string} title
* @param {string} content
* @param {string} [summary]
* @param {object} [options]
*
* @returns {Promise}
*/
async create(title, content, summary, options) {
return this.request({
action: 'edit',
title: String(title),
text: content,
summary: summary,
createonly: true,
bot: true,
token: this.csrfToken,
...options,
}).then((data) => data.edit);
}
/**
* Post a new section to the page.
*
* @param {string|number} title - title or pageid (as number)
* @param {string} header
* @param {string} message wikitext message
* @param {Object} [additionalParams] Additional API parameters, e.g. `{ redirect: true }`
*/
async newSection(title, header, message, additionalParams) {
return this.request({
action: 'edit',
...(0, utils_1.makeTitle)(title),
section: 'new',
summary: header,
text: message,
bot: true,
token: this.csrfToken,
...additionalParams,
}).then((data) => data.edit);
}
/**
* Deletes a page
*
* @param {string|number} title - title or pageid (as number)
* @param {string} [summary]
* @param {object} [options]
* @returns {Promise}
*/
async delete(title, summary, options) {
return this.request({
action: 'delete',
...(0, utils_1.makeTitle)(title),
reason: summary,
token: this.csrfToken,
...options,
}).then((data) => data.delete);
}
/**
* Undeletes a page.
* Note: all deleted revisions of the page will be restored.
*
* @param {string} title
* @param {string} [summary]
* @param {object} [options]
* @returns {Promise}
*/
async undelete(title, summary, options) {
return this.request({
action: 'undelete',
title: String(title),
reason: summary,
token: this.csrfToken,
...options,
}).then((data) => data.undelete);
}
/**
* Moves a new page
*
* @param {string} fromtitle
* @param {string} totitle
* @param {string} [summary]
* @param {object} [options]
*/
async move(fromtitle, totitle, summary, options) {
return this.request({
action: 'move',
from: fromtitle,
to: totitle,
reason: summary,
movetalk: true,
token: this.csrfToken,
...options,
}).then((data) => data.move);
}
/**
* Parse wikitext. Convenience method for 'action=parse'.
*
* @param {string} content Content to parse.
* @param {Object} additionalParams Parameters object to set custom settings, e.g.
* redirects, sectionpreview. prop should not be overridden.
* @return {Promise<string>}
*/
async parseWikitext(content, additionalParams) {
return this.request({
action: 'parse',
text: String(content),
contentmodel: 'wikitext',
disablelimitreport: true,
disableeditsection: true,
formatversion: 2,
...additionalParams,
}).then(function (data) {
return data.parse.text;
});
}
/**
* Parse a given page. Convenience method for 'action=parse'.
*
* @param {string} title Title of the page to parse
* @param {Object} additionalParams Parameters object to set custom settings, e.g.
* redirects, sectionpreview. prop should not be overridden.
* @return {Promise<string>}
*/
async parseTitle(title, additionalParams) {
return this.request({
page: String(title),
formatversion: 2,
action: 'parse',
contentmodel: 'wikitext',
...additionalParams,
}).then(function (data) {
return data.parse.text;
});
}
/**
* Upload an image from the local disk to the wiki.
* If a file with the same name exists, it will be over-written.
* @param {string} filepath
* @param {string} title
* @param {string} text
* @param {object} options
* @returns {Promise<Object>}
*/
async upload(filepath, title, text, options) {
return this.request({
action: 'upload',
file: {
stream: fs.createReadStream(filepath),
name: path.basename(filepath),
},
filename: title,
text: text,
ignorewarnings: true,
token: this.csrfToken,
...options,
}, {
headers: {
'Content-Type': 'multipart/form-data',
},
}).then((data) => {
if (data.upload.warnings) {
(0, log_1.log)(`[W] The API returned warnings while uploading to ${title}:`);
(0, log_1.log)(data.upload.warnings);
}
return data.upload;
});
}
/**
* Upload an image from a web URL to the wiki
* If a file with the same name exists, it will be over-written,
* to disable this behaviour, use `ignorewarning: false` in options.
* @param {string} url
* @param {string} title
* @param {string} text
* @param {Object} options
* @returns {Promise<Object>}
*/
async uploadFromUrl(url, title, text, options) {
return this.request({
action: 'upload',
url: url,
filename: title || path.basename(url),
text: text,
ignorewarnings: true,
token: this.csrfToken,
...options,
}).then((data) => {
if (data.upload.warnings) {
(0, log_1.log)('[W] The API returned warnings while uploading to ' + title + ':');
(0, log_1.log)(data.upload.warnings);
}
return data.upload;
});
}
/**
* Download an image from the wiki.
* If you're downloading multiple images, then for better efficiency, you may want
* to query the API for the urls of all images in one request, and follow that with
* running downloadFromUrl for each one.
* @param {string|number} file - title or page ID
* @param {string} [localname] - local path (with file name) to download to,
* defaults to current directory with same file name as on the wiki.
* @returns {Promise<void>}
*/
async download(file, localname) {
return this.request({
action: 'query',
...(0, utils_1.makeTitles)(file),
prop: 'imageinfo',
iiprop: 'url',
}).then((data) => {
// TODO: handle errors
const url = data.query.pages[0].imageinfo[0].url;
const name = new this.Title(data.query.pages[0].title).getMainText();
return this.downloadFromUrl(url, localname || name);
});
}
/**
* Download an image from a URL.
* @param {string} url
* @param {string} [localname] - local path (with file name) to download to,
* defaults to current directory with same file name as that of the web image.
* @returns {Promise<void>}
*/
async downloadFromUrl(url, localname) {
return this.rawRequest({
method: 'get',
url: url,
responseType: 'stream',
}).then((response) => {
const writeStream = response.data.pipe(fs.createWriteStream(localname || path.basename(url)));
return new Promise((resolve, reject) => {
writeStream.on('finish', () => {
resolve();
});
writeStream.on('error', (err) => {
reject(err);
});
});
});
}
async saveOption(option, value) {
return this.saveOptions({ [option]: value });
}
async saveOptions(options) {
return this.request({
action: 'options',
change: Object.entries(options).map(([key, val]) => key + '=' + val),
token: this.csrfToken,
});
}
/**
* Convenience method for `action=rollback`.
*
* @param {string|number} page - page title or page id as number or MwnTitle object
* @param {string} user
* @param {Object} [params] Additional parameters
* @return {Promise}
*/
async rollback(page, user, params) {
return this.request({
action: 'rollback',
...(0, utils_1.makeTitle)(page),
user: user,
token: this.state.rollbacktoken,
...params,
}).then((data) => {
return data.rollback;
});
}
/**
* Purge one or more pages (max 500 for bots, 50 for others)
*
* @param {String[]|String|number[]|number} titles - page titles or page ids
* @param {Object} options
* @returns {Promise}
*/
async purge(titles, options) {
return this.request({
action: 'purge',
...(0, utils_1.makeTitles)(titles),
...options,
}).then((data) => data.purge);
}
/**
* Get pages with names beginning with a given prefix
* @param {string} prefix
* @param {Object} otherParams
*
* @returns {Promise<string[]>} - array of page titles (upto 5000 or 500)
*/
async getPagesByPrefix(prefix, otherParams) {
const title = this.Title.newFromText(prefix);
if (!title) {
throw new Error('invalid prefix for getPagesByPrefix');
}
return this.request({
action: 'query',
list: 'allpages',
apprefix: title.title,
apnamespace: title.namespace,
aplimit: 'max',
...otherParams,
}).then((data) => {
return data.query.allpages.map((pg) => pg.title);
});
}
/**
* Get pages in a category
* @param {string} category - name of category, with or without namespace prefix
* @param {Object} [otherParams]
* @returns {Promise<string[]>}
*/
async getPagesInCategory(category, otherParams) {
const title = this.Title.newFromText(category, 14);
return this.request({
action: 'query',
list: 'categorymembers',
cmtitle: title.toText(),
cmlimit: 'max',
...otherParams,
}).then((data) => {
return data.query.categorymembers.map((pg) => pg.title);
});
}
/**
* Search the wiki.
* @param {string} searchTerm
* @param {number} limit
* @param {("size"|"timestamp"|"wordcount"|"snippet"|"redirectitle"|"sectiontitle"|
* "redirectsnippet"|"titlesnippet"|"sectionsnippet"|"categorysnippet")[]} props
* @param {Object} otherParams
* @returns {Promise<Object>}
*/
async search(searchTerm, limit = 50, props, otherParams) {
return this.request({
action: 'query',
list: 'search',
srsearch: searchTerm,
srlimit: limit,
srprop: props || ['size', 'wordcount', 'timestamp'],
...otherParams,
}).then((data) => {
return data.query.search;
});
}
/************* BULK PROCESSING FUNCTIONS ************/
/**
* Send an API query that automatically continues till the limit is reached.
*
* @param {Object} query - The API query
* @param {number} [limit=10] - limit on the maximum number of API calls to go through
* @returns {Promise<Object[]>} - resolved with an array of responses of individual calls.
*/
continuedQuery(query, limit = 10) {
let responses = [];
let callApi = (query, count) => {
return this.request(query).then((response) => {
if (!this.options.silent) {
(0, log_1.log)(`[+] Got part ${count} of continuous API query`);
}
responses.push(response);
if (response.continue && count < limit) {
return callApi((0, utils_1.merge)(query, response.continue), count + 1);
}
else {
return responses;
}
});
};
return callApi(query, 1);
}
/**
* Generator to iterate through API response continuations.
* @generator
* @param {Object} query
* @param {number} [limit=Infinity]
* @yields {Object} a single page of the response
*/
async *continuedQueryGen(query, limit = Infinity) {
let response = { continue: {} };
for (let i = 0; i < limit; i++) {
if (response.continue) {
response = await this.request((0, utils_1.merge)(query, response.continue));
yield response;
}
else {
break;
}
}
}
/**
* Function for using API action=query with more than 50/500 items in multi-
* input fields.
*
* Multi-value fields in the query API take multiple inputs as an array
* (internally converted to a pipe-delimted string) but with a limit of 500
* (or 50 for users without apihighlimits).
* Example: the fields titles, pageids and revids in any query, ususers in
* list=users.
*
* This function allows you to send a query as if this limit didn't exist.
* The array given to the multi-input field is split into batches and individual
* queries are sent sequentially for each batch.
* A promise is returned finally resolved with the array of responses of each
* API call.
*
* The API calls are made via POST instead of GET to avoid potential 414 (URI
* too long) errors.
*
* @param {Object} query - the query object, the multi-input field should
* be an array
* @param {string} [batchFieldName=titles] - the name of the multi-input field
*
* @returns {Promise<Object[]>} - promise resolved when all the API queries have
* settled, with the array of responses.
*/
massQuery(query, batchFieldName = 'titles') {
let batchValues = query[batchFieldName];
if (!Array.isArray(batchValues)) {
throw new Error(`massQuery: batch field in query must be an array`);
}
const limit = this.hasApiHighLimit ? 500 : 50;
const numBatches = Math.ceil(batchValues.length / limit);
let batches = new Array(numBatches);
for (let i = 0; i < numBatches - 1; i++) {
batches[i] = new Array(limit);
}
batches[numBatches - 1] = new Array(batchValues.length % limit);
for (let i = 0; i < batchValues.length; i++) {
batches[Math.floor(i / limit)][i % limit] = batchValues[i];
}
let responses = new Array(numBatches);
return new Promise((resolve) => {
const sendQuery = (idx) => {
if (idx === numBatches) {
return resolve(responses);
}
query[batchFieldName] = batches[idx];
this.request(query, { method: 'post' })
.then((response) => {
responses[idx] = response;
}, (err) => {
responses[idx] = err;
})
.finally(() => {
sendQuery(idx + 1);
});
};
sendQuery(0);
});
}
/**
* Generator version of massQuery(). Iterate through pages of API results.
* @param {Object} query
* @param {string} [batchFieldName=titles]
* @param {number} [batchSize]
*/
async *massQueryGen(query, batchFieldName = 'titles', batchSize) {
let batchValues = query[batchFieldName];
if (!Array.isArray(batchValues)) {
throw new Error(`massQuery: batch field in query must be an array`);
}
const limit = batchSize || (this.hasApiHighLimit ? 500 : 50);
const batches = (0, utils_1.arrayChunk)(batchValues, limit);
const numBatches = batches.length;
for (let i = 0; i < numBatches; i++) {
query[batchFieldName] = batches[i];
yield await this.request(query, { method: 'post' });
}
}
/**
* Execute an asynchronous function on a large number of pages (or other arbitrary
* items). Designed for working with promises.
*
* @param {Array} list - list of items to execute actions upon. The array would
* usually be of page names (strings).
* @param {Function} worker - function to execute upon each item in the list. Must
* return a promise.
* @param {number} [concurrency=5] - number of concurrent operations to take place.
* Set this to 1 for sequential operations. Default 5. Set this according to how
* expensive the API calls made by worker are.
* @param {number} [retries=0] - max number of times failing actions should be retried.
* @returns {Promise<Object>} - resolved when all API calls have finished, with object
* { failures: [ ...list of failed items... ] }
*/
batchOperation(list, worker, concurrency = 5, retries = 0) {
if (!list.length) {
return Promise.resolve({ failures: {} });
}
let counts = {
successes: 0,
failures: 0,
};
let failures = [];
let incrementSuccesses = () => {
counts.successes++;
};
const incrementFailures = (item, error) => {
counts.failures++;
failures.push({ item, error });
};
const updateStatusText = () => {
const percentageFinished = Math.round(((counts.successes + counts.failures) / list.length) * 100);
const percentageSuccesses = Math.round((counts.successes / (counts.successes + counts.failures)) * 100);