get-wikipedia-article/dist/bundle.js

Fetch Wikipedia article resources (e.g. title, body, links...).

class CustomError extends Error {
  /**
   * Create a `CustomError` (same as `Error` but with an extra property called `library`).
   * @param message Argument to be passed to base constructor (`Error()`).
   * @param library Name of the library in which error will be thrown.
   */
  constructor(message, library) {
    super(message);
    this.library = library;
  }
  toString() {
    return `Error in \`${this.library}\` library:\n\t${this.message}`;
  }
}

/*
  max must be at least `min * 2 - 1`
  otherwise, min & max couldn't be used to break down any number inside range `max to (min * 2)`

  e.g. if min is 7, max can't be anything less than 13
    - range 7 - 12 = couldn't break down 13
    - range 7 - 11 = couldn't break down 12, 13
    - range 7 - 10 = couldn't break down 11, 12, 13
    - so on...
 */
// all languages in which Wikipedia is available
// script used to fetch languages object is at `./languages.js`
/* cSpell:disable */
const wikipediaLanguages = {
  aa: 'Afar',
  ab: 'Abkhazian',
  ace: 'Achinese',
  ady: 'Adyghe',
  af: 'Afrikaans',
  ak: 'Akan',
  als: 'Alemannisch',
  alt: 'Southern Altai',
  am: 'Amharic',
  ami: 'Amis',
  an: 'Aragonese',
  ang: 'Old English',
  anp: 'Angika',
  ar: 'Arabic',
  arc: 'Aramaic',
  ary: 'Moroccan Arabic',
  arz: 'Egyptian Arabic',
  as: 'Assamese',
  ast: 'Asturian',
  atj: 'Atikamekw',
  av: 'Avaric',
  avk: 'Kotava',
  awa: 'Awadhi',
  ay: 'Aymara',
  az: 'Azerbaijani',
  azb: 'South Azerbaijani',
  ba: 'Bashkir',
  ban: 'Balinese',
  bar: 'Bavarian',
  'bat-smg': 'Samogitian',
  bcl: 'Central Bikol',
  be: 'Belarusian',
  'be-x-old': 'Belarusian (Taraškievica orthography)',
  bg: 'Bulgarian',
  bh: 'Bhojpuri',
  bi: 'Bislama',
  bjn: 'Banjar',
  blk: "Pa'O",
  bm: 'Bambara',
  bn: 'Bangla',
  bo: 'Tibetan',
  bpy: 'Bishnupriya',
  br: 'Breton',
  bs: 'Bosnian',
  bug: 'Buginese',
  bxr: 'Russia Buriat',
  ca: 'Catalan',
  'cbk-zam': 'Chavacano',
  cdo: 'Min Dong Chinese',
  ce: 'Chechen',
  ceb: 'Cebuano',
  ch: 'Chamorro',
  cho: 'Choctaw',
  chr: 'Cherokee',
  chy: 'Cheyenne',
  ckb: 'Central Kurdish',
  co: 'Corsican',
  cr: 'Cree',
  crh: 'Crimean Tatar',
  cs: 'Czech',
  csb: 'Kashubian',
  cu: 'Church Slavic',
  cv: 'Chuvash',
  cy: 'Welsh',
  da: 'Danish',
  dag: 'Dagbani',
  de: 'German',
  din: 'Dinka',
  diq: 'Zazaki',
  dsb: 'Lower Sorbian',
  dty: 'Doteli',
  dv: 'Divehi',
  dz: 'Dzongkha',
  ee: 'Ewe',
  el: 'Greek',
  eml: 'Emiliano-Romagnolo',
  en: 'English',
  eo: 'Esperanto',
  es: 'Spanish',
  et: 'Estonian',
  eu: 'Basque',
  ext: 'Extremaduran',
  fa: 'Persian',
  ff: 'Fula',
  fi: 'Finnish',
  'fiu-vro': 'võro',
  fj: 'Fijian',
  fo: 'Faroese',
  fr: 'French',
  frp: 'Arpitan',
  frr: 'Northern Frisian',
  fur: 'Friulian',
  fy: 'Western Frisian',
  ga: 'Irish',
  gag: 'Gagauz',
  gan: 'Gan Chinese',
  gcr: 'Guianan Creole',
  gd: 'Scottish Gaelic',
  gl: 'Galician',
  glk: 'Gilaki',
  gn: 'Guarani',
  gom: 'Goan Konkani',
  gor: 'Gorontalo',
  got: 'Gothic',
  gu: 'Gujarati',
  guc: 'Wayuu',
  gur: 'Frafra',
  guw: 'Gun',
  gv: 'Manx',
  ha: 'Hausa',
  hak: 'Hakka Chinese',
  haw: 'Hawaiian',
  he: 'Hebrew',
  hi: 'Hindi',
  hif: 'Fiji Hindi',
  ho: 'Hiri Motu',
  hr: 'Croatian',
  hsb: 'Upper Sorbian',
  ht: 'Haitian Creole',
  hu: 'Hungarian',
  hy: 'Armenian',
  hyw: 'Western Armenian',
  hz: 'Herero',
  ia: 'Interlingua',
  id: 'Indonesian',
  ie: 'Interlingue',
  ig: 'Igbo',
  ii: 'Sichuan Yi',
  ik: 'Inupiaq',
  ilo: 'Iloko',
  inh: 'Ingush',
  io: 'Ido',
  is: 'Icelandic',
  it: 'Italian',
  iu: 'Inuktitut',
  ja: 'Japanese',
  jam: 'Jamaican Creole English',
  jbo: 'Lojban',
  jv: 'Javanese',
  ka: 'Georgian',
  kaa: 'Kara-Kalpak',
  kab: 'Kabyle',
  kbd: 'Kabardian',
  kbp: 'Kabiye',
  kcg: 'Tyap',
  kg: 'Kongo',
  ki: 'Kikuyu',
  kj: 'Kuanyama',
  kk: 'Kazakh',
  kl: 'Kalaallisut',
  km: 'Khmer',
  kn: 'Kannada',
  ko: 'Korean',
  koi: 'Komi-Permyak',
  kr: 'Kanuri',
  krc: 'Karachay-Balkar',
  ks: 'Kashmiri',
  ksh: 'Colognian',
  ku: 'Kurdish',
  kv: 'Komi',
  kw: 'Cornish',
  ky: 'Kyrgyz',
  la: 'Latin',
  lad: 'Ladino',
  lb: 'Luxembourgish',
  lbe: 'Lak',
  lez: 'Lezghian',
  lfn: 'Lingua Franca Nova',
  lg: 'Ganda',
  li: 'Limburgish',
  lij: 'Ligurian',
  lld: 'Ladin',
  lmo: 'Lombard',
  ln: 'Lingala',
  lo: 'Lao',
  lrc: 'Northern Luri',
  lt: 'Lithuanian',
  ltg: 'Latgalian',
  lv: 'Latvian',
  mad: 'Madurese',
  mai: 'Maithili',
  'map-bms': 'Basa Banyumasan',
  mdf: 'Moksha',
  mg: 'Malagasy',
  mh: 'Marshallese',
  mhr: 'Eastern Mari',
  mi: 'Māori',
  min: 'Minangkabau',
  mk: 'Macedonian',
  ml: 'Malayalam',
  mn: 'Mongolian',
  mni: 'Manipuri',
  mnw: 'Mon',
  mr: 'Marathi',
  mrj: 'Western Mari',
  ms: 'Malay',
  mt: 'Maltese',
  mus: 'Muscogee',
  mwl: 'Mirandese',
  my: 'Burmese',
  myv: 'Erzya',
  mzn: 'Mazanderani',
  na: 'Nauru',
  nah: 'Nāhuatl',
  nap: 'Neapolitan',
  nds: 'Low German',
  'nds-nl': 'Low Saxon',
  ne: 'Nepali',
  new: 'Newari',
  ng: 'Ndonga',
  nia: 'Nias',
  nl: 'Dutch',
  nn: 'Norwegian Nynorsk',
  no: 'Norwegian',
  nov: 'Novial',
  nqo: 'N’Ko',
  nrm: 'Norman',
  nso: 'Northern Sotho',
  nv: 'Navajo',
  ny: 'Nyanja',
  oc: 'Occitan',
  olo: 'Livvi-Karelian',
  om: 'Oromo',
  or: 'Odia',
  os: 'Ossetic',
  pa: 'Punjabi',
  pag: 'Pangasinan',
  pam: 'Pampanga',
  pap: 'Papiamento',
  pcd: 'Picard',
  pcm: 'Nigerian Pidgin',
  pdc: 'Pennsylvania German',
  pfl: 'Palatine German',
  pi: 'Pali',
  pih: 'Norfuk / Pitkern',
  pl: 'Polish',
  pms: 'Piedmontese',
  pnb: 'Western Punjabi',
  pnt: 'Pontic',
  ps: 'Pashto',
  pt: 'Portuguese',
  pwn: 'Paiwan',
  qu: 'Quechua',
  rm: 'Romansh',
  rmy: 'Vlax Romani',
  rn: 'Rundi',
  ro: 'Romanian',
  'roa-rup': 'Aromanian',
  'roa-tara': 'Tarantino',
  ru: 'Russian',
  rue: 'Rusyn',
  rw: 'Kinyarwanda',
  sa: 'Sanskrit',
  sah: 'Yakut',
  sat: 'Santali',
  sc: 'Sardinian',
  scn: 'Sicilian',
  sco: 'Scots',
  sd: 'Sindhi',
  se: 'Northern Sami',
  sg: 'Sango',
  sh: 'Serbo-Croatian',
  shi: 'Tachelhit',
  shn: 'Shan',
  si: 'Sinhala',
  simple: 'Simple English',
  sk: 'Slovak',
  skr: 'Saraiki',
  sl: 'Slovenian',
  sm: 'Samoan',
  smn: 'Inari Sami',
  sn: 'Shona',
  so: 'Somali',
  sq: 'Albanian',
  sr: 'Serbian',
  srn: 'Sranan Tongo',
  ss: 'Swati',
  st: 'Southern Sotho',
  stq: 'Saterland Frisian',
  su: 'Sundanese',
  sv: 'Swedish',
  sw: 'Swahili',
  szl: 'Silesian',
  szy: 'Sakizaya',
  ta: 'Tamil',
  tay: 'Tayal',
  tcy: 'Tulu',
  te: 'Telugu',
  tet: 'Tetum',
  tg: 'Tajik',
  th: 'Thai',
  ti: 'Tigrinya',
  tk: 'Turkmen',
  tl: 'Tagalog',
  tn: 'Tswana',
  to: 'Tongan',
  tpi: 'Tok Pisin',
  tr: 'Turkish',
  trv: 'Taroko',
  ts: 'Tsonga',
  tt: 'Tatar',
  tum: 'Tumbuka',
  tw: 'Twi',
  ty: 'Tahitian',
  tyv: 'Tuvinian',
  udm: 'Udmurt',
  ug: 'Uyghur',
  uk: 'Ukrainian',
  ur: 'Urdu',
  uz: 'Uzbek',
  ve: 'Venda',
  vec: 'Venetian',
  vep: 'Veps',
  vi: 'Vietnamese',
  vls: 'West Flemish',
  vo: 'Volapük',
  wa: 'Walloon',
  war: 'Waray',
  wo: 'Wolof',
  wuu: 'Wu Chinese',
  xal: 'Kalmyk',
  xh: 'Xhosa',
  xmf: 'Mingrelian',
  yi: 'Yiddish',
  yo: 'Yoruba',
  za: 'Zhuang',
  zea: 'Zeelandic',
  zh: 'Chinese',
  'zh-classical': 'Classical Chinese',
  'zh-min-nan': 'Chinese (Min Nan)',
  'zh-yue': 'Cantonese',
  zu: 'Zulu'
};
/* cSpell:enable */

// messages aren't inside an object because
// tree-shaking isn't possible when exporting an object
// https://medium.com/@rauschma/note-that-default-exporting-objects-is-usually-an-anti-pattern-if-you-want-to-export-the-cf674423ac38#.nibatprx3
// used at `fullfiller/src/validate`
//
const invalidLanguage = (language, languages) => `Expected \`language\` argument to be one of the supported languages (${languages.join(', ')}).`;
const articleNotFound = 'Wikipedia does not have an article with this exact title. Try again using a different query.';
const articleIsDisambiguation = suggestions => `This query points to a Wikipedia disambiguation page. You've got to be more specific.${suggestions.length > 0 ? ` Query suggestions:\n- ${suggestions.splice(0, 10).join('\n- ')}.` : ` No query suggestions were found.`}`;

/**
 * Summary is the initial chunk of text, everything before the first subtitle.
 * It's possible to get it with API `&action=query&prop=extracts&exintro`.
 * But the entire article was fetched already, so there's no need to make another API call.
 *
 * @summary Extract summary from article body.
 * @param body Wikipedia article body.
 * @param format Article format.
 * @returns Wikipedia article summary.
 */
function extractSummaryFromBody(body, format) {
  const plaintextRE = /^[\s\S]*?(?=\n\n\n==)/;
  const htmlRE = /^[\s\S]*?(?=\n\n<h2>)/;
  const RE = format === 'plain' ? plaintextRE : htmlRE;
  const summary = RE.exec(body)?.[0];
  return summary;
}

/**
 * Join Wikipedia API base URL and `queries`.
 * @param queries Object to be converted to query string.
 * @returns URL to make API call.
 */
function generateRequestURL(language, queries) {
  const baseAPI = `https://${language}.wikipedia.org/w/api.php?&format=json&origin=*&`;
  const queryString = Object.entries(queries).map(([key, value]) => value === undefined ? key : `${key}=${value.toString()}`).join('&');
  const requestURL = baseAPI + queryString;
  return requestURL;
}
/**
 * @summary Fetch response from Wikipedia API.
 * @param queries Object containing query string parameters.
 * @returns Object containing json.query.pages[pageID] and json.continue (if it exists) contents.
 */
async function fetchResource(language, queries) {
  const url = generateRequestURL(language, queries);
  const json = await (await fetch(url)).json();
  // API's response isn't returned in its entirety by this function.
  // That's because the main part of the response is nested 3 layers deep.
  // As shown in the `response` interface.
  const {
    pages
  } = json.query;
  const pagesID = Object.keys(pages)[0];
  const resp = {
    ...pages[pagesID],
    // some resources responses (e.g. links) include `continue` when there's more to fetch
    ...(json.continue && json.continue)
  };
  return resp;
}

/**
 * Fetch Wikipedia article body.
 * @param title Wikipedia article title.
 * @param format Which one of the 2 formats available in the Wikipedia API.
 * @param related Wikipedia related articles titles (to recommend in case of error).
 * @returns Wikipedia article body.
 */
async function getArticleBody(language, title, format) {
  const queries = {
    action: 'query',
    prop: 'extracts',
    ...(format === 'plain' && {
      explaintext: undefined
    }),
    redirects: undefined,
    titles: encodeURIComponent(title)
  };
  const resp = await fetchResource(language, queries);
  const body = resp.extract;
  return body;
}

/**
 * Fetch all Wikipedia categories the given article belongs to.
 * @param title Wikipedia article title.
 * @returns Array of Wikipedia categories names.
 */
async function getArticleCategories(language, title) {
  const queries = {
    action: 'query',
    prop: 'categories',
    redirects: undefined,
    cllimit: 'max',
    clshow: '!hidden',
    titles: encodeURIComponent(title)
  };
  // There's no reason to use a recursive function like in getArticleLinks,
  // even the articles containing the most categories don't have more than 500 categories
  // (https://en.wikipedia.org/wiki/Special:MostCategories);
  // request below will fetch at most 500 normal (not hidden) categories
  const resp = await fetchResource(language, queries);
  const categories = resp.categories.map(obj => obj.title.replace(/^Category:/, ''));
  return categories;
}

async function getLinksRecursively(language, queries) {
  const resp = await fetchResource(language, queries);
  const links = resp.links.map(obj => obj.title);
  return !('plcontinue' in resp) ? links : links.concat(await getLinksRecursively(language, {
    ...queries,
    plcontinue: encodeURIComponent(resp.plcontinue)
  }));
}
/**
 * Fetch all Wikipedia articles that are linked in the given article.
 * @param title Wikipedia article title.
 * @returns Array of Wikipedia articles titles.
 */
async function getArticleLinks(language, title) {
  const queries = {
    action: 'query',
    prop: 'links',
    redirects: undefined,
    pllimit: 'max',
    plnamespace: '0',
    titles: encodeURIComponent(title)
  };
  const links = await getLinksRecursively(language, queries);
  return links;
}

/**
 * Fetch Wikipedia article summary.
 * @param title Wikipedia article title.
 * @param format Which one of the 2 formats available in the Wikipedia API.
 * @param related Wikipedia related articles titles (to recommend in case of error).
 * @returns Wikipedia article body.
 */
async function getArticleSummary(language, title, format) {
  const queries = {
    action: 'query',
    prop: 'extracts',
    exintro: undefined,
    redirects: undefined,
    ...(format === 'plain' && {
      explaintext: undefined
    }),
    titles: encodeURIComponent(title)
  };
  const resp = await fetchResource(language, queries);
  const summary = resp.extract;
  return summary;
}

/**
 * Fetch Wikipedia article terms.
 * @param title Wikipedia article title.
 * @param include Which terms to fetch (alias, description and/or label).
 * @returns Array of Wikipedia article terms.
 */
async function getArticleTerms(language, title, include) {
  const queries = {
    action: 'query',
    prop: 'pageterms',
    redirects: undefined,
    wbptlanguage: 'en',
    wbptterms: include.join('|'),
    titles: encodeURIComponent(title)
  };
  const resp = await fetchResource(language, queries);
  const {
    terms
  } = resp;
  // if (include.length !== Object.keys(terms).length) {
  //   const print = (array: string[]) => array.sort().join(', ');
  //   console.warn(
  //     `Article lacks some of the terms requested. Requested: ${print(
  //       include
  //     )}. Fetched: ${print(Object.keys(terms))}.`
  //   );
  // }
  return terms;
}

/**
 * Fetch Wikipedia article(s) title(s) that match query.
 * @param query Search string.
 * @param single Fetch only a single result.
 * @throws Error if no results were found.
 * @returns Array of title(s).
 */
async function getMatchingArticlesTitles(language, query, single = false) {
  const requestURL = generateRequestURL(language, {
    action: 'opensearch',
    limit: single ? 1 : 'max',
    // `redirects=resolve` is required otherwise response may contain the name of a redirect
    // instead of the target page; page and redirect have slight different titles
    // e.g. query `lord of the rings` would return `Lord of the rings`
    // which is the redirect for `The Lord of the Rings`
    redirects: 'resolve',
    search: encodeURIComponent(query)
  });
  const resp = await fetch(requestURL);
  const json = await resp.json();
  const titles = json[1];
  if (titles.length === 0) throw new CustomError(articleNotFound, 'get-wikipedia-article');
  return titles;
}

async function queryPointsToADisambiguationPage(language, title) {
  const queries = {
    action: 'query',
    prop: 'pageprops',
    ppprop: 'disambiguation',
    redirects: undefined,
    titles: encodeURIComponent(title)
  };
  const resp = await fetchResource(language, queries);
  const pointsToDisambiguation = resp.pageprops?.disambiguation !== undefined;
  return pointsToDisambiguation;
}

const includeDefault = ['title', 'body'];
/**
 * Fetch Wikipedia article's resources (e.g. title, body, links...).
 * @param query Search string.
 * @param options Miscellaneous options.
 * @throws Error if `query` doesn't return any results.
 * @throws Error if `article.title` points to a disambiguation page.
 * @returns Object containing requested resources.
 */
async function getWikipediaArticle(query, {
  language = 'en',
  include = includeDefault,
  format = 'plain'
} = {}) {
  if (!Object.keys(wikipediaLanguages).includes(language)) {
    throw new CustomError(invalidLanguage(language, Object.keys(wikipediaLanguages)), 'get-wikipedia-article');
  }
  if (include.length === 0) include.push(...includeDefault);
  const article = {};
  // fetch title, related
  if (include.includes('title') && include.includes('related')) {
    // first result will be selected as the article to be fetched
    const [title, ...related] = await getMatchingArticlesTitles(language, query);
    article.title = title;
    article.related = related;
  } else if (include.includes('title')) {
    const [title] = await getMatchingArticlesTitles(language, query);
    article.title = title;
  } else if (include.includes('related')) {
    const [, ...related] = await getMatchingArticlesTitles(language, query);
    article.related = related;
  }
  // API calls to `action=query` using `query` instead of `article.title`
  // are allowed because `redirects` parameter is being used
  const titleQuery = article.title || query;
  // the only option other than to make a separate request at main function checking if page is
  // disambiguation, would be to check if page is disambiguation at every resource request
  if (await queryPointsToADisambiguationPage(language, titleQuery)) {
    throw new CustomError(articleIsDisambiguation(article.related || [] /* suggestions */), 'get-wikipedia-article');
  }
  // fetch body
  if (include.includes('body')) {
    article.body = await getArticleBody(language, titleQuery, format);
  }
  // fetch summary
  if (include.includes('summary')) {
    article.summary = article.body ? extractSummaryFromBody(article.body, format) : await getArticleSummary(language, titleQuery, format);
  }
  // fetch categories
  if (include.includes('categories')) {
    article.categories = await getArticleCategories(language, titleQuery);
  }
  // fetch links
  if (include.includes('links')) {
    article.links = await getArticleLinks(language, titleQuery);
  }
  // fetch terms
  const termsToInclude = ['alias', 'label', 'description'].filter(term => include.includes(term));
  if (termsToInclude.length > 0) {
    const terms = await getArticleTerms(language, titleQuery, termsToInclude);
    Object.keys(terms).forEach(term => {
      article[term] = terms[term];
    });
  }
  return article;
}

export { getWikipediaArticle as default };