wikibase-dump-filter/bin/wikibase-dump-filter.js

Filter and format a newline-delimited JSON stream of Wikibase entities

#!/usr/bin/env node
import { resolve } from 'node:path'
import program from 'commander'
import filterFormatAndSerializeEntity from '../lib/filter_format_and_serialize_entity.js'
import { getDirname, readJsonFile } from '../lib/fs.js'
import getEntitiesStream from '../lib/get_entities_stream.js'
import handleErrors from '../lib/handle_errors.js'
import help from '../lib/help.js'
import { beforeFilter, afterFilter, afterNegativeFilter, updateWithLastData } from '../lib/progress_bar_hooks.js'
import { logHeaders } from '../lib/update_progress_bar.js'

const dirname = getDirname(import.meta.url)
const { version } = readJsonFile(resolve(dirname, '../package.json'))

const list = val => val.split(',')

const options = program
  .version(version)
  // filters
  .option('-t, --type <type>', help.type)
  .option('-c, --claim <claim>', help.claim)
  .option('-i, --sitelink <sitelink>', help.sitelink)
  // formatters
  .option('-o, --omit <attributes>', help.omit, list)
  .option('-k, --keep <attributes>', help.keep, list)
  .option('-l, --languages <languages>', help.languages, list)
  .option('-s, --simplify [boolean|options]', help.simplify)
  .option('-z, --include-size [boolean]', help.includesize)
  // misc
  .option('-p, --progress', help.progress)
  .option('-q, --quiet', help.quiet)
  .parse(process.argv)

let showProgressBar
let afterLastLine = () => {}

if (process.stdout.isTTY) {
  // When output is the terminal, show only the kept lines
  // unless progress logs are explicitly requested
  showProgressBar = options.progress === true && options.quiet !== true
} else {
  showProgressBar = options.quiet !== true
}

if (showProgressBar) {
  logHeaders()
  updateWithLastData()
  Object.assign(options, { beforeFilter, afterFilter, afterNegativeFilter })
  afterLastLine = updateWithLastData
}

getEntitiesStream(process.stdin, options)
.filterAndMap(filterFormatAndSerializeEntity(options))
.on('error', handleErrors)
.on('close', afterLastLine)
.pipe(process.stdout)
.on('error', handleErrors)