cetem-publico/test.js

A wrapper for CETEMPúblico, an European Portuguese corpus of news extracts from the newspaper Público, with 180 million words tagged automatically using PALAVRAS.

var zlib   = require('zlib');
var gunzip = zlib.createGunzip().on('error', err => {
  console.warn(`Could not read file`);
  console.warn(err);
});

var readChunk = require('read-chunk');
var fileType = require('file-type');
var buffer = readChunk.sync('/home/andrefs/.cetem-publico/CETEMPublicoAnotado2019.gz', 0, fileType.minimumBytes);
var ft = fileType(buffer);
var isGzip = ft && /application\/gzip/.test(ft.mime);
var input = isGzip ?
  fs.createReadStream('/home/andrefs/.cetem-publico/CETEMPublicoAnotado2019.gz').pipe(gunzip) :
  fs.createReadStream('/home/andrefs/.cetem-publico/CETEMPublicoAnotado2019.gz');
var rl = readline.createInterface({input});

rl.on('line', (line) => {
  console.log(`Line from file: ${line}`);
});