redact-pii/test-redact-pii-memory-leak.js

Remove personally identifiable information from text.

process.env.GOOGLE_APPLICATION_CREDENTIALS = require('path').resolve(
  // __dirname + '/../node-mordor/credentials/node-mordor-prod-key.json'
  __dirname + '/../node-mordor/credentials/node-mordor-dev-key.json'
);

// const fs = require('fs');
const Bluebird = require('bluebird');
const _ = require('lodash');
const { SyncRedactor, AsyncRedactor, GoogleDLPRedactor } = require('./lib/index.js');

const asyncRedactorOptions = {
  builtInRedactors: {
    zipcode: {
      enabled: false
    },
    digits: {
      enabled: false
    }
  },
  customRedactors: {
    after: [
      new GoogleDLPRedactor({
        clientOptions: {
          keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS
        }
      })
    ]
  }
};
const redactor = new AsyncRedactor(asyncRedactorOptions);

(async function(){

  for (let i=0; i<1000; i++) {

    const strs = [];
    for (let j=0; j<1000; j++) {
      const random = Math.floor(Math.random()*1000000);
      strs.push(`Hi, my email is john.doe.${random}@example.com and I live in Alabama`);
    }

    await Bluebird.map(strs, async (str, index) => {
      try {
        const redacted = await redactor.redactAsync(str);
        console.log(str, redacted, i, index);
        if (index % 10 === 0) {
          console.log('######### memory:', process.memoryUsage().heapUsed/1024/1024);
        }
      } catch (e) {
        console.error(e);
      }
    }, {concurrency: 5});
  }

})();