redact-pii
Version:
Remove personally identifiable information from text.
349 lines (329 loc) • 13.1 kB
JavaScript
// process.env.GOOGLE_APPLICATION_CREDENTIALS = require('path').resolve(
// __dirname + '/../node-mordor/credentials/node-mordor-prod-key.json'
// // __dirname + '/../node-mordor/credentials/node-mordor-dev-key.json'
// );
const fs = require('fs');
const { SyncRedactor, AsyncRedactor, GoogleDLPRedactor } = require('./lib/index.js');
// const bigText = fs.readFileSync('./big.txt', 'utf8');
// const redactor = new AsyncRedactor({
// customRedactors: {
// before: [
// new GoogleDLPRedactor({
// // disableAutoBatchWhenContentSizeExceedsLimit: true,
// // maxContentSizeForBatch: 128000
// })
// ]
// }
// });
const redactor = new AsyncRedactor({
// builtInRedactors: { zipcode: { enabled: false }, digits: { enabled: false } },
// builtInRedactors: { names: { enabled: false } },
customRedactors: {
// before: [],
// after: [
after: [
new GoogleDLPRedactor({
// clientOptions: { keyFilename: '/Users/jared/dev/solvvy/node-mordor/credentials/node-mordor-prod-key.json' },
// inspectConfig: {
// ruleSet: [
// {
// infoTypes: [
// {
// name: 'PERSON_NAME'
// }
// ],
// rules: [
// {
// exclusionRule: {
// matchingType: 'MATCHING_TYPE_FULL_MATCH',
// regex: {
// pattern: 'UK|Canada|Australia|RR Donnelley|RR|Donnelley|UPS|DHL Express UK'
// }
// }
// }
// ]
// },
// {
// infoTypes: [
// {
// name: 'FIRST_NAME'
// }
// ],
// rules: [
// {
// exclusionRule: {
// matchingType: 'MATCHING_TYPE_FULL_MATCH',
// regex: {
// pattern: 'UK|Canada|Australia|RR Donnelley|RR|Donnelley|UPS|DHL Express UK'
// }
// }
// }
// ]
// },
// {
// infoTypes: [
// {
// name: 'LAST_NAME'
// }
// ],
// rules: [
// {
// exclusionRule: {
// matchingType: 'MATCHING_TYPE_FULL_MATCH',
// regex: {
// pattern: 'UK|Canada|Australia|RR Donnelley|RR|Donnelley|UPS|DHL Express UK'
// }
// }
// }
// ]
// },
// {
// infoTypes: [
// {
// name: 'LOCATION'
// }
// ],
// rules: [
// {
// exclusionRule: {
// matchingType: 'MATCHING_TYPE_FULL_MATCH',
// regex: {
// pattern: 'UK|Canada|Australia|RR Donnelley|RR|Donnelley|UPS|DHL Express UK'
// }
// }
// }
// ]
// }
// ]
// },
// excludeInfoTypes: [],
// includeInfoTypes: []
})
]
}
});
// const redactor = new GoogleDLPRedactor({
// disableAutoBatchWhenContentSizeExceedsLimit: true,
// maxContentSizeForBatch: 7
// });
// const sampleText = 'Hi David Johnson, Please give me a call at 555-555-5555';
// const unredactedText = sampleText + bigText + sampleText;
// const unredactedText = "Chat started: 2019-08-07 07:29 PM UTC\n\n(07:29:38 PM) robin fatherree: how much have i cashed out for\n(07:29:40 PM) Earnin Support: Our wait time is typically less than 2 minutes. Please stay with us so we can review your issue. We are here to help!\n(07:29:43 PM) *** Carl S. joined the chat ***\n(07:29:53 PM) Carl S.: Hi, Robin! Thanks for reaching out to us. I'm Carl from Earnin support. Please give me a moment as I pull up your account so I can assist you with your concern.\n(07:31:13 PM) Carl S.: Upon checking here, you have cashed out a total of $250 for this current pay period. You still have $200 left on your max to cash out for the pay period.\n(07:31:20 PM) Carl S.: Is there anything else I can help you with?\n(07:33:03 PM) Carl S.: Are you still there?\n(07:34:09 PM) Carl S.: Unfortunately, due to us being idle for a long period of time, I will need to close our chat now. Don't worry, though! If you need further assistance, just chat us again and we'll review this conversation so you won't need to repeat yourself. Thanks!\n(07:34:14 PM) *** Carl S. left the chat ***";
// const unredactedText = "my name is John S.";
// const unredactedText = "I live in Los Angeles, CA.";
const unredactedText2 =
`Do you ship packages with any of the following carriers?
USPSFedExUPSRoyal
MailHermes UK
Parcelforce Worldwide
DPD UK
FedEx – Canada
FedEx UK
Canada Post
Purolator
Stamps.com
Sendle
Australia Post
StarTrack
Fulfillment by Amazon (FBA)
UPS Australia
RR Donnelley
UPS UK
DHL Express
UPS Canada
SEKO Omni-Channel Logistics
DHL Express Canada
DHL Express Australia
Landmark Global
DHL Express UK
Asendia USAAPC
Globegistics
FirstMile
IMEX Global Solutions
OnTrac
Access Worldwide
Newgistics
Rakuten Super Logistics
DCL Logistics
DHL eCommerce
LSO
Parcel Technology Solutions
`;
const unredactedText = `
Rerverse payment. I thought I'd accidentally charged my client Marit' Parten's card. But now see it says cash and there's nothing to reverse. I need to double check on that. Cna you please help me? Rerverse payment. I thought I'd accidentally charged my client Marit' Parten's card. But now see it says cash and there's nothing to reverse. I need to double check on that. Cna you please help me?
`;
console.log('unredactedText length', unredactedText.length);
(async function() {
const redactedText = await redactor.redactAsync(unredactedText);
console.log('redactedText length', redactedText.length);
console.log('redactedText', redactedText);
// console.log('begin snippet', redactedText.substring(0, 500));
// console.log('end snippet', redactedText.substring(redactedText.length - 500));
// const input = `
// Go foo foo foo FTW!
// Phone #1: 801-555-1234
// Phone #2: 801-555-0000
// Ref #1: 181212-000270
// Ref #2: 181211-008247
// Email #1: user@example.com
// Email #2: exclude@example.com
// Swedish passport number: 27386473
// IP address: 127.0.0.1
// Name: John Smith
// Address: 123 Park Ave, New York City, NY 10002
// Url: http://example.com/index.html
// `;
// const input = '(foo\n/(hiHighland Ave\n/foo bar(age/ New York';
// const input = "Hi Jared also known as PERSON_NAME,\r\n\r\nThank you for trying HelloFresh. I'm sorry for the confusion with your discount offer. I see you applied the code \"8R300W234\". The code gives you a $25 discount on the first and second order and free shipping on the first delivery. I see you created a subscription for a Classic Plan (2 meals for 2 people). The regular price of the box is $39.96 + $7.99 shipping. I see that for the first delivery you were charged $14.96. Don't worry, you're not being charged for shipping.\r\n\r\nIf you need any further assistance, feel free to check out URL or contact us! Just call (646) 846-FOOD (3663) or reply here so I can help you directly. \r\n\r\nPERSON_NAME,\r\nPERSON_NAME";
// const input = "Just call (646) 846-FOOD or (646) 846-3663.";
// console.log('REDACTED: ', await redactor.redactAsync(input));
})();
// const redactor = Redactor({
// enableGoogleCloudDLP: true,
// googleCloudDLPOptions: {
// // clientOptions: {
// //
// // },
// disableFallbackRedaction: true,
// inspectOverrides: {
// inspectConfig: {
// // infoTypes: [
// // { name: 'PHONE_NUMBER' },
// // { name: 'EMAIL_ADDRESS' }
// // ],
// // customInfoTypes: [{
// // infoType: {
// // name: 'F**'
// // },
// // regex: {
// // pattern: '\\bfoo\\b'
// // }
// // }],
// ruleSet: [
// {
// infoTypes: [{ name: 'PHONE_NUMBER' }],
// rules: [
// {
// exclusionRule: {
// matchingType: 'MATCHING_TYPE_FULL_MATCH',
// regex: {
// pattern: '\\d{6}-\\d{6}'
// }
// }
// }
// ]
// },
// // {
// // infoTypes: [
// // {
// // name: 'EMAIL_ADDRESS'
// // }
// // ],
// // rules: [
// // {
// // exclusionRule: {
// // dictionary: {
// // wordList: {
// // words: [
// // 'exclude@example.com'
// // ]
// // }
// // },
// // matchingType: 'MATCHING_TYPE_FULL_MATCH'
// // }
// // }
// // ]
// // }
// ]
// },
// // includeInfoTypes: ['SWEDEN_PASSPORT'],
// // excludeInfoTypes: ['IP_ADDRESS'],
// }
// },
// zipcode: false,
// digits: false,
// url: false,
// emailAddress: false,
// phoneNumber: false,
// ipAddress: false
// });
// payoneer
// const redactor = new Redactor(
// {
// enableGoogleCloudDLP: true
// "phoneNumber": false,
// "digits": false,
// "googleCloudDLPOptions": {
// "inspectConfig": {
// "ruleSet": [
// {
// "infoTypes": [
// {
// "name": "PHONE_NUMBER"
// }
// ],
// "rules": [
// {
// "exclusionRule": {
// "matchingType": "MATCHING_TYPE_FULL_MATCH",
// "regex": {
// "pattern": "\\d{6}-\\d{6}"
// }
// }
// }
// ]
// }
// ]
// }
// }
// }
// );
// // Tastyworks
// const redactor = new Redactor(
// {
// "TASTYWORKS_ACCOUNT_NUMBER": new RegExp("\\d\\d?[a-zA-Z]{2}\\d{5}"),
// }
// );
// Tastyworks
/*
const redactor = new Redactor(
{
enableGoogleCloudDLP: true,
// "googleCloudDLPOptions": {
// "inspectOverrides": {
// "excludeInfoTypes": ["ETHNIC_GROUP", "GENDER"]
// }
// }
}
);
*/
/*
(async function () {
const input = `
Go foo foo foo FTW!
Phone #1: 801-555-1234
Phone #2: 801-555-0000
Ref #1: 181212-000270
Ref #2: 181211-008247
Email #1: user@example.com
Email #2: exclude@example.com
Swedish passport number: 27386473
IP address: 127.0.0.1
Name: John Smith
Address: 123 Park Ave, New York City, NY 10002
Url: http://example.com/index.html
`;
// const input = '(foo\n/(hiHighland Ave\n/foo bar(age/ New York';
// const input = "Hi Jared also known as PERSON_NAME,\r\n\r\nThank you for trying HelloFresh. I'm sorry for the confusion with your discount offer. I see you applied the code \"8R300W234\". The code gives you a $25 discount on the first and second order and free shipping on the first delivery. I see you created a subscription for a Classic Plan (2 meals for 2 people). The regular price of the box is $39.96 + $7.99 shipping. I see that for the first delivery you were charged $14.96. Don't worry, you're not being charged for shipping.\r\n\r\nIf you need any further assistance, feel free to check out URL or contact us! Just call (646) 846-FOOD (3663) or reply here so I can help you directly. \r\n\r\nPERSON_NAME,\r\nPERSON_NAME";
// const input = "Just call (646) 846-FOOD or (646) 846-3663.";
// console.log('REDACTED: ', await redactor.redact(input));
// console.log('REDACTED: ', await redactor.redact('example@example.com'));
// console.log('REDACTED: ', await redactor.redact('Hi my name is John.'));
// console.log('REDACTED: ', await redactor.redact('My phone is 111-222-3333 and my ref is 181211-008247'));
// console.log('REDACTED: ', await redactor.redact('how do I reset my password?\nMy account number is 5WT52295'));
// console.log('REDACTED: ', await redactor.redact('how do I reset my password?\nMy account number is 15AB52295'));
// console.log('REDACTED: ', await redactor.redact('I deleted three items (brownies, black bean zucchini, banana bread oats) but it only allowed me to add one (chick pea cookies).'));
console.log('REDACTED: ', await redactor.redact('Why are the black beans so yummy in my little rumbly tummy? My home state is California.'));
// console.log('REDACTED: ', await redactor.redact('My phone number is (415) 555-0890. My main email is example@example.com and my backup is foo@example.com.'));
})();
*/