tabular-data-differ
Version:
A very efficient library for diffing two sorted streams of tabular data, such as CSV files.
1,445 lines (1,436 loc) • 106 kB
text/typescript
import fs from 'fs';
import { describe, expect, test } from '@jest/globals';
import { DifferOptions, diff, UnorderedStreamsError, Differ, UniqueKeyViolationError, DuplicateKeyHandler } from './differ';
import { FormatWriter, FormatHeader, RowDiff, FormatFooter, CsvFormatReader, defaultRowComparer, Column, Row, CellValue, stringComparer, cellComparer } from './formats';
import { ArrayInputStream, FileOutputStream, NullOutputStream } from './streams';
class FakeFormatWriter implements FormatWriter{
public header?: FormatHeader;
public diffs: RowDiff[] = [];
public footer?: FormatFooter;
open(): Promise<void> {
return Promise.resolve();
}
writeHeader(header: FormatHeader): Promise<void> {
this.header = header;
return Promise.resolve();
}
writeDiff(rowDiff: RowDiff): Promise<void> {
this.diffs.push(rowDiff);
return Promise.resolve();
}
writeFooter(footer: FormatFooter): Promise<void> {
this.footer = footer;
return Promise.resolve();
}
close(): Promise<void> {
return Promise.resolve();
}
}
type DiffOptions = Omit<DifferOptions, "oldSource" | "newSource"> & {
oldLines: string[],
newLines: string[],
keepSameRows?: boolean,
changeLimit?: number,
};
async function diffStrings(options: DiffOptions): Promise<FakeFormatWriter> {
const writer = new FakeFormatWriter();
await diff({
...options,
oldSource: {
format: 'csv',
stream: new ArrayInputStream(options.oldLines)
},
newSource: {
format: 'csv',
stream: new ArrayInputStream(options.newLines),
},
}).to({
destination: {
format: 'custom',
writer,
},
keepSameRows: options.keepSameRows,
changeLimit: options.changeLimit,
});
return writer;
}
function readAllText(path: string): string {
return fs.readFileSync(path).toString();
}
describe('differ', () => {
beforeAll(() => {
if(!fs.existsSync('./output')) {
fs.mkdirSync('./output');
}
if(!fs.existsSync('./output/files')) {
fs.mkdirSync('./output/files');
}
});
describe('validation errors', () => {
test('should reject unknown source format', async () => {
await expect(async () => {
await diff({
oldSource: {
format: <any>'foobar',
stream: './tests/a.csv',
},
newSource: {
format: 'csv',
stream: './tests/b.csv',
},
keys: ['id'],
}).to('null');
}).rejects.toThrowError(`Unknown source format 'foobar'`);
});
test('should reject unknown destination format', async () => {
await expect(async () => {
await diff({
oldSource: {
format: 'csv',
stream: './tests/a.csv',
},
newSource: {
format: 'csv',
stream: './tests/b.csv',
},
keys: ['id'],
}).to({
destination: {
format: <any>'foo',
stream: 'console',
},
});
}).rejects.toThrowError(`Unknown destination format 'foo'`);
});
test('should detect invalid ordering in ascending mode', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'3,dave,44',
'2,rachel,22',
],
keys: ['ID'],
})).rejects.toThrowError(`Expected rows to be ordered by \"ID ASC\" in new source but received:
previous=3,dave,44
current=2,rachel,22`);
});
test('should detect invalid ordering in descending mode', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
'3,dave,44',
'2,rachel,22',
'1,john,33',
],
newLines: [
'ID,NAME,AGE',
'3,dave,44',
'1,john,33',
'2,rachel,22',
],
keys: [{
name: 'ID',
order: 'DESC',
}],
})).rejects.toThrowError(new UnorderedStreamsError(`Expected rows to be ordered by "ID DESC" in new source but received:
previous=1,john,33
current=2,rachel,22`));
});
test('should detect primary key violation in old source', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
'3,dave bis,444',
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
})).rejects.toThrowError(new UniqueKeyViolationError(`Expected rows to be unique by "ID" in old source but received:
previous=3,dave,44
current=3,dave bis,444
Note that you can resolve this conflict automatically using the duplicateKeyHandling option.`));
});
test('should detect primary key violation in new source', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
'3,dave bis,444',
'4,noemie,11',
],
keys: ['ID'],
})).rejects.toThrowError(new UniqueKeyViolationError(`Expected rows to be unique by "ID" in new source but received:
previous=3,dave,44
current=3,dave bis,444
Note that you can resolve this conflict automatically using the duplicateKeyHandling option.`));
});
test('should detect duplicate keys and return the first row', async () => {
const writer = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
'3,dave bis,444',
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
duplicateKeyHandling: 'keepFirstRow',
keepSameRows: true,
});
expect(writer.diffs).toEqual([
{
delta: 0,
status: 'same',
oldRow: [ '1', 'john', '33' ],
newRow: [ '1', 'john', '33' ]
},
{
delta: 0,
status: 'same',
oldRow: [ '2', 'rachel', '22' ],
newRow: [ '2', 'rachel', '22' ]
},
{
delta: 0,
status: 'same',
oldRow: [ '3', 'dave', '44' ],
newRow: [ '3', 'dave', '44' ]
},
{ delta: -1, status: 'deleted', oldRow: [ '4', 'noemie', '11' ] }
]);
});
test('should detect duplicate keys and return the last row', async () => {
const writer = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
'3,dave bis,444',
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
duplicateKeyHandling: 'keepLastRow',
keepSameRows: true,
});
expect(writer.diffs).toEqual([
{
delta: 0,
status: 'same',
oldRow: [ '1', 'john', '33' ],
newRow: [ '1', 'john', '33' ]
},
{
delta: 0,
status: 'same',
oldRow: [ '2', 'rachel', '22' ],
newRow: [ '2', 'rachel', '22' ]
},
{
delta: 0,
status: 'modified',
oldRow: [ '3', 'dave bis', '444' ],
newRow: [ '3', 'dave', '44' ]
},
{ delta: -1, status: 'deleted', oldRow: [ '4', 'noemie', '11' ] }
]);
});
test('should detect duplicate keys and call aggregate function', async () => {
let duplicateRows: Row[] = [];
const duplicateKeyHandler: DuplicateKeyHandler = (rows) => {
if (duplicateRows.length === 0) {
duplicateRows = rows;
}
return rows[rows.length - 1];
};
const writer = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
'3,dave bis,444',
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
duplicateKeyHandling: duplicateKeyHandler,
keepSameRows: true,
});
expect(writer.diffs).toEqual([
{
delta: 0,
status: 'same',
oldRow: [ '1', 'john', '33' ],
newRow: [ '1', 'john', '33' ]
},
{
delta: 0,
status: 'same',
oldRow: [ '2', 'rachel', '22' ],
newRow: [ '2', 'rachel', '22' ]
},
{
delta: 0,
status: 'modified',
oldRow: [ '3', 'dave bis', '444' ],
newRow: [ '3', 'dave', '44' ]
},
{ delta: -1, status: 'deleted', oldRow: [ '4', 'noemie', '11' ] }
]);
expect(duplicateRows).toEqual([
[ '3', 'dave', '44' ],
[ '3', 'dave bis', '444' ]
]);
});
test('should detect duplicate keys and call aggregate function, with buffer overflow', async () => {
const dups = [];
for (let i = 0; i < 100; i++) {
dups.push(`3,dave bis${i},444`);
}
let duplicateRows: Row[] = [];
const duplicateKeyHandler: DuplicateKeyHandler = (rows) => {
if (duplicateRows.length === 0) {
duplicateRows = rows;
}
return rows[rows.length - 1];
};
const writer = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
...dups,
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
duplicateKeyHandling: duplicateKeyHandler,
duplicateRowBufferOverflow: true,
duplicateRowBufferSize: 10,
keepSameRows: true,
});
expect(writer.diffs).toEqual([
{
delta: 0,
status: 'same',
oldRow: [ '1', 'john', '33' ],
newRow: [ '1', 'john', '33' ]
},
{
delta: 0,
status: 'same',
oldRow: [ '2', 'rachel', '22' ],
newRow: [ '2', 'rachel', '22' ]
},
{
delta: 0,
status: 'modified',
oldRow: [ '3', 'dave bis99', '444' ],
newRow: [ '3', 'dave', '44' ]
},
{ delta: -1, status: 'deleted', oldRow: [ '4', 'noemie', '11' ] }
]);
expect(duplicateRows).toEqual([
[ '3', 'dave bis90', '444' ],
[ '3', 'dave bis91', '444' ],
[ '3', 'dave bis92', '444' ],
[ '3', 'dave bis93', '444' ],
[ '3', 'dave bis94', '444' ],
[ '3', 'dave bis95', '444' ],
[ '3', 'dave bis96', '444' ],
[ '3', 'dave bis97', '444' ],
[ '3', 'dave bis98', '444' ],
[ '3', 'dave bis99', '444' ]
]);
});
test('should detect duplicate keys and throw an error when the buffer exceeds the limit', async () => {
const dups = [];
for (let i = 0; i < 10; i++) {
dups.push('3,dave bis,444');
}
expect(diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
...dups,
'4,noemie,11',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
duplicateKeyHandling: (rows) => rows[0],
duplicateRowBufferSize: 5,
keepSameRows: true,
})).rejects.toThrowError('Too many duplicate rows');
});
test('should be able to execute twice', async () => {
const differ = diff({
oldSource: './tests/a.csv',
newSource: './tests/b.csv',
keys: ['id'],
});
const stats1 = await differ.to('./output/files/output.csv');
const output1= readAllText('./output/files/output.csv');
const stats2 = await differ.to('./output/files/output.csv');
const output2= readAllText('./output/files/output.csv');
expect(stats1.totalChanges).toBe(6);
expect(stats2).toEqual(stats1);
expect(output2).toEqual(output1);
});
test('should not open output file twice', async () => {
const f = new FileOutputStream('./output/files/output.csv');
await f.open();
try {
await expect(async () => await f.open()).rejects.toThrowError('file \"./output/files/output.csv\" is already open');
} finally {
await f.close();
}
});
test('should have columns in old source', async () => {
await expect(() => diffStrings({
oldLines: [
],
newLines: [
'ID,NAME,AGE',
],
keys: ['ID'],
})).rejects.toThrowError('Expected to find columns in old source');
});
test('should have columns in new source', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
],
newLines: [
],
keys: ['ID'],
})).rejects.toThrowError('Expected to find columns in new source');
});
test('should find keys in old columns', async () => {
await expect(() => diffStrings({
oldLines: [
'CODE,NAME,AGE',
],
newLines: [
'ID,NAME,AGE',
],
keys: ['ID'],
})).rejects.toThrowError(`Could not find key 'ID' in old stream`);
});
test('should find keys in new columns', async () => {
await expect(() => diffStrings({
oldLines: [
'ID,NAME,AGE',
'a1,a,33',
],
newLines: [
'CODE,NAME,AGE',
'a1,a,33',
],
keys: ['ID'],
})).rejects.toThrowError(`Could not find key 'ID' in new stream`);
});
test('should not allow calling diffs() twice', async () => {
const ctx = await diff({
oldSource: './tests/a.csv',
newSource: './tests/b.csv',
keys: ['id'],
}).start();
const diffs = [];
for await (const rowDiff of ctx.diffs()) {
diffs.push(rowDiff);
}
expect(diffs.length).toBe(11);
expect(ctx.isOpen).toBeFalsy();
await expect(async () => {
for await (const rowDiff of ctx.diffs()) {
}
}).rejects.toThrowError('Cannot get diffs on closed streams. You should call "Differ.start()" again.');
});
test('should not allow calling to() twice', async () => {
const ctx = await diff({
oldSource: './tests/a.csv',
newSource: './tests/b.csv',
keys: ['id'],
}).start();
expect(ctx.isOpen).toBeTruthy();
const stats = await ctx.to('null');
expect(stats.totalComparisons).toBe(11);
expect(ctx.isOpen).toBeFalsy();
await expect(async () => {
await ctx.to('null');
}).rejects.toThrowError('Cannot get diffs on closed streams. You should call "Differ.start()" again.');
});
test('should allow calling start() twice', async () => {
const differ = diff({
oldSource: './tests/a.csv',
newSource: './tests/b.csv',
keys: ['id'],
});
const ctx = await differ.start();
expect(ctx.isOpen).toBeTruthy();
const diffs = [];
for await (const rowDiff of ctx.diffs()) {
diffs.push(rowDiff);
}
expect(diffs.length).toBe(11);
expect(ctx.isOpen).toBeFalsy();
const ctx2 = await differ.start();
expect(ctx2.isOpen).toBeTruthy();
expect(ctx2).not.toBe(ctx);
const diffs2 = [];
for await (const rowDiff of ctx2.diffs()) {
diffs2.push(rowDiff);
}
expect(ctx2.isOpen).toBeFalsy();
expect(diffs2.length).toBe(11);
expect(diffs2).toEqual(diffs);
});
});
describe('changes', () => {
test('both files are empty', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
],
newLines: [
'ID,NAME,AGE',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 0,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 0,
changePercent: 0,
});
});
test('old is empty', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['1','john','33'],
},
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['2','rachel','22'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 2,
totalChanges: 2,
added: 2,
modified: 0,
deleted: 0,
same: 0,
changePercent: 100,
});
});
test('new is empty', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
],
newLines: [
'ID,NAME,AGE',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'deleted',
delta: -1,
oldRow: ['1','john','33'],
newRow: undefined,
},
{
status: 'deleted',
delta: -1,
oldRow: ['2','rachel','22'],
newRow: undefined,
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 2,
totalChanges: 2,
added: 0,
modified: 0,
deleted: 2,
same: 0,
changePercent: 100,
});
});
test('same and do not keep same rows', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('same and keep same rows', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
keepSameRows: true,
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'same',
delta: 0,
oldRow: ['1','john','33'],
newRow: ['1','john','33'],
},
{
status: 'same',
delta: 0,
oldRow: ['2','rachel','22'],
newRow: ['2','rachel','22'],
},
{
status: 'same',
delta: 0,
oldRow: ['3','dave','44'],
newRow: ['3','dave','44'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('same with reordered columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,AGE,NAME',
'1,33,john',
'2,22,rachel',
'3,44,dave',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'AGE', 'NAME']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('same with excluded columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,20',
'3,dave,44',
],
keys: ['ID'],
excludedColumns: ['AGE'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('same with included columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,20',
'3,dave,44',
],
keys: ['ID'],
includedColumns: ['ID', 'NAME'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('1 modified', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,20',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([{
status: 'modified',
delta: 0,
oldRow: ['2','rachel','22'],
newRow: ['2','rachel','20'],
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 0,
modified: 1,
deleted: 0,
same: 2,
changePercent: 33.33,
});
});
test('all modified', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,30',
'2,rachel,20',
'3,dave,40',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'modified',
delta: 0,
oldRow: ['1','john','30'],
newRow: ['1','john','33'],
},
{
status: 'modified',
delta: 0,
oldRow: ['2','rachel','20'],
newRow: ['2','rachel','22'],
},
{
status: 'modified',
delta: 0,
oldRow: ['3','dave','40'],
newRow: ['3','dave','44'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 3,
added: 0,
modified: 3,
deleted: 0,
same: 0,
changePercent: 100,
});
});
test('1 modified with reordered columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,AGE,NAME',
'1,33,john',
'2,20,rachel',
'3,44,dave',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'AGE', 'NAME']);
expect(res.diffs).toEqual([{
status: 'modified',
delta: 0,
oldRow: ['2','22','rachel'],
newRow: ['2','20','rachel'],
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 0,
modified: 1,
deleted: 0,
same: 2,
changePercent: 33.33,
});
});
test('1 modified with excluded columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rach,22',
'3,dave,44',
],
keys: ['ID'],
excludedColumns: ['AGE'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([{
status: 'modified',
delta: 0,
oldRow: ['2','rachel'],
newRow: ['2','rach'],
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 0,
modified: 1,
deleted: 0,
same: 2,
changePercent: 33.33,
});
});
test('1 added with excluded columns', async () => {
// this test will also help boost code coverage in normalizeOldRow
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
],
keys: ['ID'],
excludedColumns: ['AGE'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([
{ delta: 1, status: 'added', newRow: [ '2', 'rachel' ] }
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 2,
totalChanges: 1,
added: 1,
modified: 0,
deleted: 0,
same: 1,
changePercent: 50,
});
});
test('1 deleted with excluded columns', async () => {
// this test will also help boost code coverage in normalizeNewRow
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
],
keys: ['ID'],
excludedColumns: ['AGE'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([
{ delta: -1, status: 'deleted', oldRow: [ '2', 'rachel' ] }
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 2,
totalChanges: 1,
added: 0,
modified: 0,
deleted: 1,
same: 1,
changePercent: 50,
});
});
test('1 modified with included columns', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rach,22',
'3,dave,44',
],
keys: ['ID'],
includedColumns: ['ID', 'NAME'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME']);
expect(res.diffs).toEqual([{
status: 'modified',
delta: 0,
oldRow: ['2','rachel'],
newRow: ['2','rach'],
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 0,
modified: 1,
deleted: 0,
same: 2,
changePercent: 33.33,
});
});
test('No modification but adding a new column should force the rows to be modified', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE,NEW_COL',
'1,john,33,new1',
'2,rachel,22,new2',
'3,dave,44,new3',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE', 'NEW_COL']);
expect(res.diffs).toEqual([
{
delta: 0,
status: 'modified',
oldRow: [ '1', 'john', '33', '' ],
newRow: [ '1', 'john', '33', 'new1' ]
},
{
delta: 0,
status: 'modified',
oldRow: [ '2', 'rachel', '22', '' ],
newRow: [ '2', 'rachel', '22', 'new2' ]
},
{
delta: 0,
status: 'modified',
oldRow: [ '3', 'dave', '44', '' ],
newRow: [ '3', 'dave', '44', 'new3' ]
}
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 3,
added: 0,
modified: 3,
deleted: 0,
same: 0,
changePercent: 100,
});
});
test('No modification but removing an old column should be transparent', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE,REMOVED_COL',
'1,john,33,rem1',
'2,rachel,22,rem2',
'3,dave,44,rem3',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 0,
added: 0,
modified: 0,
deleted: 0,
same: 3,
changePercent: 0,
});
});
test('1 deleted', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([{
status: 'deleted',
delta: -1,
oldRow: ['2','rachel','22'],
newRow: undefined,
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 0,
modified: 0,
deleted: 1,
same: 2,
changePercent: 33.33,
});
});
test('1 added', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,20',
'3,dave,44',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['2','rachel','20'],
}]);
expect(res.footer?.stats).toEqual({
totalComparisons: 3,
totalChanges: 1,
added: 1,
modified: 0,
deleted: 0,
same: 2,
changePercent: 33.33,
});
});
test('only new rows and previous rows have been deleted', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'4,paula,11',
'5,jane,66',
],
keys: ['ID'],
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'deleted',
delta: -1,
oldRow: ['1','john','33'],
newRow: undefined,
},
{
status: 'deleted',
delta: -1,
oldRow: ['2','rachel','22'],
newRow: undefined,
},
{
status: 'deleted',
delta: -1,
oldRow: ['3','dave','44'],
newRow: undefined,
},
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['4','paula','11'],
},
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['5','jane','66'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 5,
totalChanges: 5,
added: 2,
modified: 0,
deleted: 3,
same: 0,
changePercent: 100,
});
});
test('same, modified, added and deleted', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,22',
'3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'1,john,33',
'2,rachel,20',
'4,paula,11',
],
keys: ['ID'],
keepSameRows: true,
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'same',
delta: 0,
oldRow: ['1','john','33'],
newRow: ['1','john','33'],
},
{
status: 'modified',
delta: 0,
oldRow: ['2','rachel','22'],
newRow: ['2','rachel','20'],
},
{
status: 'deleted',
delta: -1,
oldRow: ['3','dave','44'],
newRow: undefined,
},
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['4','paula','11'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 4,
totalChanges: 3,
added: 1,
modified: 1,
deleted: 1,
same: 1,
changePercent: 75,
});
});
test('same, modified, added and deleted with a case insensitive primary key', async () => {
const caseInsensitiveCompare = function(a: CellValue, b: CellValue): number {
if (typeof a === 'string' && typeof b === 'string') {
return stringComparer(a.toLowerCase(), b.toLowerCase());
}
return cellComparer(a, b);
}
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'a1,john,33',
'a2,rachel,22',
'a3,dave,44',
],
newLines: [
'ID,NAME,AGE',
'A1,john,33',
'A2,rachel,20',
'A4,paula,11',
],
keys: [
{
name: 'ID',
comparer: caseInsensitiveCompare,
}
],
keepSameRows: true,
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'same',
delta: 0,
oldRow: ['a1','john','33'],
newRow: ['A1','john','33'],
},
{
status: 'modified',
delta: 0,
oldRow: ['a2','rachel','22'],
newRow: ['A2','rachel','20'],
},
{
status: 'deleted',
delta: -1,
oldRow: ['a3','dave','44'],
newRow: undefined,
},
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['A4','paula','11'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 4,
totalChanges: 3,
added: 1,
modified: 1,
deleted: 1,
same: 1,
changePercent: 75,
});
});
test('same, modified, added and deleted, in descending order', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'3,dave,44',
'2,rachel,22',
'1,john,33',
],
newLines: [
'ID,NAME,AGE',
'4,paula,11',
'2,rachel,20',
'1,john,33',
],
keys: [{
name: 'ID',
order: 'DESC',
}],
keepSameRows: true,
});
expect(res.header?.columns).toEqual(['ID', 'NAME', 'AGE']);
expect(res.diffs).toEqual([
{
status: 'added',
delta: 1,
oldRow: undefined,
newRow: ['4','paula','11'],
},
{
status: 'deleted',
delta: -1,
oldRow: ['3','dave','44'],
newRow: undefined,
},
{
status: 'modified',
delta: 0,
oldRow: ['2','rachel','22'],
newRow: ['2','rachel','20'],
},
{
status: 'same',
delta: 0,
oldRow: ['1','john','33'],
newRow: ['1','john','33'],
},
]);
expect(res.footer?.stats).toEqual({
totalComparisons: 4,
totalChanges: 3,
added: 1,
modified: 1,
deleted: 1,
same: 1,
changePercent: 75,
});
});
test('same, modified, added and deleted, with a number primary key', async () => {
const res = await diffStrings({
oldLines: [
'ID,NAME,AGE',
'1,john,11',
'2,rachel,22',
'3,dave,33',
'11,john,111',
'12,rachel,122',
'13,dave,133',
'21,john,211',
'22,rachel,222',
'23,dave,233',
],
newLines: [
'ID,NAME,AGE',
'1,john,11',
'2,rachel,2',
'11,john,111',
'12,rachel,122',
'13,dave,133',
'