@meta-sql/lineage/dist/index.js

Library for processing lineage from SQL

'use strict';var h=class e{map=new Map;hasher;constructor(n){this.hasher=n||(t=>JSON.stringify(t));}forEach(n,t){this.map.forEach(r=>{n.call(t,r,r,this);});}entries(){return this.map.entries()}keys(){return this.map.keys()}values(){return this.map.values()}[Symbol.iterator](){return this.map.values()}[Symbol.toStringTag]="HashSet";get size(){return this.map.size}add(n){return this.map.set(this.hasher(n),n),this}delete(n){return this.map.delete(this.hasher(n))}clear(){return this.map.clear(),this}has(n){return this.map.has(this.hasher(n))}intersection(n){let t=new e(this.hasher);for(let r of n)this.has(r)&&t.add(r);return t}};var C=new Set(["COUNT"]),E=new Set(["MD5","SHA1","SHA2","SHA256","SHA512","MURMUR3","SPOOKY_HASH_V2_32","SPOOKY_HASH_V2_64"]),I={type:"DIRECT",subtype:"TRANSFORMATION",masking:false},R={type:"DIRECT",subtype:"IDENTITY",masking:false},N={type:"DIRECT",subtype:"AGGREGATION",masking:false};function f(e,n){if(!e)return n;if(n.type!=="DIRECT"||e.type!=="DIRECT")throw new Error("Indirect transformations not supported yet");let t;return e.subtype==="AGGREGATION"?t=e:n.subtype==="AGGREGATION"?t=n:e.subtype==="TRANSFORMATION"?t=e:t=n,{...t,masking:e.masking||n.masking}}var S=class extends h{constructor(n){super(t=>`${t.type}-${t.subtype}-${t.masking?"MASKED":"UNMASKED"}`),n&&n.forEach(t=>this.add(t));}};function O(e){return typeof e=="object"&&e!==null&&"as"in e&&"expr"in e&&typeof e.expr=="object"&&e.expr!==null}function F(e){return `${e.table?`${e.table}.`:""}${A(e)}`}function _(e){let n=e.split("."),t=n.pop()||"",r=n.length>0?n.join("."):void 0;return {name:t,table:r}}function A(e){return typeof e.column=="string"?e.column:typeof e.column.expr.value=="string"?e.column.expr.value:null}function x(e){return e.as?typeof e.as=="string"?e.as:e.as.value:e.expr.type==="column_ref"?A(e.expr):null}function c(e,n){switch(e.type){case "column_ref":{let t=F(e);return t?{[t]:new S([f(n,R)])}:{}}case "binary_expr":{let{left:t,right:r}=e,s={};return Object.entries(c(t,f(n,I))).forEach(([a,o])=>{s[a]=o;}),Object.entries(c(r,f(n,I))).forEach(([a,o])=>{let i=s[a];i?s[a]=i.intersection(o):s[a]=o;}),s}case "aggr_func":{let t=e;return c(t.args.expr,f(n,{...N,masking:C.has(t.name)}))}case "function":{let t=e;return t.args?.value.reduce((r,s)=>{let a=c(s,f(n,{...I,masking:t.name.name.length>0&&E.has(t.name.name.at(-1).value)}));return Object.entries(a).forEach(([o,i])=>{r[o]=r[o]?r[o].intersection(i):i;}),r},{})??{}}default:return {}}}function D(e){let n=[],t=[],r=[],s=new Map;return e.with&&e.with.forEach(a=>{let o=a.stmt.ast??a.stmt;s.set(a.name.value,{...o,as:a.name.value,with:[...r]}),r.push(a);}),e.from&&(Array.isArray(e.from)?e.from:[e.from]).forEach(o=>{if("table"in o){let i=s.get(o.table);i?t.push({...i,as:o.as??i.as}):n.push(o);}else "expr"in o&&t.push({...o.expr.ast,as:o.as,with:r});}),{regularTables:n,selectTables:t}}function G(e,n){let t=new S;return e.forEach(r=>{n.forEach(s=>{t.add(f(r,s));});}),t}function d(e,n,t,r){let s=c(t.expr);r&&(s=Object.entries(s).reduce((T,[g,u])=>(T[g]=G(r,u),T),{}));let{regularTables:a,selectTables:o}=D(e),i=[];for(let[T,g]of Object.entries(s)){let u=_(T),y=a.find(l=>(!u.table||u.table===l.table||u.table===l.as)&&n.tables.some(p=>p.name===l.table&&p.columns.some(m=>m===u.name)));if(y)i.push({namespace:n.namespace,name:y.table,field:u.name,transformations:Array.from(g)});else for(let l of o){if(u.table&&u.table!==l.as)continue;let p=l.columns.find(b=>x(b)===u.name),m;if(p)m=p;else if(m=t,m.expr.type==="column_ref"){let b=m.expr;b.table=null;}i.push(...d(l,n,m,g));}}return i}function M(e,n){let t=0;return e.columns.reduce((r,s)=>{if(!O(s))return r;let a=x(s);return a||(a=`unknown_${t++}`),{...r,[a]:{inputFields:d(e,n,s)}}},{})}exports.DIRECT_AGGREGATION=N;exports.DIRECT_IDENTITY=R;exports.DIRECT_TRANSFORMATION=I;exports.formatInputColumnName=F;exports.getColumnLineage=d;exports.getDirectTransformationsFromExprValue=c;exports.getInputColumnName=A;exports.getLineage=M;exports.getOutputColumnName=x;exports.getTableExpressionsFromSelect=D;exports.isColumn=O;exports.mergeTransformationSet=G;exports.parseInputColumnName=_;//# sourceMappingURL=index.js.map
//# sourceMappingURL=index.js.map