@meta-sql/lineage
Version:
Library for processing lineage from SQL
1 lines • 23 kB
Source Map (JSON)
{"version":3,"sources":["../src/hashset.ts","../src/index.ts"],"names":["HashSet","_HashSet","hasher","value","callbackfn","thisArg","other","intersection","MASKING_AGG_FUNCTIONS","MASKING_FUNCTIONS","DIRECT_TRANSFORMATION","DIRECT_IDENTITY","DIRECT_AGGREGATION","mergeTransformations","parent","child","leading","TransformationSet","values","isColumn","selectColumn","formatInputColumnName","column","getInputColumnName","parseInputColumnName","parts","name","table","getOutputColumnName","getDirectTransformationsFromExprValue","expr","parentTransformation","inputColumnName","left","right","merged","key","prev","aggExpr","funcExpr","acc","arg","argTransformations","getTableExpressionsFromSelect","select","regularTables","selectTables","previousWiths","withByNames","withItem","s","item","matchingWith","mergeTransformationSet","tp","tc","getColumnLineage","schema","transformations","transformationsByColumns","columnName","childTransformations","inputFields","inputColumn","t","c","selectTable","matchingColumn","nextColumn","getLineage","unkownCount","outputFieldName"],"mappings":"AAAO,IAAMA,CAAAA,CAAN,MAAMC,CAA6B,CAChC,GAAA,CAAsB,IAAI,GAAA,CAC1B,MAAA,CAER,WAAA,CAAYC,CAAAA,CAA+B,CACzC,IAAA,CAAK,MAAA,CAASA,CAAAA,GAAYC,CAAAA,EAAa,IAAA,CAAK,SAAA,CAAUA,CAAK,CAAA,EAC7D,CACA,OAAA,CACEC,CAAAA,CACAC,CAAAA,CACM,CACN,IAAA,CAAK,GAAA,CAAI,OAAA,CAASF,CAAAA,EAAU,CAC1BC,CAAAA,CAAW,IAAA,CAAKC,CAAAA,CAASF,CAAAA,CAAOA,CAAAA,CAAO,IAAI,EAC7C,CAAC,EACH,CAEA,OAAA,EAA+B,CAC7B,OAAO,IAAA,CAAK,GAAA,CAAI,OAAA,EAClB,CAEA,IAAA,EAAuB,CACrB,OAAO,IAAA,CAAK,GAAA,CAAI,IAAA,EAClB,CAEA,MAAA,EAAyB,CACvB,OAAO,IAAA,CAAK,GAAA,CAAI,MAAA,EAClB,CAEA,CAAC,MAAA,CAAO,QAAQ,CAAA,EAAoB,CAClC,OAAO,IAAA,CAAK,GAAA,CAAI,MAAA,EAClB,CAEA,CAAC,MAAA,CAAO,WAAW,EAAY,SAAA,CAE/B,IAAI,IAAA,EAAe,CACjB,OAAO,IAAA,CAAK,GAAA,CAAI,IAClB,CAEA,GAAA,CAAIA,CAAAA,CAAgB,CAClB,OAAA,IAAA,CAAK,GAAA,CAAI,GAAA,CAAI,IAAA,CAAK,MAAA,CAAOA,CAAK,CAAA,CAAGA,CAAK,CAAA,CAC/B,IACT,CAEA,MAAA,CAAOA,CAAAA,CAAmB,CACxB,OAAO,IAAA,CAAK,GAAA,CAAI,MAAA,CAAO,IAAA,CAAK,MAAA,CAAOA,CAAK,CAAC,CAC3C,CAEA,KAAA,EAAc,CACZ,OAAA,IAAA,CAAK,GAAA,CAAI,KAAA,EAAM,CACR,IACT,CAEA,GAAA,CAAIA,CAAAA,CAAmB,CACrB,OAAO,IAAA,CAAK,GAAA,CAAI,GAAA,CAAI,IAAA,CAAK,MAAA,CAAOA,CAAK,CAAC,CACxC,CAEA,YAAA,CAAaG,CAAAA,CAA2B,CACtC,IAAMC,CAAAA,CAAe,IAAIN,CAAAA,CAAW,IAAA,CAAK,MAAM,CAAA,CAC/C,IAAA,IAAWE,CAAAA,IAASG,CAAAA,CACd,IAAA,CAAK,GAAA,CAAIH,CAAK,CAAA,EAChBI,CAAAA,CAAa,GAAA,CAAIJ,CAAK,CAAA,CAG1B,OAAOI,CACT,CACF,CAAA,CC3CA,IAAMC,CAAAA,CAAwB,IAAI,GAAA,CAAI,CAAC,OAAO,CAAC,CAAA,CAEzCC,CAAAA,CAAoB,IAAI,GAAA,CAAI,CAChC,KAAA,CACA,MAAA,CACA,MAAA,CACA,QAAA,CACA,QAAA,CACA,SAAA,CACA,mBAAA,CACA,mBACF,CAAC,CAAA,CAEYC,CAAAA,CAAwC,CACnD,IAAA,CAAM,QAAA,CACN,OAAA,CAAS,gBAAA,CACT,OAAA,CAAS,KACX,CAAA,CAEaC,CAAAA,CAAkC,CAC7C,IAAA,CAAM,QAAA,CACN,OAAA,CAAS,UAAA,CACT,OAAA,CAAS,KACX,CAAA,CAEaC,CAAAA,CAAqC,CAChD,IAAA,CAAM,QAAA,CACN,OAAA,CAAS,aAAA,CACT,OAAA,CAAS,KACX,EAEA,SAASC,CAAAA,CACPC,CAAAA,CACAC,CAAAA,CACgB,CAChB,GAAI,CAACD,CAAAA,CACH,OAAOC,CAAAA,CAGT,GAAIA,CAAAA,CAAM,IAAA,GAAS,QAAA,EAAYD,CAAAA,CAAO,IAAA,GAAS,QAAA,CAC7C,MAAM,IAAI,KAAA,CAAM,4CAA4C,CAAA,CAG9D,IAAIE,CAAAA,CAIJ,OAAIF,CAAAA,CAAO,OAAA,GAAY,aAAA,CACrBE,CAAAA,CAAUF,CAAAA,CACDC,CAAAA,CAAM,OAAA,GAAY,aAAA,CAC3BC,CAAAA,CAAUD,CAAAA,CACDD,CAAAA,CAAO,OAAA,GAAY,gBAAA,CAC5BE,CAAAA,CAAUF,CAAAA,CAEVE,CAAAA,CAAUD,CAAAA,CAGL,CAAE,GAAGC,CAAAA,CAAS,OAAA,CAASF,CAAAA,CAAO,OAAA,EAAWC,CAAAA,CAAM,OAAQ,CAChE,CAEA,IAAME,CAAAA,CAAN,cAAgCjB,CAAwB,CACtD,WAAA,CAAYkB,CAAAA,CAAoC,CAC9C,KAAA,CACGf,CAAAA,EACC,CAAA,EAAGA,CAAAA,CAAM,IAAI,CAAA,CAAA,EAAIA,CAAAA,CAAM,OAAO,CAAA,CAAA,EAAIA,CAAAA,CAAM,OAAA,CAAU,QAAA,CAAW,UAAU,CAAA,CAC3E,CAAA,CAEIe,CAAAA,EACFA,CAAAA,CAAO,OAAA,CAASf,CAAAA,EAAU,IAAA,CAAK,GAAA,CAAIA,CAAK,CAAC,EAE7C,CACF,CAAA,CAyBO,SAASgB,CAAAA,CACdC,CAAAA,CAC2B,CAC3B,OACE,OAAOA,CAAAA,EAAiB,QAAA,EACxBA,CAAAA,GAAiB,IAAA,EACjB,IAAA,GAAQA,CAAAA,EACR,MAAA,GAAUA,CAAAA,EACV,OAAOA,CAAAA,CAAa,IAAA,EAAS,QAAA,EAC7BA,CAAAA,CAAa,IAAA,GAAS,IAE1B,CAEO,SAASC,CAAAA,CAAsBC,CAAAA,CAA+B,CACnE,OAAO,CAAA,EAAGA,CAAAA,CAAO,KAAA,CAAQ,CAAA,EAAGA,CAAAA,CAAO,KAAK,CAAA,CAAA,CAAA,CAAM,EAAE,CAAA,EAAGC,CAAAA,CACjDD,CACF,CAAC,CAAA,CACH,CAEO,SAASE,CAAAA,CAAqBF,CAAAA,CAA6B,CAChE,IAAMG,CAAAA,CAAQH,CAAAA,CAAO,KAAA,CAAM,GAAG,CAAA,CACxBI,CAAAA,CAAOD,CAAAA,CAAM,GAAA,EAAI,EAAK,EAAA,CACtBE,CAAAA,CAAQF,CAAAA,CAAM,MAAA,CAAS,CAAA,CAAIA,CAAAA,CAAM,IAAA,CAAK,GAAG,CAAA,CAAI,MAAA,CAEnD,OAAO,CAAE,IAAA,CAAAC,CAAAA,CAAM,KAAA,CAAAC,CAAM,CACvB,CAEO,SAASJ,CAAAA,CAAmBD,CAAAA,CAAsC,CACvE,OAAO,OAAOA,CAAAA,CAAO,MAAA,EAAW,QAAA,CAC5BA,CAAAA,CAAO,MAAA,CACP,OAAOA,CAAAA,CAAO,MAAA,CAAO,IAAA,CAAK,KAAA,EAAU,QAAA,CAClCA,CAAAA,CAAO,MAAA,CAAO,IAAA,CAAK,KAAA,CACnB,IACR,CAEO,SAASM,CAAAA,CAAoBN,CAAAA,CAAkC,CACpE,OAAIA,CAAAA,CAAO,EAAA,CACF,OAAOA,CAAAA,CAAO,EAAA,EAAO,QAAA,CAAWA,CAAAA,CAAO,EAAA,CAAKA,CAAAA,CAAO,EAAA,CAAG,KAAA,CACpDA,CAAAA,CAAO,IAAA,CAAK,IAAA,GAAS,YAAA,CACvBC,CAAAA,CAAmBD,CAAAA,CAAO,IAAqB,CAAA,CAGjD,IACT,CAEO,SAASO,CAAAA,CACdC,CAAAA,CACAC,CAAAA,CACmC,CACnC,OAAQD,CAAAA,CAAK,IAAA,EACX,KAAK,YAAA,CAAc,CACjB,IAAME,CAAAA,CAAkBX,CAAAA,CAAsBS,CAAqB,CAAA,CAEnE,OAAOE,CAAAA,CACH,CACE,CAACA,CAAe,EAAG,IAAIf,CAAAA,CAAkB,CACvCJ,CAAAA,CAAqBkB,CAAAA,CAAsBpB,CAAe,CAC5D,CAAC,CACH,CAAA,CACA,EACN,CACA,KAAK,aAAA,CAAe,CAClB,GAAM,CAAE,IAAA,CAAAsB,CAAAA,CAAM,KAAA,CAAAC,CAAM,CAAA,CAAIJ,CAAAA,CAElBK,CAAAA,CAA4C,EAAC,CAEnD,OAAA,MAAA,CAAO,OAAA,CACLN,CAAAA,CACEI,CAAAA,CACApB,CAAAA,CAAqBkB,CAAAA,CAAsBrB,CAAqB,CAClE,CACF,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAC0B,CAAAA,CAAKjC,CAAK,CAAA,GAAM,CAC1BgC,CAAAA,CAAOC,CAAG,CAAA,CAAIjC,EAChB,CAAC,CAAA,CAED,MAAA,CAAO,OAAA,CACL0B,CAAAA,CACEK,CAAAA,CACArB,CAAAA,CAAqBkB,CAAAA,CAAsBrB,CAAqB,CAClE,CACF,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAC0B,CAAAA,CAAKjC,CAAK,CAAA,GAAM,CAC1B,IAAMkC,CAAAA,CAAOF,EAAOC,CAAG,CAAA,CAEnBC,CAAAA,CACFF,CAAAA,CAAOC,CAAG,CAAA,CAAIC,CAAAA,CAAK,YAAA,CAAalC,CAAK,CAAA,CAErCgC,CAAAA,CAAOC,CAAG,CAAA,CAAIjC,EAElB,CAAC,CAAA,CAEMgC,CACT,CACA,KAAK,WAAA,CAAa,CAChB,IAAMG,CAAAA,CAAUR,CAAAA,CAEhB,OAAOD,CAAAA,CACLS,CAAAA,CAAQ,IAAA,CAAK,IAAA,CACbzB,CAAAA,CAAqBkB,CAAAA,CAAsB,CACzC,GAAGnB,CAAAA,CACH,OAAA,CAASJ,CAAAA,CAAsB,GAAA,CAAI8B,CAAAA,CAAQ,IAAI,CACjD,CAAC,CACH,CACF,CACA,KAAK,UAAA,CAAY,CACf,IAAMC,CAAAA,CAAWT,CAAAA,CAEjB,OACES,CAAAA,CAAS,IAAA,EAAM,KAAA,CAAM,MAAA,CACnB,CAACC,CAAAA,CAAKC,CAAAA,GAAQ,CACZ,IAAMC,CAAAA,CAAqBb,CAAAA,CACzBY,CAAAA,CACA5B,CAAAA,CAAqBkB,CAAAA,CAAsB,CACzC,GAAGrB,CAAAA,CACH,OAAA,CACE6B,CAAAA,CAAS,IAAA,CAAK,IAAA,CAAK,MAAA,CAAS,CAAA,EAC5B9B,CAAAA,CAAkB,GAAA,CAAI8B,CAAAA,CAAS,IAAA,CAAK,IAAA,CAAK,EAAA,CAAG,EAAE,CAAA,CAAG,KAAK,CAC1D,CAAC,CACH,CAAA,CAEA,OAAA,MAAA,CAAO,OAAA,CAAQG,CAAkB,CAAA,CAAE,OAAA,CAAQ,CAAC,CAACN,CAAAA,CAAKjC,CAAK,CAAA,GAAM,CAC3DqC,CAAAA,CAAIJ,CAAG,CAAA,CAAII,CAAAA,CAAIJ,CAAG,CAAA,CAAII,CAAAA,CAAIJ,CAAG,CAAA,CAAE,YAAA,CAAajC,CAAK,CAAA,CAAIA,EACvD,CAAC,CAAA,CAEMqC,CACT,CAAA,CACA,EACF,CAAA,EAAK,EAET,CACA,QACE,OAAO,EACX,CACF,CAEO,SAASG,CAAAA,CAA8BC,CAAAA,CAG5C,CACA,IAAMC,CAAAA,CAA4B,EAAC,CAC7BC,CAAAA,CAAkC,EAAC,CAEnCC,CAAAA,CAAwB,EAAC,CACzBC,CAAAA,CAA4C,IAAI,GAAA,CAEtD,OAAIJ,CAAAA,CAAO,IAAA,EACTA,CAAAA,CAAO,IAAA,CAAK,OAAA,CAASK,CAAAA,EAAa,CAChC,IAAMC,CAAAA,CAAID,CAAAA,CAAS,IAAA,CAAK,GAAA,EAAOA,CAAAA,CAAS,IAAA,CAExCD,CAAAA,CAAY,GAAA,CAAIC,CAAAA,CAAS,IAAA,CAAK,KAAA,CAAO,CACnC,GAAGC,CAAAA,CACH,EAAA,CAAID,CAAAA,CAAS,IAAA,CAAK,KAAA,CAClB,IAAA,CAAM,CAAC,GAAGF,CAAa,CACzB,CAAC,CAAA,CAEDA,CAAAA,CAAc,IAAA,CAAKE,CAAQ,EAC7B,CAAC,CAAA,CAGCL,CAAAA,CAAO,IAAA,EAAA,CACS,KAAA,CAAM,OAAA,CAAQA,CAAAA,CAAO,IAAI,CAAA,CAAIA,CAAAA,CAAO,IAAA,CAAO,CAACA,CAAAA,CAAO,IAAI,CAAA,EAE/D,OAAA,CAASO,CAAAA,EAAS,CAC1B,GAAI,OAAA,GAAWA,CAAAA,CAAM,CAEnB,IAAMC,CAAAA,CAAeJ,CAAAA,CAAY,GAAA,CAAIG,CAAAA,CAAK,KAAK,CAAA,CAE3CC,CAAAA,CACFN,CAAAA,CAAa,IAAA,CAAK,CAChB,GAAGM,CAAAA,CACH,EAAA,CAAID,CAAAA,CAAK,EAAA,EAAMC,CAAAA,CAAa,EAC9B,CAAC,CAAA,CAEDP,CAAAA,CAAc,IAAA,CAAKM,CAAI,EAE3B,CAAA,KAAW,MAAA,GAAUA,CAAAA,EACnBL,CAAAA,CAAa,IAAA,CAAK,CAChB,GAAGK,CAAAA,CAAK,IAAA,CAAK,GAAA,CACb,EAAA,CAAIA,CAAAA,CAAK,EAAA,CACT,IAAA,CAAMJ,CACR,CAAC,EAEL,CAAC,CAAA,CAGI,CAAE,aAAA,CAAAF,CAAAA,CAAe,YAAA,CAAAC,CAAa,CACvC,CAEO,SAASO,CAAAA,CACdvC,EACAC,CAAAA,CACmB,CACnB,IAAMoB,CAAAA,CAAS,IAAIlB,CAAAA,CAEnB,OAAAH,CAAAA,CAAO,OAAA,CAASwC,CAAAA,EAAO,CACrBvC,CAAAA,CAAM,OAAA,CAASwC,CAAAA,EAAO,CACpBpB,CAAAA,CAAO,GAAA,CAAItB,CAAAA,CAAqByC,CAAAA,CAAIC,CAAE,CAAC,EACzC,CAAC,EACH,CAAC,CAAA,CAEMpB,CACT,CAEO,SAASqB,CAAAA,CACdZ,CAAAA,CACAa,CAAAA,CACAnC,CAAAA,CACAoC,CAAAA,CACc,CACd,IAAIC,CAAAA,CAA2B9B,CAAAA,CAC7BP,CAAAA,CAAO,IACT,CAAA,CAEIoC,CAAAA,GACFC,CAAAA,CAA2B,MAAA,CAAO,OAAA,CAAQA,CAAwB,CAAA,CAAE,MAAA,CAClE,CAACnB,CAAAA,CAAK,CAACoB,CAAAA,CAAYC,CAAoB,CAAA,IACrCrB,CAAAA,CAAIoB,CAAU,CAAA,CAAIP,CAAAA,CAChBK,CAAAA,CACAG,CACF,CAAA,CAEOrB,CAAAA,CAAAA,CAET,EACF,CAAA,CAAA,CAGF,GAAM,CAAE,aAAA,CAAAK,CAAAA,CAAe,YAAA,CAAAC,CAAa,CAAA,CAAIH,CAAAA,CAA8BC,CAAM,CAAA,CAEtEkB,CAAAA,CAA4B,EAAC,CAEnC,IAAA,GAAW,CAAC9B,CAAAA,CAAiB0B,CAAe,CAAA,GAAK,MAAA,CAAO,OAAA,CACtDC,CACF,CAAA,CAAG,CACD,IAAMI,CAAAA,CAAcvC,CAAAA,CAAqBQ,CAAe,CAAA,CAElDL,CAAAA,CAAQkB,CAAAA,CAAc,IAAA,CACzBmB,CAAAA,EAAAA,CACE,CAACD,CAAAA,CAAY,KAAA,EACZA,CAAAA,CAAY,KAAA,GAAUC,CAAAA,CAAE,KAAA,EACxBD,CAAAA,CAAY,KAAA,GAAUC,CAAAA,CAAE,EAAA,GAC1BP,CAAAA,CAAO,MAAA,CAAO,IAAA,CACXP,CAAAA,EACCA,CAAAA,CAAE,IAAA,GAASc,CAAAA,CAAE,KAAA,EAASd,CAAAA,CAAE,OAAA,CAAQ,IAAA,CAAMe,CAAAA,EAAMA,CAAAA,GAAMF,CAAAA,CAAY,IAAI,CACtE,CACJ,CAAA,CAEA,GAAIpC,CAAAA,CACFmC,EAAY,IAAA,CAAK,CACf,SAAA,CAAWL,CAAAA,CAAO,SAAA,CAClB,IAAA,CAAM9B,CAAAA,CAAM,KAAA,CACZ,KAAA,CAAOoC,CAAAA,CAAY,IAAA,CACnB,eAAA,CAAiB,KAAA,CAAM,IAAA,CAAKL,CAAe,CAC7C,CAAC,CAAA,CAAA,KAED,IAAA,IAAWQ,CAAAA,IAAepB,CAAAA,CAAc,CACtC,GAAIiB,CAAAA,CAAY,KAAA,EAASA,CAAAA,CAAY,KAAA,GAAUG,CAAAA,CAAY,EAAA,CACzD,SAGF,IAAMC,CAAAA,CAAiBD,CAAAA,CAAY,OAAA,CAAQ,IAAA,CACxCD,CAAAA,EAAMrC,CAAAA,CAAoBqC,CAAC,CAAA,GAAMF,CAAAA,CAAY,IAChD,CAAA,CAEIK,CAAAA,CAEJ,GAAID,CAAAA,CACFC,CAAAA,CAAaD,CAAAA,CAAAA,KAAAA,GAEbC,CAAAA,CAAa9C,CAAAA,CAGT8C,CAAAA,CAAW,IAAA,CAAK,IAAA,GAAS,YAAA,CAAc,CACzC,IAAMtC,CAAAA,CAAOsC,CAAAA,CAAW,IAAA,CAExBtC,CAAAA,CAAK,KAAA,CAAQ,KACf,CAGFgC,CAAAA,CAAY,IAAA,CACV,GAAGN,CAAAA,CAAiBU,CAAAA,CAAaT,CAAAA,CAAQW,CAAAA,CAAYV,CAAe,CACtE,EACF,CAEJ,CAEA,OAAOI,CACT,CAEO,SAASO,CAAAA,CACdzB,CAAAA,CACAa,CAAAA,CACqC,CACrC,IAAIa,CAAAA,CAAc,CAAA,CAElB,OAAO1B,CAAAA,CAAO,OAAA,CAAQ,MAAA,CAAO,CAACJ,CAAAA,CAAKlB,CAAAA,GAAW,CAC5C,GAAI,CAACH,CAAAA,CAASG,CAAM,CAAA,CAClB,OAAOkB,CAAAA,CAGT,IAAI+B,CAAAA,CAAkB3C,CAAAA,CAAoBN,CAAM,CAAA,CAEhD,OAAKiD,CAAAA,GACHA,CAAAA,CAAkB,CAAA,QAAA,EAAWD,CAAAA,EAAa,CAAA,CAAA,CAAA,CAGrC,CACL,GAAG9B,CAAAA,CACH,CAAC+B,CAAe,EAAG,CACjB,WAAA,CAAaf,CAAAA,CAAiBZ,CAAAA,CAAQa,CAAAA,CAAQnC,CAAM,CACtD,CACF,CACF,CAAA,CAAG,EAAE,CACP","file":"index.mjs","sourcesContent":["export class HashSet<T> implements Set<T> {\n private map: Map<string, T> = new Map();\n private hasher: (value: T) => string;\n\n constructor(hasher?: (value: T) => string) {\n this.hasher = hasher || ((value: T) => JSON.stringify(value));\n }\n forEach(\n callbackfn: (value: T, value2: T, set: Set<T>) => void,\n thisArg?: unknown\n ): void {\n this.map.forEach((value) => {\n callbackfn.call(thisArg, value, value, this);\n });\n }\n\n entries(): SetIterator<[T, T]> {\n return this.map.entries() as SetIterator<[T, T]>;\n }\n\n keys(): SetIterator<T> {\n return this.map.keys() as SetIterator<T>;\n }\n\n values(): SetIterator<T> {\n return this.map.values() as SetIterator<T>;\n }\n\n [Symbol.iterator](): SetIterator<T> {\n return this.map.values() as SetIterator<T>;\n }\n\n [Symbol.toStringTag]: string = \"HashSet\";\n\n get size(): number {\n return this.map.size;\n }\n\n add(value: T): this {\n this.map.set(this.hasher(value), value);\n return this;\n }\n\n delete(value: T): boolean {\n return this.map.delete(this.hasher(value));\n }\n\n clear(): this {\n this.map.clear();\n return this;\n }\n\n has(value: T): boolean {\n return this.map.has(this.hasher(value));\n }\n\n intersection(other: Set<T>): HashSet<T> {\n const intersection = new HashSet<T>(this.hasher);\n for (const value of other) {\n if (this.has(value)) {\n intersection.add(value);\n }\n }\n return intersection;\n }\n}\n","import {\n type ColumnLineageDatasetFacet,\n type InputField,\n type Transformation as _Transformation,\n} from \"@meta-sql/open-lineage\";\nimport {\n Select,\n Column as AstColumn,\n ColumnRefItem,\n BaseFrom,\n Binary,\n ExpressionValue,\n AggrFunc,\n Function as AstFunction,\n With,\n} from \"node-sql-parser\";\nimport { HashSet } from \"./hashset\";\n\ntype Transformation = Exclude<_Transformation, \"masking\"> & {\n masking: boolean; // output boolean only for easier testing\n};\n\nconst MASKING_AGG_FUNCTIONS = new Set([\"COUNT\"]);\n\nconst MASKING_FUNCTIONS = new Set([\n \"MD5\",\n \"SHA1\",\n \"SHA2\",\n \"SHA256\",\n \"SHA512\",\n \"MURMUR3\",\n \"SPOOKY_HASH_V2_32\",\n \"SPOOKY_HASH_V2_64\",\n]);\n\nexport const DIRECT_TRANSFORMATION: Transformation = {\n type: \"DIRECT\",\n subtype: \"TRANSFORMATION\",\n masking: false,\n};\n\nexport const DIRECT_IDENTITY: Transformation = {\n type: \"DIRECT\",\n subtype: \"IDENTITY\",\n masking: false,\n};\n\nexport const DIRECT_AGGREGATION: Transformation = {\n type: \"DIRECT\",\n subtype: \"AGGREGATION\",\n masking: false,\n};\n\nfunction mergeTransformations(\n parent: Transformation | undefined,\n child: Transformation\n): Transformation {\n if (!parent) {\n return child;\n }\n\n if (child.type !== \"DIRECT\" || parent.type !== \"DIRECT\") {\n throw new Error(\"Indirect transformations not supported yet\");\n }\n\n let leading: Transformation;\n\n // agg > transformation > identity\n\n if (parent.subtype === \"AGGREGATION\") {\n leading = parent;\n } else if (child.subtype === \"AGGREGATION\") {\n leading = child;\n } else if (parent.subtype === \"TRANSFORMATION\") {\n leading = parent;\n } else {\n leading = child;\n }\n\n return { ...leading, masking: parent.masking || child.masking };\n}\n\nclass TransformationSet extends HashSet<Transformation> {\n constructor(values?: readonly Transformation[]) {\n super(\n (value: Transformation) =>\n `${value.type}-${value.subtype}-${value.masking ? \"MASKED\" : \"UNMASKED\"}`\n );\n\n if (values) {\n values.forEach((value) => this.add(value));\n }\n }\n}\n\nexport type Column = {\n name: string;\n};\n\nexport type Table = {\n name: string;\n columns: string[];\n};\n\nexport type Schema = {\n namespace: string;\n tables: Table[];\n};\n\nexport type InputColumn = {\n name: string;\n table?: string;\n};\n\nexport type SelectWithAlias = Select & {\n as?: string | null;\n};\n\nexport function isColumn(\n selectColumn: Select[\"columns\"][number]\n): selectColumn is AstColumn {\n return (\n typeof selectColumn === \"object\" &&\n selectColumn !== null &&\n \"as\" in selectColumn &&\n \"expr\" in selectColumn &&\n typeof selectColumn.expr === \"object\" &&\n selectColumn.expr !== null\n );\n}\n\nexport function formatInputColumnName(column: ColumnRefItem): string {\n return `${column.table ? `${column.table}.` : \"\"}${getInputColumnName(\n column\n )}`;\n}\n\nexport function parseInputColumnName(column: string): InputColumn {\n const parts = column.split(\".\");\n const name = parts.pop() || \"\";\n const table = parts.length > 0 ? parts.join(\".\") : undefined;\n\n return { name, table };\n}\n\nexport function getInputColumnName(column: ColumnRefItem): string | null {\n return typeof column.column === \"string\"\n ? column.column\n : typeof column.column.expr.value === \"string\"\n ? column.column.expr.value\n : null;\n}\n\nexport function getOutputColumnName(column: AstColumn): string | null {\n if (column.as) {\n return typeof column.as === \"string\" ? column.as : column.as.value;\n } else if (column.expr.type === \"column_ref\") {\n return getInputColumnName(column.expr as ColumnRefItem);\n }\n\n return null;\n}\n\nexport function getDirectTransformationsFromExprValue(\n expr: ExpressionValue,\n parentTransformation?: Transformation\n): Record<string, TransformationSet> {\n switch (expr.type) {\n case \"column_ref\": {\n const inputColumnName = formatInputColumnName(expr as ColumnRefItem);\n\n return inputColumnName\n ? {\n [inputColumnName]: new TransformationSet([\n mergeTransformations(parentTransformation, DIRECT_IDENTITY),\n ]),\n }\n : {};\n }\n case \"binary_expr\": {\n const { left, right } = expr as Binary;\n\n const merged: Record<string, TransformationSet> = {};\n\n Object.entries(\n getDirectTransformationsFromExprValue(\n left,\n mergeTransformations(parentTransformation, DIRECT_TRANSFORMATION)\n )\n ).forEach(([key, value]) => {\n merged[key] = value;\n });\n\n Object.entries(\n getDirectTransformationsFromExprValue(\n right,\n mergeTransformations(parentTransformation, DIRECT_TRANSFORMATION)\n )\n ).forEach(([key, value]) => {\n const prev = merged[key];\n\n if (prev) {\n merged[key] = prev.intersection(value);\n } else {\n merged[key] = value;\n }\n });\n\n return merged;\n }\n case \"aggr_func\": {\n const aggExpr = expr as AggrFunc;\n\n return getDirectTransformationsFromExprValue(\n aggExpr.args.expr,\n mergeTransformations(parentTransformation, {\n ...DIRECT_AGGREGATION,\n masking: MASKING_AGG_FUNCTIONS.has(aggExpr.name),\n })\n );\n }\n case \"function\": {\n const funcExpr = expr as AstFunction;\n\n return (\n funcExpr.args?.value.reduce(\n (acc, arg) => {\n const argTransformations = getDirectTransformationsFromExprValue(\n arg,\n mergeTransformations(parentTransformation, {\n ...DIRECT_TRANSFORMATION,\n masking:\n funcExpr.name.name.length > 0 &&\n MASKING_FUNCTIONS.has(funcExpr.name.name.at(-1)!.value),\n })\n );\n\n Object.entries(argTransformations).forEach(([key, value]) => {\n acc[key] = acc[key] ? acc[key].intersection(value) : value;\n });\n\n return acc;\n },\n {} as Record<string, TransformationSet>\n ) ?? {}\n );\n }\n default:\n return {};\n }\n}\n\nexport function getTableExpressionsFromSelect(select: Select): {\n regularTables: BaseFrom[];\n selectTables: SelectWithAlias[];\n} {\n const regularTables: BaseFrom[] = [];\n const selectTables: SelectWithAlias[] = [];\n\n const previousWiths: With[] = [];\n const withByNames: Map<string, SelectWithAlias> = new Map();\n\n if (select.with) {\n select.with.forEach((withItem) => {\n const s = withItem.stmt.ast ?? withItem.stmt;\n\n withByNames.set(withItem.name.value, {\n ...s,\n as: withItem.name.value,\n with: [...previousWiths], // keep previous with statements\n });\n\n previousWiths.push(withItem);\n });\n }\n\n if (select.from) {\n const fromItems = Array.isArray(select.from) ? select.from : [select.from];\n\n fromItems.forEach((item) => {\n if (\"table\" in item) {\n // might mention with statemnt in our select\n const matchingWith = withByNames.get(item.table);\n\n if (matchingWith) {\n selectTables.push({\n ...matchingWith,\n as: item.as ?? matchingWith.as,\n });\n } else {\n regularTables.push(item);\n }\n } else if (\"expr\" in item) {\n selectTables.push({\n ...item.expr.ast,\n as: item.as,\n with: previousWiths, // propagate previous withs\n });\n }\n });\n }\n\n return { regularTables, selectTables };\n}\n\nexport function mergeTransformationSet(\n parent: TransformationSet,\n child: TransformationSet\n): TransformationSet {\n const merged = new TransformationSet();\n\n parent.forEach((tp) => {\n child.forEach((tc) => {\n merged.add(mergeTransformations(tp, tc));\n });\n });\n\n return merged;\n}\n\nexport function getColumnLineage(\n select: Select,\n schema: Schema,\n column: AstColumn,\n transformations?: TransformationSet\n): InputField[] {\n let transformationsByColumns = getDirectTransformationsFromExprValue(\n column.expr\n );\n\n if (transformations) {\n transformationsByColumns = Object.entries(transformationsByColumns).reduce(\n (acc, [columnName, childTransformations]) => {\n acc[columnName] = mergeTransformationSet(\n transformations,\n childTransformations\n );\n\n return acc;\n },\n {} as Record<string, TransformationSet>\n );\n }\n\n const { regularTables, selectTables } = getTableExpressionsFromSelect(select);\n\n const inputFields: InputField[] = [];\n\n for (const [inputColumnName, transformations] of Object.entries(\n transformationsByColumns\n )) {\n const inputColumn = parseInputColumnName(inputColumnName);\n\n const table = regularTables.find(\n (t) =>\n (!inputColumn.table ||\n inputColumn.table === t.table ||\n inputColumn.table === t.as) &&\n schema.tables.some(\n (s) =>\n s.name === t.table && s.columns.some((c) => c === inputColumn.name)\n )\n );\n\n if (table) {\n inputFields.push({\n namespace: schema.namespace,\n name: table.table,\n field: inputColumn.name,\n transformations: Array.from(transformations),\n });\n } else {\n for (const selectTable of selectTables) {\n if (inputColumn.table && inputColumn.table !== selectTable.as) {\n continue;\n }\n\n const matchingColumn = selectTable.columns.find(\n (c) => getOutputColumnName(c) === inputColumn.name\n );\n\n let nextColumn: AstColumn;\n\n if (matchingColumn) {\n nextColumn = matchingColumn;\n } else {\n nextColumn = column;\n\n // stop propogating table of column as it is only in the context of the select\n if (nextColumn.expr.type === \"column_ref\") {\n const expr = nextColumn.expr as ColumnRefItem;\n\n expr.table = null;\n }\n }\n\n inputFields.push(\n ...getColumnLineage(selectTable, schema, nextColumn, transformations)\n );\n }\n }\n }\n\n return inputFields;\n}\n\nexport function getLineage(\n select: Select,\n schema: Schema\n): ColumnLineageDatasetFacet[\"fields\"] {\n let unkownCount = 0;\n\n return select.columns.reduce((acc, column) => {\n if (!isColumn(column)) {\n return acc;\n }\n\n let outputFieldName = getOutputColumnName(column);\n\n if (!outputFieldName) {\n outputFieldName = `unknown_${unkownCount++}`;\n }\n\n return {\n ...acc,\n [outputFieldName]: {\n inputFields: getColumnLineage(select, schema, column),\n },\n };\n }, {});\n}\n"]}