@decoder-leco/poc-eurostat-data-transformers
Version:
Decoder l'eco ELT module for eurostat population projection 2019/2024
74 lines (73 loc) • 3.12 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Transform = void 0;
const fs = __importStar(require("node:fs"));
const pl = __importStar(require("nodejs-polars"));
/**
* Transformation du DATASET depuis le repetoire des raw dataset
* & copie de la transformation dans un repertoire locale
*/
class Transform {
rawPath;
transformPath;
/**
* TRANSFORMATION DU DATASET DANS UN NOUVEAU FICHIER
*
* @param rawPath local path [ex: ./rawData/proj_19np.csv DO NOT FORGET ./ ATM]
* @param transformPath local path [ex: ./transformedData/proj_19np_transformed.csv DO NOT FORGET ./ ATM]
*/
constructor(rawPath, transformPath) {
this.rawPath = rawPath;
this.transformPath = transformPath;
this.rawPath = rawPath;
this.transformPath = transformPath;
}
async run() {
await this.handleDirs();
await this.transform();
}
async handleDirs() {
if (fs.existsSync(this.transformPath.split("/")[this.transformPath.substring(0, 2) == "./" ? 1 : 0]) == false)
fs.mkdirSync("./" + this.transformPath.split("/")[this.transformPath.substring(0, 2) == "./" ? 1 : 0]);
}
async transform() {
const data = fs.readFileSync(this.rawPath, { encoding: 'utf8' });
let df = pl.readCSV(data.replace(/\\/, ',').replace(/,/g, '\t'), { sep: "\t" }); // [c|t]sv to tsv
df = df.rename({ "TIME_PERIOD": "time" }).rename({ "OBS_FLAG": "population_proj" })
.select('DATAFLOW', 'LAST UPDATE', 'time', 'freq', 'unit', 'sex', 'projection', 'age', 'population_proj')
.filter((pl.col("projection").str.contains("BSL"))
.and(pl.col("sex").str.contains(/M|F/))).withColumn(
//pl.col("time").str.strptime(pl.Dtypes[pl.Datetime]).alias("newdate"),
pl.col('age').str.replace('Y_GE100', 'Y_OPEN'));
try {
df.writeCSV(this.transformPath);
}
catch (err) {
console.log("error: " + err);
}
}
}
exports.Transform = Transform;