lamed_io
Version:
454 lines (397 loc) • 13.8 kB
JavaScript
console.log(`Starting ${__filename}...`) // comment line to remove simple logging
// Purpose: The purpose of this....
// Date Created: 6/1/2018
// Created by : Perez Lamed van Niekerk
// ------------------------------------------------------
/* jshint esversion: 6 */
// ------------------------------------------------------
const _fs = require('fs')
require('ltype_string')
const _core = require('lamed_core')
const _isValid = require('valid-filename')
const _makeValid = require('filenamify')
const _LineByLineReader = require('line-by-line') // Reading of really big files
const _test = require('lamed_test')
const { con, notOk } = _test
con.useChalk(require('chalk'))
// con.traceSet(0)
const _config = { configFile: '' } // Default config structure
const _configName = 'config.json' // Default config name
/**
* Return true if the operating system is Windows
*/
function isWindows () {
if (notOk(_isWindows)) {
const _os = _core.getOS()
_isWindows = (_os === _core.enum_OS.Windows)
}
return _isWindows
}
let _isWindows
/**
* Read contents of a file
* @param file
* @returns {Promise<any>}
*/
function readFile (file, showLog = true) {
if (showLog) con.logGreen(` -->Reading: '${file}'...`)
return new Promise((resolve, reject) => {
if (_core.exist(file) === false) reject(new Error(`File: "${file}" does not exist.`))
_fs.readFile(file, function (err, data) {
if (err) reject(err)
if (data) resolve(data.toString('utf8'))
})
})
}
/**
* Read text to a file Synchronously
* @param {string} file - The file to read
*/
function readFileSync (file, showLog = true) {
if (showLog) con.logGreen(` -->Reading: '${file}'...`)
return _fs.readFileSync(file, 'utf8')
}
/**
* Convert a document into an array of lines
* @param {string} file - the context of the file
*/
function file_2Array(file) { // eslint-disable-line
file = file.replaceAll('\r', '')
return file.split('\n')
}
/**
* Read file synchronously into an array of lines
* @param {string} file - The file to read
* @param {boolean} showLog - If true then print file that is read on the console
* @returns {array} - Array of lines read
*/
function readFileSync_2Array (file, showLog = true) { // eslint-disable-line
const fileIn1 = readFileSync(file, showLog)
return file_2Array(fileIn1)
}
/**
* Write text to a file
* @param {string} file - The file to write
* @param {string} text - The text to write
* @param {boolean} showLog - If true then print file that is written to the console
*/
function writeFile (file, text, showLog = true) {
if (showLog) con.logGreen(` -->Writing: '${file}'...`)
if (file.includes('/con.')) throw new Error('Can not write file with name \'con\'\n')
return new Promise((resolve, reject) => {
_fs.writeFile(file, text, function (err) {
if (err) reject(err)
resolve()
})
})
}
/**
* Write text to a file synchronously
* @param {string} file - The file to write
* @param {string} text - The text to write
* @param {boolean} showLog - If true then print file that is written to the console
*/
function writeFileSync (file, text, showLog = true) {
if (showLog) con.logGreen(` -->Writing: '${file}'...`)
if (file.includes('/con.')) throw new Error('Can not write file with name \'con\'\n')
_fs.writeFileSync(file, text)
}
/**
* Write array to a file synchronously
* @param {string} file - The file to write
* @param {array} array - The array to write
* @param {boolean} showLog - If true then print file that is written to the console
*/
function writeFileSync_FromArray (file, array, showLog = true) { // eslint-disable-line
let text = ''
if (isWindows()) text = array.join('\r\n')
else text = array.join('\n')
writeFileSync(file, text, showLog)
}
/**
* Copy file synchronously
* @param {string} fromFile - The file to copy from
* @param {string} toFile - The file to copy to
*/
function copyFileSync (fromFile, toFile) {
// con.log({ fromFile, toFile })
_fs.copyFileSync(fromFile, toFile)
}
/**
* Test if the folder exist. If not create it
* @param {string} folder - The folder to create
*/
function mkdir (folder) {
if (_core.existFolder(folder) === false) _fs.mkdirSync(folder, { recursive: true })
}
function fileGetExt (file) {
const pos1 = file.indexOf('.')
if (pos1 === -1) return { file, ext: '' }
const ext = file.substr(pos1)
file = file.substr(0, pos1)
return { file, ext }
}
function fileSetStamp (file, timeStamp = false, now = new Date()) {
// let now = new Date()
// if (now === undefined) now = new Date()
const extObj = fileGetExt(file)
let stamp = ''
if (timeStamp) stamp = '_T_' + _core.Date_2TimeStr(now, '_') // time stamp
stamp = '_' + _core.Date_2Str(now, '_') + stamp // date stamp
return extObj.file + stamp + extObj.ext
}
/**
* Add ext to the file name if non exist else replace the extension with the new one
* @param {string} file - The filename
* @param {string} ext - The extension
*/
function fileSetExtension (file, ext = '') {
if (ext === '') return file
if (ext.includes('.') === false) throw new Error(`Extension '${ext}' must include '.'\n`)
const extObj = fileGetExt(file)
return extObj.file + ext
}
/**
* Return the file name part from a path
* @param {string} path - The path to evaluate
* @param {boolean} keepExtension - If true then keep the extension. Default is true
* @returns {string} - The file name
*/
function fileFromPath (path, keepExtension = true) {
path = path.replaceAll('\\', '/')
let file = path.split('/').pop()
if (keepExtension === false) {
// Remove extension
const parts = file.split('.')
parts.pop()
file = parts.join('.')
}
return file
}
/**
* It makes more sense to put 'rootfolder(), About() and Version()'
* calculation into 'lamed_io' and to include it from 'lamed_folder'
*/
let __rootFolder = ''
// To be called internally only
function rootFolderCalculate (dirname) {
if (notOk(dirname)) dirname = __dirname
__rootFolder = dirname.replaceAll('\\', '/')
// remove '/src', '/test' '/node_modules' from the path
const pos1 = __rootFolder.indexOf('/src')
if (pos1 > 0) __rootFolder = __rootFolder.substring(0, pos1 + 1)
const pos2 = __rootFolder.indexOf('/test')
if (pos2 > 0) __rootFolder = __rootFolder.substring(0, pos2 + 1)
const pos = __rootFolder.indexOf('/node_modules')
if (pos > 0) __rootFolder = __rootFolder.substring(0, pos + 1)
const pos3 = __rootFolder.indexOf('/app')
if (pos3 > 0) __rootFolder = __rootFolder.substring(0, pos3 + 1)
// con.log({ __rootFolder })
return __rootFolder
}
rootFolderCalculate()
/*
Return the root folder for the project
Return the root folder
All '\\' is replaced with '/' as this works on windows also and is consistent
*/
function rootFolder (dirname = '') {
if (dirname !== '') return rootFolderCalculate(dirname)
return __rootFolder
}
/**
* Return the path to the package.json file
* @returns {string}
*/
function packageJson_Path (dirname = '') { // eslint-disable-line
if (dirname !== '') rootFolderCalculate(dirname)
return `${__rootFolder}package.json`
}
/**
* Return the package.json file
* @returns {string}
*/
function packageJson (dirname = '') {
if (_packageJson === '') _packageJson = require(packageJson_Path(dirname))
return _packageJson
}
let _packageJson = ''
/* Get the application version */
function Version (dirname = '', showVersion = true) {
const pack = packageJson(dirname)
const version = pack.version
if (showVersion) con.log(`Version: ${version}`)
return version
}
/* return application about message */
function About (dirname = '', showAbout = true) {
const pack = packageJson(dirname)
const about = pack.name + ' (' + Version('', false) + ')'
if (showAbout) {
con.clear()
con.log(about)
}
return about
}
/**
* Change all '\\' to '/' for the folder
* @param {string} folder - The folder
* @param {bool} includeFinalSlash - If true ensure folder ends with '/'. Default is true
*/
function folderFix (folder, _dirname, includeFinalSlash = true) {
// This is good match with lamed_io --> export to lamed_folder
if (notOk(folder)) folder = _dirname
if (notOk(folder)) folder = __dirname
folder = folder.replaceAll('\\', '/')
if (includeFinalSlash) {
if (folder[folder.length - 1] !== '/') folder += '/'
} else {
if (folder[folder.length - 1] === '/') folder = folder.substring(0, folder.length - 1)
}
return folder
}
function memoryBytesAsMB (bytes) {
const result = Math.round(bytes * 100 / 1024 / 1024) / 100
return result
}
function memoryUsage (doGarbageCollection = false) {
if (doGarbageCollection) global.gc()
const usage = process.memoryUsage() // { rss, heapTotal, heapUsed }
// con.log({ usage })
const rss = memoryBytesAsMB(usage.rss) // Resident Set Size, it is the total memory allocated for the process execution
const heapTotal = memoryBytesAsMB(usage.heapTotal) // total size of the allocated heap
const heapUsed = memoryBytesAsMB(usage.heapUsed) // actual memory used during the execution of our process
return { rss, heapTotal, heapUsed }
}
function logMemoryUsage (tag) {
const mem = memoryUsage()
con.log(` -->${tag}: rss: ${mem.rss}MB; heap total: ${mem.heapTotal}MB; heap used: ${mem.heapUsed}MB`)
}
/**
* Read contents of a big file line per line
* @param {string} file - The file to read
* @param {func} callback - The callback function(lines). Return false to stop reading
* @param {int} buffer - Amount of lines to read before calling the callback. Default is 100.
* @param {int} maxLines - The maximum lines to read from file. This is a default safety of 100k lines. Make -1 to read till end of file
* @param {bool} showLog - If true then show log feedback
* @returns {void} - Feedback is given with callback function
*/
function readBigFile (file, callback, buffer = 100, maxLines = 100000, showLog = true) {
if (showLog) con.logGreen(` -->READING: '${file}'...`)
let lineNo = 0
const lines = []
let stop = false
const stream = new _LineByLineReader(file)
stream.on('error', function (err) { throw err })
stream.on('line', function (line) {
stream.pause() // pause emitting of lines...
lineNo++
lines.push(line)
if (lineNo % buffer === 0) {
stop = !callback(lines)
if (showLog) logMemoryUsage(lineNo)
}
if (stop) stream.close()
if (lineNo > maxLines) {
stream.close()
con.logRed(`Stream exceeds max lines of ${maxLines}! (change maxLines = -1 to read till the end)`)
}
stream.resume()
})
stream.on('end', function () {
con.log(`------END READING: ${file} ]-----------`)
})
}
/**
* Read first number of lines from a big file
* @param {string} file - The file to read
* @param {int} totalLines - Total lines to read. Default is 100
* @param {bool} showLog - If true, show progress log
*/
function readBigFileLines (file, totalLines = 100, showLog = true) {
return new Promise((resolve, reject) => {
try {
readBigFile(file, (lines) => {
resolve(lines)
}, totalLines)
} catch (e) {
reject(e)
}
})
}
/**
* Read _jsonName
* @param {string} dirname - The folder to read. Default is current folder
* @param {string} configName - The name of the config file. Default is 'config.json'
* @param {string} newConfig - The default structure of a new config file. Default is { configFile: '' }
* @returns {object} - csv config object { configFile }
*/
function configRead (dirname, configName, newConfig) {
if (notOk(configName)) configName = _configName
const folder = folderFix(dirname, __dirname)
const configFile = folder + configName
if (_core.existFile(configFile, false) === false) {
// config does not exist -> Create it
if (notOk(newConfig)) newConfig = _config
newConfig.configFile = configFile
writeFileSync(configFile, JSON.stringify(newConfig, null, 2))
}
const json = readFileSync(configFile)
const config = JSON.parse(json)
config.configFile = configFile
return config
}
/**
* Write the config to file
* @param {object} config - The config object
*/
function configWrite (config) {
if (notOk(config)) {
throw new Error('Config can not be null!\n')
}
// Get file path from the .configFile setting
const file = config.configFile
if (notOk(file)) throw new Error('config.configFile not defined!\n')
const json = JSON.stringify(config, null, 2)
writeFileSync(file, json)
}
// Exports --------------------------
const _store = require('./localstorage')
module.exports = {
// Check Functions
exist: _core.exist,
existFile: _core.existFile,
existFolder: _core.existFolder,
isRootFolder: _core.isRootFolder,
isValid: _isValid,
isWindows,
// Read & Write Functions
readFile,
readFileSync,
readFileSync_2Array,
readBigFile,
readBigFileLines,
configRead,
configWrite,
file_2Array,
writeFile,
writeFileSync,
writeFileSync_FromArray,
LocalStorage: _store.LocalStorage,
// File Functions
fileGetExt,
fileSetExtension,
fileSetStamp,
fileFromPath,
packageJson,
makeValid: _makeValid,
// Folder Functions
folderFix,
rootFolder,
// rootFolderCalculate,
packageJson_Path,
mkdir,
copyFileSync,
Version,
About
}