als-store/lib/store.js

Library for streamlined file management and advanced data caching, featuring intelligent file searching, dynamic cache control, and flexible file operations

const { remove, pathExists, rename, ensureDirSync } = require('fs-extra')
const { readdir } = require('als-readdir')
const promises = require('als-promises')
const { join } = require('path')
const File = require('./file')
const sortFn = require('./sort')
const { isFunction, number, optional, string, validateFn } = require('als-schema')
const Schema = require('als-schema')
const { paramsToName } = require('./name-params')
const defined = v => v !== null && v !== undefined && v !== ''
class Store {
   static limit = 10000;
   static paralelPromises = 100;
   constructor(params = {}) {
      const { dirPath, schema, maxAge, maxActiveAge } = params
      this.dirPath = string(3, 255)(dirPath)
      ensureDirSync(dirPath)
      this.schema = typeof schema === 'object' ? new Schema(schema) : null
      this.maxAge = validateFn([optional, number(1000)])(maxAge)
      this.maxActiveAge = validateFn([optional, number(1000)])(maxActiveAge)
      this.#reset()
      this.toRemove = []
   }

   #reset() {
      this.results = []
      this.errors = []
      this.options = { filters: [] }
      this.n = 0
      this.skiped = 0
      return this
   }

   async get(defValues=false) {
      const { paralelPromises } = Store
      const { level = Infinity, values=defValues, filters = [], limit, skip, dir = '', sort, desc = false } = this.options
      const dirPath = dir ? join(this.dirPath, dir) : this.dirPath
      this.#reset()
      const cutAfter = filters.length > 0 || sort !== undefined
      const params = cutAfter ? {} : { limit, skip }
      await this.#fileList(dirPath, level, params)
      this.#outDated()
      await promises(this.results.map(async (file) => await file.getStats()), paralelPromises, [], this.errors)
      if (filters.length) this.#filter(filters)
      if (sort) this.results = sortFn(this.results, sort, desc)
      if (cutAfter) this.#cut(limit, skip)
      if (values) await promises(this.results.map(async (file) => await file.getValue()), paralelPromises, [], this.errors)
      this.#removeOutdated()
      return this
   }

   async #fileList(dirPath, level, { skip = 0, limit = Infinity }) {
      if (this.n >= limit) return
      const files = await readdir(dirPath, { withFileTypes: true })
      for (const file of files) {
         if (this.n >= limit) break
         const { path, name } = file
         if (file.isFile()) {
            if (this.skiped < skip) { this.skiped++; continue; }
            const fileObj = new File(path, name, this.schema)
            this.results.push(fileObj)
            this.n++
         } else if (file.isDirectory()) {
            if (level > 0) await this.#fileList(join(dirPath, name), level - 1, { skip, limit })
         }
      }
   }

   async first(value) {
      this.options.limit = 1
      await this.get(value)
      return this.results[0] || null
   }

   file(path,value=false) {
      string(3)(path)
      const parts = path.split('/').filter(Boolean)
      const $name = parts.pop()
      const dir = parts.join('/')
      return this.dir(dir).filter(({name}) => $name === name).first(value)
   }


   skip(skip) { if(defined(skip)) this.options.skip = number(0)(skip); return this }
   limit(limit) { if(defined(limit)) this.options.limit = number(0)(limit); return this }
   level(level) { if(defined(level)) this.options.level = number(0)(level); return this }
   dir(dir) { if(defined(dir)) this.options.dir = string(1, 255)(dir); return this }
   filter(...fns) {
      if(fns)
      fns.forEach(fn => { this.options.filters.push(isFunction()(fn)) });
      return this
   }
   sort(sort) { if(defined(sort)) this.options.sort = string(1, 255)(sort); return this }
   desc() { this.options.desc = true; return this }
   values() { this.options.values = true; return this }
   create(name = '', value = '') {
      const parts = name.split('/').filter(Boolean)
      name = parts.pop()
      const addToDir = parts.join('/')
      const dir = this.options.dir
      const path = dir ? join(this.dirPath, dir, addToDir) : join(this.dirPath,addToDir)
      // const path = dir ? join(this.dirPath, dir) : this.dirPath
      this.#reset()
      if(this.schema && typeof name === 'object') name = paramsToName(name)
      const file = new File(path, string(0, 255)(name), this.schema)
      if (value) file.value = value
      return file
   }

   async remove() {
      const { level, filters = [], limit, skip, dir } = this.options
      if (level !== undefined || filters.length || limit || skip || !dir) {
         await this.get()
         await promises(this.results.map(async file => await file.remove()), Store.paralelPromises)
      } else if (dir) await this.#removeDir(dir)
   }

   async #removeDir(dirName) {
      const dirPath = join(this.dirPath, dirName)
      if (await pathExists(dirPath) === false) return
      await remove(dirPath)
      File.removeDirFromCache(dirPath)
      this.#reset()
   }

   async renameDir(dirName, newDirname) {
      const { dir } = this.options
      const baseDir = dir ? join(this.dirPath, dir) : this.dirPath
      const dirPath = join(baseDir, dirName)
      const newDirPath = join(baseDir, newDirname)
      if (await pathExists(dirPath) === false) return
      if (await pathExists(newDirPath)) return
      await rename(dirPath, newDirPath)
      File.renameDirInCache(baseDir, dirName, newDirname)
   }

   #removeOutdated() {
      const toRemove = this.toRemove.splice(0, 50) // remove each time 50 files
      try { promises(toRemove.map(async (file) => await file.remove())) } catch (error) { }
   }

   #outDated() {
      const { maxAge, maxActiveAge } = this
      if (!maxAge && !maxActiveAge) return
      const now = Date.now()
      this.results = this.results.filter(file => {
         const { birthtimeMs, atimeMs, mtimeMs } = file.stats
         const outDated = maxAge && (now - birthtimeMs) > maxAge
         const outActiveDated = maxActiveAge && ((now - atimeMs) > maxActiveAge || (now - mtimeMs) > maxActiveAge)
         if (outDated || outActiveDated) {
            this.toRemove.push(file)
            return false
         }
         return true
      })
   }

   #filter(filters) {
      filters.forEach(fn => {
         this.results = this.results.filter(file => fn(file))
      })
   }

   #cut(limit = Store.limit, skip = 0) {
      skip = Math.min(skip, this.results.length);
      limit = Math.min(limit, this.results.length - skip);
      this.results = this.results.slice(skip, skip + limit);
   }
}
module.exports = Store