@silver-zepp/easy-storage/dist/easy-tsdb.js

The Easy Storage library offers a suite of storage solutions designed for ZeppOS applications, providing a range of data persistence strategies to accommodate different data management needs.

/** @about Easy Storage @min_zeppos 2.0 @author: Silver, Zepp Health. @license: MIT */
import{debugLog}from"./core/core";import{Storage}from"./storage";export class EasyTSDB{#data_in_ram={};#query_cache={};#index={};#cur_index_checksum="";#has_pending_writes=false;#db_cleared=false;#autosave_timeout_id=null;#defaults={directory:"easy_timeseries_db",time_frame:"hour",max_ram:.2*1024*1024,autosave_interval:600};#user_options;constructor(options={}){this.#user_options={...this.#defaults,...options};this.#setupDirectoryStructure();this.#loadIndex()}writePoint(measurement,value,timestamp=Date.now()){const date=new Date(timestamp);const year=date.getUTCFullYear();const month=String(date.getUTCMonth()+1).padStart(2,"0");const day=String(date.getUTCDate()).padStart(2,"0");const hour=String(date.getUTCHours()).padStart(2,"0");const minute=String(date.getUTCMinutes()).padStart(2,"0");const base_path=`${this.#user_options.directory}/${year}_${month}_${day}`;let file_path=base_path;if(this.#user_options.time_frame==="hour"){file_path=`${base_path}_${hour}.json`}else if(this.#user_options.time_frame==="minute"){file_path=`${base_path}_${hour}_${minute}.json`}if(!this.#data_in_ram[file_path]){this.#data_in_ram[file_path]=[]}this.#data_in_ram[file_path].push({m:measurement,v:value,t:timestamp});this.#has_pending_writes=true;this.#resetAutosaveTimeout();if(this.#calculateUsageOfRAM()>this.#user_options.max_ram){this.flush()}}flush(){if(!this.#has_pending_writes&&!this.#db_cleared){return}for(const[file_path,new_data_points]of Object.entries(this.#data_in_ram)){let old_data_points=[];if(Storage.Exists(file_path)){const old_data_str=Storage.ReadFile(file_path);if(old_data_str){old_data_points=JSON.parse(old_data_str)}}const merged_data_points=old_data_points.concat(new_data_points);Storage.WriteFile(file_path,JSON.stringify(merged_data_points));this.#updateIndex(file_path)}this.#persistIndexIfNeeded();this.#has_pending_writes=false;this.#db_cleared=false;this.#data_in_ram={}}query(start_time,end_time,aggregation_type,cb_custom_aggregator=null){const start_utc=new Date(start_time).toISOString();const end_utc=new Date(end_time).toISOString();const cache_key=`${start_utc}_${end_utc}_${aggregation_type}`;if(this.#query_cache[cache_key]){return this.#query_cache[cache_key]}const data_points=this.#collectDataPointsForRange(start_utc,end_utc);debugLog(3,`Querying from ${start_utc} to ${end_utc} with type ${aggregation_type}`);debugLog(3,"data_points:",data_points);let result;if(cb_custom_aggregator&&typeof cb_custom_aggregator==="function"){result=cb_custom_aggregator(data_points)}else{result=this.#performBuiltInAggregation(data_points,aggregation_type)}this.#query_cache[cache_key]=result;return result}retrieveDataSeries(start_time,end_time){const start_utc=new Date(start_time).toISOString();const end_utc=new Date(end_time).toISOString();const data_points=this.#collectDataPointsForRange(start_utc,end_utc);return data_points.map(dp=>({timestamp:dp.t,value:dp.v,measurement:dp.m}))}purge(older_than){const threshold_date=new Date(older_than);let is_index_modified=false;for(const date in this.#index){const file_date=new Date(date.split("_").join("-"));if(file_date<threshold_date){const day_index=this.#index[date];for(const hour in day_index){if(this.#user_options.time_frame==="hour"){const file_path=`${this.#user_options.directory}/${date}_${hour}.json`;Storage.RemoveFile(file_path)}else{for(const minute in day_index[hour]){const file_path=`${this.#user_options.directory}/${date}_${hour}_${minute}.json`;Storage.RemoveFile(file_path)}}}delete this.#index[date];is_index_modified=true}}if(is_index_modified){this.#persistIndex()}}databaseClear(consent){if(consent!=="YES"){debugLog(1,"You have to pass 'YES' to indicate you know what you're doing.");return}if(this.#autosave_timeout_id!==null){clearTimeout(this.#autosave_timeout_id);this.#autosave_timeout_id=null}const files=Storage.ListDirectory(this.#user_options.directory);files.forEach(file=>{const full_path=`${this.#user_options.directory}/${file}`;Storage.RemoveFile(full_path)});Storage.RemoveFile(`${this.#user_options.directory}/index.json`);Storage.RemoveFile(`${this.#user_options.directory}/index_backup.json`);this.#data_in_ram={};this.#index={};this.#query_cache={};this.#db_cleared=true;this.#has_pending_writes=false;debugLog(3,"Database cleared successfully.")}databaseClose(){if(this.#has_pending_writes||this.#db_cleared){this.flush()}if(this.#autosave_timeout_id!==null){clearTimeout(this.#autosave_timeout_id)}this.#persistIndexIfNeeded()}databaseBackup(backup_path="easy_tsdb_backup.json",include_index=false){const backup_dir="easy_tsdb_backups";const full_path=`${backup_dir}/${backup_path}`;Storage.MakeDirectory(backup_dir);const backup={database_directory:this.#user_options.directory,data_points:{},index:include_index?this.#index:undefined};const files=Storage.ListDirectory(this.#user_options.directory);files.forEach(file=>{if(file==="index.json"||file==="index_backup.json"){return}const data=Storage.ReadFile(full_path);if(data){backup.data_points[file]=JSON.parse(data)}});if(include_index){backup.index=this.#index}const backup_json=JSON.stringify(backup,null,2);Storage.WriteFile(backup_path,backup_json);debugLog(1,`Backup successfully saved to ${backup_path}`)}databaseRestore(consent,backup_path="easy_tsdb_backup.json",recalculate_index=true){if(consent!=="YES"){debugLog(1,"Explicit consent not provided. Restore operation aborted.");return}const backup_dir="easy_tsdb_backups";const full_path=`${backup_dir}/${backup_path}`;try{const backup=JSON.parse(Storage.ReadFile(full_path));this.#user_options.directory=backup.database_directory;this.databaseClear("YES");Object.entries(backup.data_points).forEach(([file,data])=>{Storage.WriteFile(full_path,JSON.stringify(data))});if(backup.index&&!recalculate_index){this.#index=backup.index}else{this.#rebuildIndex()}debugLog(1,`Database successfully restored from ${backup_path}.`)}catch(error){debugLog(1,"Failed to restore database:",error)}}#rebuildIndex(){debugLog(1,"Rebuilding index...");this.#index={};const files=Storage.ListDirectory(this.#user_options.directory);files.forEach(file=>{if(file==="index.json"||file==="index_backup.json"){return}const file_path=`${this.#user_options.directory}/${file}`;const data=Storage.ReadFile(file_path);if(data){backup.data_points[file]=JSON.parse(data)}});debugLog(1,"Index rebuilt.")}#resetAutosaveTimeout(){if(this.#autosave_timeout_id!==null){clearTimeout(this.#autosave_timeout_id)}this.#autosave_timeout_id=setTimeout(()=>{if(this.#has_pending_writes||this.#db_cleared){this.flush();this.#db_cleared=false;this.#persistIndexIfNeeded()}},this.#defaults.autosave_interval*1e3)}#collectDataPointsForRange(start_time,end_time){let bugfixed_start_time=new Date(start_time);bugfixed_start_time.setUTCDate(bugfixed_start_time.getUTCDate()-1);let current=new Date(bugfixed_start_time.toISOString());const end=new Date(end_time);let data_points=[];while(current<=end){const data_key=this.#generateDateKey(current);const hour=String(current.getUTCHours()).padStart(2,"0");const minute=this.#user_options.time_frame==="minute"?String(current.getUTCMinutes()).padStart(2,"0"):null;if(this.#index[data_key]&&this.#index[data_key][hour]&&(!minute||this.#index[data_key][hour][minute])){const file_path=this.#user_options.time_frame==="hour"?`${this.#user_options.directory}/${data_key}_${hour}.json`:`${this.#user_options.directory}/${data_key}_${hour}_${minute}.json`;const file_data_points=this.#getDataPointsFromFile(file_path);data_points=[...data_points,...file_data_points]}current=this.#incrementDate(current)}return data_points}#getDataPointsFromFile(file_path){if(!Storage.Exists(file_path)){debugLog(1,`No data file at path: ${file_path}, moving on.`);return[]}try{const data_points_str=Storage.ReadFile(file_path);if(data_points_str.trim().length>0){const data_points=JSON.parse(data_points_str);debugLog(2,`Was able to read the data from file: ${file_path}`);const data_points_proxied=data_points.map(dp=>this.#wrapDataPoint(dp));return data_points_proxied}else{debugLog(2,`File at path: ${file_path} is empty, moving on.`);return[]}}catch(error){debugLog(2,`Error reading data from file: ${file_path} - ${error.message}`);return[]}}#wrapDataPoint(data_point){return new Proxy(data_point,{get(target,property,receiver){if(property==="value")return target.v;if(property==="measurement")return target.m;if(property==="timestamp")return target.t;return Reflect.get(...arguments)}})}#generateDateKey(date){const year=date.getFullYear();const month=String(date.getMonth()+1).padStart(2,"0");const day=String(date.getDate()).padStart(2,"0");const new_date_key=`${year}_${month}_${day}`;return new_date_key}#performBuiltInAggregation(data_points,aggregation_type){debugLog(2,`Performing aggregation: ${aggregation_type} on data: ${JSON.stringify(data_points)}`);let result;if(data_points.length===0)return undefined;if(aggregation_type.startsWith("percentile_")){const percentile_value=parseInt(aggregation_type.split("_")[1],10);result=this.#calculatePercentile(data_points,percentile_value)}else if(aggregation_type==="trend"){if(data_points.length>1){const firstPoint=data_points[0].v;const lastPoint=data_points[data_points.length-1].v;result=lastPoint>firstPoint?"up":lastPoint<firstPoint?"down":"steady"}else{result="steady"}}else{switch(aggregation_type){case"raw":result=data_points;break;case"sum":result=data_points.reduce((acc,point)=>acc+point.v,0);break;case"average":result=data_points.reduce((acc,point)=>acc+point.v,0)/data_points.length;break;case"min":result=Math.min(...data_points.map(point=>point.v));break;case"max":result=Math.max(...data_points.map(point=>point.v));break;case"count":result=data_points.length;break;case"median":const sorted_vals=data_points.map(dp=>dp.v).sort((a,b)=>a-b);const middle_index=Math.floor(sorted_vals.length/2);result=sorted_vals.length%2!==0?sorted_vals[middle_index]:(sorted_vals[middle_index-1]+sorted_vals[middle_index])/2;break;case"mode":const frequency_map={};data_points.forEach(dp=>{if(!frequency_map[dp.v])frequency_map[dp.v]=0;frequency_map[dp.v]++});const max_frequency=Math.max(...Object.values(frequency_map));result=Object.keys(frequency_map).filter(key=>frequency_map[key]===max_frequency).map(parseFloat);if(result.length===1)result=result[0];break;case"stddev":if(data_points.length>1){const mean=data_points.reduce((acc,dp)=>acc+dp.v,0)/data_points.length;const variance=data_points.reduce((acc,dp)=>acc+Math.pow(dp.v-mean,2),0)/(data_points.length-1);result=Math.sqrt(variance)}else{result=undefined}break;case"first":result=data_points.length>0?data_points[0].v:undefined;break;case"last":result=data_points.length>0?data_points[data_points.length-1].v:undefined;break;case"range":const max_val=Math.max(...data_points.map(point=>point.v));const min_val=Math.min(...data_points.map(point=>point.v));result=max_val-min_val;break;case"iqr":const sorted=data_points.map(dp=>dp.v).sort((a,b)=>a-b);const q1=sorted[Math.floor(sorted.length/4)];const q3_pos=Math.floor(3*sorted.length/4);const q3=sorted.length%2===0?(sorted[q3_pos]+sorted[q3_pos-1])/2:sorted[q3_pos];result=q3-q1;break;case"variance":if(data_points.length>1){const mean=data_points.reduce((acc,dp)=>acc+dp.v,0)/data_points.length;result=data_points.reduce((acc,dp)=>acc+Math.pow(dp.v-mean,2),0)/(data_points.length-1)}else{result=undefined}break;case"rate_of_change":if(data_points.length>1){result=[];for(let i=1;i<data_points.length;i++){const rate=(data_points[i].v-data_points[i-1].v)/data_points[i-1].v;result.push(rate)}}else{result=undefined}break;default:throw new Error("Unsupported aggregation type")}}debugLog(3,`Aggregation result: ${result}`);return result}#calculatePercentile(data_points,percentile_rank){const sorted_values=data_points.map(dp=>dp.value).sort((a,b)=>a-b);const rank=percentile_rank/100*(sorted_values.length-1)+1;const index=Math.floor(rank)-1;const frac=rank%1;if(sorted_values.length===0)return undefined;if(frac===0)return sorted_values[index];else return sorted_values[index]+frac*(sorted_values[index+1]-sorted_values[index])}#calculateUsageOfRAM(){let ttl_size=0;for(const data_points of Object.values(this.#data_in_ram)){const size=JSON.stringify(data_points).length;ttl_size+=size}return ttl_size}#persistIndex(){const index_data=JSON.stringify(this.#index);const index_checksum=this.#calculateIndexChecksum(index_data);const index_content=JSON.stringify({index_data:index_data,index_checksum:index_checksum});Storage.WriteFile(`${this.#user_options.directory}/index.json`,index_content);Storage.WriteFile(`${this.#user_options.directory}/index_backup.json`,index_content);this.#cur_index_checksum=index_checksum}#loadIndex(){const index_path=`${this.#user_options.directory}/index.json`;const backup_index_path=`${this.#user_options.directory}/index_backup.json`;if(Storage.Exists(index_path)){const save_data=Storage.ReadFile(index_path);if(this.#tryLoadIndex(save_data)){return}}debugLog(3,"Attempting to recover index from backup.");if(Storage.Exists(backup_index_path)){const backup_data=Storage.ReadFile(backup_index_path);if(this.#tryLoadIndex(backup_data)){debugLog(2,"Successfully recovered index from backup.");return}}debugLog(3,"Both main and backup index files are unavailable or corrupt. Initializing an empty index.");this.#initializeEmptyIndex();this.#persistIndex()}#tryLoadIndex(saved_data){try{const{index_data,index_checksum}=JSON.parse(saved_data);const calculated_checksum=this.#calculateIndexChecksum(index_data);if(calculated_checksum===index_checksum){this.#index=JSON.parse(index_data);this.#cur_index_checksum=index_checksum;return true}}catch(error){debugLog(2,`Error loading or parsing index file: ${error}.`)}return false}#initializeEmptyIndex(){this.#index={};this.#cur_index_checksum=this.#calculateIndexChecksum(JSON.stringify(this.#index))}#updateIndex(file_path){const regex=/(\d{4}_\d{2}_\d{2})_(\d{2})(?:_(\d{2}))?\.json$/;const match=file_path.match(regex);if(match){const date_key=match[1];const hour=match[2];const minute=match[3];if(!this.#index[date_key]){this.#index[date_key]={}}if(this.#user_options.time_frame==="hour"){this.#index[date_key][hour]=1}else if(this.#user_options.time_frame==="minute"){if(!this.#index[date_key][hour]){this.#index[date_key][hour]={}}this.#index[date_key][hour][minute]=1}}}#incrementDate(cur_date){const new_date=new Date(cur_date);if(this.#user_options.time_frame==="hour"){new_date.setUTCHours(new_date.getUTCHours()+1,0,0,0)}else if(this.#user_options.time_frame==="minute"){new_date.setUTCMinutes(new_date.getUTCMinutes()+1,0,0)}return new_date}#setupDirectoryStructure(){Storage.MakeDirectory(this.#user_options.directory)}#persistIndexIfNeeded(){const index_data=JSON.stringify(this.#index);const new_index_checksum=this.#calculateIndexChecksum(index_data);if(this.#cur_index_checksum!==new_index_checksum){this.#persistIndex();this.#cur_index_checksum=new_index_checksum}}#calculateIndexChecksum(index_str){let checksum=0;for(let i=0;i<index_str.length;i++){checksum=(checksum+index_str.charCodeAt(i))%65535}const checksum_str=checksum.toString();debugLog(3,`Index checksum: ${checksum_str}`);return checksum_str}}