generalised-datastore/Queries.js

Generalised Datastore (GDS) — a decentralized data engine and RPC layer built for Web 4.0 applications, enabling distributed joins, caching, and remote program execution across nodes.

import { hash } from "godprotocol/utils/hash.js";
import Events from "./Events.js";
import { _id } from "./utils/functions.js";

class Queries extends Events {
  constructor() {
    super();
  }

  write_file = async (path, data, options = {}) => {
    let folder = this.folder.ds ? this.folder : this;
    if (folder.account) {
      let { write_bulks, address } = options;
      options.address = address || folder.address;
      options.account = folder.account;
      if (write_bulks) {
        write_bulks = write_bulks.split("-");
        let ds = folder.ds;
        if (write_bulks.length === 1) {
          let bulk = ds.bulks[write_bulks[0]];
          if (!bulk) {
            bulk = {};
            ds.bulks[write_bulks[0]] = bulk;
          }
          bulk[path] = { path, content: data, options };
        } else {
          let contents = ds.bulks[write_bulks[1]];
          delete ds.bulks[write_bulks];
          contents[path] = { content: data, path, options };

          await folder.ds.manager.oracle.write(null, Object.values(contents));
          // for (let p in contents){
          //   await folder.ds.manager.oracle.write(contents[p].path, contents[p].content)
          // }
        }
      } else
        await folder.ds.manager.oracle.write(path, data, {
          account: folder.account,
          address: address || folder.address,
        });
    } else {
      await folder.ds.set_caches(path, data);
    }
  };

  read_file = async (path, options = {}) => {
    let { when } = options;
    if (!path.includes("/")) path = `${this.path}/${path}`;
    if (when) path = `${path}/${when}`;

    let cont,
      folder = this.folder.ds ? this.folder : this;
    if (folder.account) {
      cont = await folder.ds.manager.oracle.read(path, {
        account: folder.account,
        address: options.address || folder.address,
      });
    } else {
      cont = await folder.ds.get_cache(path);
    }

    try {
      cont = cont && JSON.parse(cont);
    } catch (e) {}

    return cont;
  };

  _read_data = async (query, options = {}) => {
    let { excludes, skip, limit, cursor, no_retrieve, or, one, rm, no_joins } =
      options;
    no_joins = rm || no_joins;

    excludes = excludes || new Array();
    limit = one ? 1 : Number(limit) || -1;
    skip = Number(skip) || -1;

    let id = cursor || this.config.config.recent_file;

    let matches = new Array(),
      ids = new Array();

    while (id && limit) {
      let pth = `${this.path}/${id}`,
        content = await this.read_file(pth);

      if (content.deleted) break;

      if (excludes.includes(id)) {
        id = content.previous;
        continue;
      }

      if (!no_retrieve && this.config.config.is_index.length) {
        let parent_addr = this.address.split("/").slice(0, -1).join("/");
        let content_folder = await this.ds.folder(`${parent_addr}`, {
          account: this.account,
        });
        content = await content_folder.read_data(
          { _id: content.content._id },
          { one: true, rm: true }
        );
      }
      if (await this.pass(content.content, query, or)) {
        if (skip <= 0) {
          if (limit > 0) limit--;
          matches.push(rm ? { ...content } : content.content);
        } else skip--;
      }
      ids.push(id);
      id = content.previous;
      if (ids.includes(id)) break;
    }

    !no_joins && (await this.read_joins(matches));

    return one ? matches[0] : matches;
  };

  read_joins = async (data) => {
    return data;
  };

  read_data = async (query, options = {}) => {
    let data = await this._read_data(query, options);

    return data;
  };

  array_comparison = (arr, comparison) => {
    let find = false;
    for (let a = 0; a < arr.length; a++) {
      if (arr[a] === comparison) {
        find = true;
        break;
      }
    }
    return find;
  };

  pass = async (content, query, or = false) => {
    let pass = new Array();

    if (query == null) return true;

    for (let q in query) {
      let qval = query[q],
        lval = content[q];

      if (qval === undefined) continue;

      if (Array.isArray(lval) || Array.isArray(qval)) {
        let arr1, arr2;
        if (!Array.isArray(lval)) {
          arr1 = new Array(lval);
          arr2 = qval;
        } else if (!Array.isArray(qval)) {
          arr1 = new Array(qval);
          arr2 = lval;
        } else (arr1 = qval), (arr2 = lval);

        let m = false;
        for (let l = 0; l < arr1.length; l++)
          if (this.array_comparison(arr2, arr1[l])) {
            m = true;
            break;
          }

        pass.push(m);
      } else if (typeof qval === "object") {
        if (Object(qval).hasOwnProperty("$ne")) pass.push(lval !== qval["$ne"]);
        else if (Object(qval).hasOwnProperty("$e"))
          pass.push(lval === qval["$e"]);
        else if (Object(qval).hasOwnProperty("$gt"))
          pass.push(lval > qval["$gt"]);
        else if (Object(qval).hasOwnProperty("$lt"))
          pass.push(lval < qval["$lt"]);
        else if (Object(qval).hasOwnProperty("$gte"))
          pass.push(lval >= qval["$gte"]);
        else if (Object(qval).hasOwnProperty("$lte"))
          pass.push(lval <= qval["$lte"]);
        else if (Object(qval).hasOwnProperty("$includes"))
          pass.push(lval.includes && lval.includes(qval["$includes"]));
      } else pass.push(lval === qval);
    }

    if (or) return !!pass.find((p) => p);
    else {
      for (let p = 0; p < pass.length; p++) if (!pass[p]) return false;
      return true;
    }
  };

  operators = ["ne", "e", "lte", "gte", "lt", "gt", "includes"];

  update_next = async (file, _id, prev = false, write_bulks = null) => {
    if (!file) return;

    let addr = prev === "future" ? file : `${this.address}/${file}`;

    addr = `${addr}`;
    let content = await this.read_file(addr);

    if (prev === "future") {
      content.future = _id;
    } else if (_id === "deleted") {
      content.deleted = true;
    } else content[prev ? "previous" : "next"] = _id;

    await this.write_file(addr, JSON.stringify(content), { write_bulks });
  };

  write_join_object = async ({ address, value, options }) => {
    let folder = await this.ds.folder(address, { account: this.account });
    let res = await folder.write(value, {
      write_bulks: options.write_bulks,
      replace: options.replace,
    });

    return res;
  };

  handle_joins = async (data, options) => {
    for (let j in this.config.config.joins) {
      let joi = this.config.config.joins[j],
        address;
      let value = data[j];
      if (typeof joi === "string") {
        address = joi;
      } else address = joi.address;
      if (!address) continue;

      if (value && typeof value === "object") {
        if (Array.isArray(value)) {
          for (let v = 0; v < value.length; v++) {
            let item = value[v];
            let res = await this.write_join_object({
              address,
              value: item,
              options,
            });
            data[j][v] = res._id;
          }
        } else {
          let res = await this.write_join_object({ address, value, options });
          data[j] = res._id;
        }
      }
    }
  };

  _write_data = async (data, options = {}) => {
    let { replace, update, persist, write_bulks } = options;
    let timestamp;

    data = { ...data };
    if (!data.created) {
      data.created = Date.now();
    }
    data.updated = Date.now();
    timestamp = data.updated;

    if (!data._id) data._id = _id();

    write_bulks = write_bulks || Math.random().toString();

    await this.handle_joins(data, { ...options, write_bulks });

    let datapath = `${this.path}/${data._id}`;
    let content = await this.read_file(datapath);
    let exists = content && !content.deleted,
      previous = this.config.config.recent_file;

    if (!exists) {
      await this.write_file(
        datapath,
        JSON.stringify({ content: data, previous }),
        { write_bulks }
      );
      if (previous) {
        await this.update_next(previous, data._id, null, write_bulks);
      }
    } else if (replace) {
      let prev_content = JSON.stringify(content);
      if (this.config.config.recent_file !== data._id && !update) {
        if (content.previous)
          await this.update_next(
            content.previous,
            content.next,
            null,
            write_bulks
          );
        if (content.next)
          await this.update_next(
            content.next,
            content.previous,
            true,
            write_bulks
          );

        content.previous = this.config.config.recent_file;
        await this.update_next(
          this.config.config.recent_file,
          data._id,
          null,
          write_bulks
        );
        this.config.config.recent_file = data._id;
      }
      content.past = await this.hash_content(content);
      content.content = data;

      if (this.config.config.blocks) {
        prev_content = JSON.parse(prev_content);
        await this.write_file(
          `${datapath}.blocks/${content.past}`,
          JSON.stringify(prev_content),
          { write_bulks }
        );

        if (prev_content.past) {
          await this.update_next(
            `${datapath}.blocks/${prev_content.past}`,
            content.past,
            "future",
            write_bulks
          );
        }
      }

      await this.write_file(datapath, JSON.stringify(content), { write_bulks });
    }

    !update &&
      (!exists || replace) &&
      (await this.config.write(
        data,
        options.write_bulks || `write-${write_bulks}`
      ));

    let res = {
      _id: data._id,
      created: data.created,
      updated: data.updated,
      existed: exists || false,
      replaced: replace || false,
      address: this.address,
      timestamp,
      datapath,
    };

    return res;
  };

  hash_content = async (content) => {
    let data = { ...content.data };
    delete data.updated;

    return hash(JSON.stringify(data).concat(content.past));
  };

  write_data = async (data, options = {}) => {
    let res = await this._write_data(data, options);

    return res;
  };

  update_snips = async (entry, update_query) => {
    let prior_values = new Object();
    for (let prop in update_query) {
      let update_value = update_query[prop];

      prior_values[prop] = entry[prop];
      if (typeof update_value === "object") {
        if (Object(update_value).hasOwnProperty("$inc")) {
          if (
            typeof entry[prop] === "number" &&
            typeof update_value["$inc"] === "number"
          )
            entry[prop] += update_value["$inc"];
          else if (
            typeof entry[prop] !== "number" &&
            typeof update_value["$inc"] === "number"
          )
            entry[prop] = update_value["$inc"];
        } else if (Object(update_value).hasOwnProperty("$dec")) {
          if (
            typeof entry[prop] === "number" &&
            typeof update_value["$dec"] === "number"
          )
            entry[prop] -= update_value["$dec"];
          else if (
            typeof entry[prop] !== "number" &&
            typeof update_value["$dec"] === "number"
          )
            entry[prop] = update_value["$dec"] * -1;
        } else if (Object(update_value).hasOwnProperty("$set")) {
          let value = update_value["$set"];
          if (!Array.isArray(value)) value = new Array(value);
          if (Array.isArray(entry[prop]))
            value.map(
              (val) => !entry[prop].includes(val) && entry[prop].push(val)
            );
          else if (value.includes(entry[prop]))
            entry[prop] = new Array(...value);
          else entry[prop] = new Array(entry[prop], ...value);
        } else if (
          Object(update_value).hasOwnProperty("$unshift") ||
          Object(update_value).hasOwnProperty("$push")
        ) {
          let value = update_value["$push"] || update_value["$unshift"];
          if (!Array.isArray(value)) value = new Array(value);
          if (Array.isArray(entry[prop]))
            entry[prop][
              Object(update_value).hasOwnProperty("$push") ? "push" : "unshift"
            ](...value);
          else {
            let init_line_prop = entry[prop];
            entry[prop] = new Array(...value);
            if (!init_line_prop) entry[prop].unshift(init_line_prop);
          }
        } else if (Object(update_value).hasOwnProperty("$splice")) {
          let value = update_value["$splice"],
            line_value = entry[prop];
          if (!Array.isArray(value)) value = new Array(value);
          if (Array.isArray(line_value)) {
            entry[prop] = line_value.filter((val) => !value.includes(val));
          } else if (!Array.isArray(line_value) && line_value === value)
            delete entry[prop];
        } else entry[prop] = update_value;
      } else entry[prop] = update_value;
    }

    return prior_values;
  };

  update_data = async (query, update_query, options = {}) => {
    let data = await this._read_data(query, options);

    if (data && !Array.isArray(data)) data = [data];
    for (let d = 0; d < data.length; d++) {
      let datum = data[d];
      let bfr = JSON.stringify(datum);
      let prior_values = await this.update_snips(datum, update_query);

      await this.config.update(
        bfr,
        datum,
        prior_values,
        options.write_bulks || "update"
      );
      await this._write_data(datum, { replace: true, update: true });
    }

    return data; // await this.respond(data, {method: 'update'})
  };

  remove_joins = async (datum) => {};

  remove_data = async (query, options) => {
    let data = await this._read_data(query, {
      ...options,
      rm: true,
      no_retrieve: !!this.config.config.is_index.length,
    });

    if (data && !Array.isArray(data)) data = [data];
    let datum;
    while (data.length) {
      datum = data.pop();

      await this.update_next(datum.content._id, "deleted");
      if (datum.previous) await this.update_next(datum.previous, datum.next);
      if (datum.next) await this.update_next(datum.next, datum.previous, true);

      await this.remove_joins(datum);
      await this.config.remove(datum, options.write_bulks || "remove");
    }
    return datum;
  };
}

export default Queries;