east-analysistools/analysistools.mjs

description: Tools to analyze the performance of Crypto Currencies

import { abs, min, max, sum, ceil , log, exp} from "mathjs";
import { MongoClient } from "mongodb";
import uTools from "updatetools";
import wt from "discrete-wavelets";

async function refactorData(crypt, timeSpan) {
  // take data in hour format and create dataset for longer time perid
  // right now the data is hours
  // longer time periods needed are 6hr, days, weeks
  // 6hr = 6 data points
  // 1day= 24 data points
  // 7 days= 168 data points
  // Example:
  // for 24 refactor
  //      period = 24
  //      for the latest of each 24 data points we get the date and close values
  //      for open we use the earliest of the 24 data points
  //      volume = sum of the volumes for all 24 data points
  //      low = minimum(all the lows for all 24 points)
  //      high = maximum ( all the highs for all 24 datapoints)

  var validTimeSpans = [6, 24, 168];
  if (!validTimeSpans.includes(timeSpan)) {
    throw new Error("Invalid timespan");
  }
  try {
    var dataSet = [];
    var dataPoint;
    var high = crypt.map((x) => x.high);
    var low = crypt.map((x) => x.low);
    var volume = crypt.map((x) => x.volume);
    var lastPoint;
    var ts1 = timeSpan - 1;
    for (let i = 0; i < crypt.length - timeSpan; i++) {
      lastPoint = i + timeSpan;
      dataPoint = {};
      dataPoint["date_time"] = crypt[i].date_time;
      dataPoint["low" + timeSpan] = min(...low.slice(i, lastPoint));
      dataPoint["high" + timeSpan] = max(...high.slice(i, lastPoint));
      dataPoint["close" + timeSpan] = crypt[i + ts1].close;
      dataPoint["volume" + timeSpan] = sum(volume.slice(i, lastPoint));
      dataSet.push(dataPoint);
    }
    return dataSet;
  } catch (err) {
    throw err;
  }
}

async function refactorSet(MDBURI, exch, product, timeSpan, Force) {
  var force = Force || false;
  const client = await MongoClient.connect(MDBURI);

  var crypto;
  var closeTime = "close" + timeSpan;
  try {
    // get latest entry of "close" + timeSpan
    var closeTime = "close" + timeSpan;
    var mostRecent = await client
      .db(exch)
      .collection(product)
      .aggregate([
        { $project: { _id: 0, date_time: 1, [closeTime]: 1 } },
        { $match: { [closeTime]: { $exists: false } } },
        { $sort: { date_time: 1 } },
        { $limit: 1 },
      ])
      .toArray();
    if ((mostRecent.length == 0) || force) {
      // there is no "closeTime" data, so get all of it
      crypto = await client
        .db(exch)
        .collection(product)
        .find()
        .sort({ date_time: 1 })
        .toArray();
    } else {
      crypto = await client
        .db(exch)
        .collection(product)
        .find({ date_time: { $gte: new Date(mostRecent[0].date_time) } })
        .sort({ date_time: 1 })
        .toArray();
    }
    var dh = await refactorData(crypto, timeSpan);
  } catch (err) {
    console.log(err);
  } finally {
    client.close();
  }
  console.log(
    "Update ",
    dh.length,
    " records for ",
    product,
    " With time spacing of ",
    timeSpan,
    " hours"
  );
  if (dh.length > 0) uTools.DBupdateSet(MDBURI, exch, product, dh);
}

async function WTIfilter(dataArray, N) {
  var coeffs = await wt.wavedec(dataArray, "D16");
  // console.log("coeffs:", coeffs)
  var slevel = coeffs.length - 1;
  if (N > slevel) N = slevel;
  for (let i = N; i < coeffs.length; i++) {
    coeffs[i].fill(0);
  }
  return await wt.waverec(coeffs, "D16");
}

async function WTSmooth(dataSet, smoothLevel) {
  var spacing = 2048,
    overlap = 64;
  var workSet = [];
  var transition = [];
  var revtransition = [];
  var setSmooth, dataSnip, lastData;
  for (let j = 0; j < overlap; j++) {
    transition.push(j / overlap);
    revtransition.push(1 - transition[j]);
  }
  dataSnip = await dataSet.slice(0, spacing);
  setSmooth = await WTIfilter(dataSnip, smoothLevel);
  lastData = setSmooth.slice(0, overlap);
  if (dataSet.length < spacing + 1) workSet = [...setSmooth];
  else
    for (let i = 0; i < dataSet.length - spacing + 1; i += spacing - overlap) {
      dataSnip = await dataSet.slice(i, i + spacing);
      setSmooth = await WTIfilter(dataSnip, smoothLevel);
      for (let j = 0; j < overlap; j++) {
        setSmooth[j] =
          lastData[j] * revtransition[j] + setSmooth[j] * transition[j];
      }
      lastData = setSmooth.slice(-overlap);
      workSet.push(...setSmooth.slice(0, -overlap));
    }
  if (workSet.length < dataSet.length) workSet.push(...lastData);

  return workSet;
}


async function smoothPadded(dSet, smoothLevel) {
  const setLength = dSet.length;
  const wtLength = 2048,
    overlap = 128;
  const wtl1 = wtLength - overlap;

  // prepend data to beginning so that it is always doing the last 2 sets
  // all prepended data is the same as the first element of the dataset
  var fillNumber = dSet[0];
  var workingSet = await Array(wtl1).fill(fillNumber);
  await workingSet.push(...dSet);

  // postPend data to make each dataset have a length of
  //  length = wtL1* N + wtLength
  // filled with last digit of set

  var requiredLength =
    ceil((workingSet.length - wtLength) / wtl1) * wtl1 + wtLength;
  fillNumber = workingSet.at(-1);
  var postLength = requiredLength - workingSet.length;
  var postSet = await Array(postLength).fill(fillNumber);
  workingSet.push(...postSet);
  // smooth the data and then chop
  var smoothSet = await WTSmooth(workingSet, smoothLevel);
  if (postLength < 1) postLength = 1;
  smoothSet = await smoothSet.slice(wtl1, -postLength);
  return smoothSet;
}

async function DBfirstRecord(MDBURI, exch, product) {
  const client = await MongoClient.connect(MDBURI);

  var crypto;

  try {
    // get latest entry of "close" + timeSpan
    crypto = await client
      .db(exch)
      .collection(product)
      .aggregate([
        { $project: { _id: 0, date_time: 1, open: 1, volume24: 1 } },
        { $match: {} },
        { $sort: { date_time: 1 } },
        { $limit: 1 },
      ])
      .sort({ date_time: 1 })
      .toArray();
    return crypto[0].date_time;
  } catch (err) {
    console.log(err);
  } finally {
    client.close();
  }
}

async function DBsmoothAll(MDBURI, exch, product) {
  const client = await MongoClient.connect(MDBURI);

  var crypto;

  try {
    // get latest entry of "close" + timeSpan
    crypto = await client
      .db(exch)
      .collection(product)
      .aggregate([
        { $project: { _id: 0, date_time: 1, open: 1, volume24: 1 } },
        { $match: {} },
      ])
      .sort({ date_time: 1 })
      .toArray();
    var dSet = await crypto.map(({ open }) => open);
    var tSet = await crypto.map(({ date_time }) => date_time);
    var vSet = await crypto.map(({ volume24 }) => volume24);
    var i = vSet.length;
    var setLength = vSet.length;
    while (vSet[i] == undefined) i--;

    for (let j = i; j < setLength; j++) {
      vSet[j] = vSet[i];
    }
    var smoothData = await smoothPadded(dSet, 2);
    // keep smoothed volumes from going negative
    // when data is missing for longer time periods
    for (let i = 0; i < vSet.length; i++) {
      if ((vSet[i]< 0.0001)) {
        vSet[i]= vSet[i-1]
      }
    }
    vSet= vSet.map((x) =>log(x));
    var smoothVolume = await smoothPadded(vSet, 1);
    smoothVolume= smoothVolume.map((x) => exp(x));

    var items = tSet.map((date_time, index) => {
      return {
        date_time: date_time,
        opensmooth: smoothData[index],
        volumesmooth: smoothVolume[index],
      };
    });

    console.log("Update smooth data for: ", product);
    await uTools.DBupdateSet(MDBURI, exch, product, items);
  } catch (err) {
    console.log(err);
  } finally {
    client.close();
  }
}

async function DBsmoothUpdate(MDBURI, exch, product) {
  var firstDate = await DBfirstRecord(MDBURI, exch, product);
  var lastDate = await uTools.DBmostRecentDate(MDBURI, exch, product);

  var totalHours =
    (new Date(lastDate).getTime() - new Date(firstDate).getTime()) / 3600000;
  var wtLength = 2048,
    overlap = 128,
    wtl1 = wtLength - overlap;
  var workingLength = totalHours + wtl1;
  var totalLength = ceil((workingLength - wtLength) / wtl1) * wtl1 + wtLength;
  var grabLength = wtLength + wtl1;
  var postPendLength = totalLength - wtl1 - totalHours;
  var getDataStartOffset = wtLength + wtl1 - postPendLength;
  var sampleStartDate = new Date(
    new Date(lastDate).getTime() - getDataStartOffset * 3600000
  );
  var dataPoints =
    (new Date(sampleStartDate).getTime() - new Date(firstDate).getTime()) /
    3600000;
  if (dataPoints < 0) sampleStartDate = new Date(firstDate);

  console.log("sampleStartDate:", sampleStartDate);

  const client = await MongoClient.connect(MDBURI);

  var crypto;

  try {
    // get latest entry of "close" + timeSpan
    crypto = await client
      .db(exch)
      .collection(product)
      .aggregate([
        { $project: { _id: 0, date_time: 1, open: 1, volume24: 1 } },
        { $match: { date_time: { $gte: sampleStartDate } } },
      ])
      .sort({ date_time: 1 })
      .toArray();
    var tSet = await crypto.map(({ date_time }) => date_time);
    var dSet = await crypto.map(({ open }) => open);
    var vSet = await crypto.map(({ volume24 }) => volume24);
    var i = vSet.length;
    var setLength = vSet.length;
    while (vSet[i] == undefined) i--;

    for (let j = i; j < setLength; j++) {
      vSet[j] = vSet[i];
    }

    for (let i = 0; i < vSet.length; i++) {
      if ((vSet[i]< 0.0001)) {
        vSet[i]= vSet[i-1]
      }
    }
    vSet= vSet.map((x) =>log(x));

    var smoothData = await smoothPadded(dSet, 2);
    var smoothVolume = await smoothPadded(vSet, 1);
    smoothVolume= smoothVolume.map((x) => exp(x));
    var items = tSet.map((date_time, index) => {
      return {
        date_time: date_time,
        opensmooth: smoothData[index],
        volumesmooth: smoothVolume[index],
      };
    });
    if (dataPoints > 128) items.slice(128, -1);
    console.log("Update smooth data for: ", product);
    await uTools.DBupdateSet(MDBURI, exch, product, items);
  } catch (err) {
    console.log(err);
  } finally {
    client.close();
  }
}

async function MRI(dSet) {
  var closeSmooth = await WTSmooth(dSet, 2048, 64);
  var ratioSet = [];
  var ratioPower = 7.0 / 90;
  if (dSet.length > 2160) {
    for (let i = 2160; i < dSet.length; i++) {
      ratioSet.push((dSet[i] / closeSmooth[i - 2160]) ** ratioPower);
    }
    const ratioSmooth = await WTSmooth(ratioSet, 512, 32);
    return ratioSmooth;
  }
}

const analysistools = {
  refactorSet,
  WTIfilter,
  WTSmooth,
  smoothPadded,
  DBsmoothAll,
  DBsmoothUpdate,
  MRI,
};

export default analysistools;