vsphere-infra/src/vsphere-soap/vsphere.ts

/*
 * Copyright 2014-2021 Firestack, all rights reserved.
 */

import { VsphereConnection } from './connection';
import { ref } from './soap.parser';
import { getMoTypeByName, iidParse, Mo } from './managed.objects';
import { moref } from '../../models/mo.props';
import { default as axios } from 'axios';
import { SSLTrust } from 'ssl-trust';
import { ix, promise, Unit, PromUtil, Tasks } from 'ts-basis';
import { Observable } from 'rxjs';
import {
  VsphereDatacenterEvent, VsphereDatacenterUtilizationSnapshot,
  VsphereDatacenterUtilizationSnapshotData,
  VsphereDatacenterUtilizationSummary, VsphereEventWebhook, VsphereTarget
} from '../../models/portable';


export class VsphereDatacenter extends ix.ReconnEntity {
  static keptData: {type: string, data: any}[] = [];
  static registry: {[key: string]: VsphereDatacenter} = {};
  static async fromTargetInfo(target: VsphereTarget, vinfra: VsphereInfra): Promise<VsphereDatacenter> {
    if (!target.key) { return null; }
    let dc: VsphereDatacenter;
    const datacenterData = vinfra.datacenterData;
    const behavior = vinfra.behavior;
    if (!datacenterData[target.key]) {
      dc = datacenterData[target.key] = new VsphereDatacenter(target.key, vinfra);
      dc.verbose = behavior.verbose;
      dc.readonlyMode = behavior.readonlyMode;
      dc.webhooks = behavior.webhooks;
      dc.jsonLogs = behavior.jsonLogs;
      dc.sourceTarget = target;
      if (target.cert) {
        if (target.cert.trim().startsWith('-----BEGIN CERTIFICATE-----')) {
          await SSLTrust.trustCert(target.cert);
        } else {
          await SSLTrust.trustCertFile(target.cert);
        }
      }
    } else {
      dc.sourceTarget = target;
    }
    return dc;
  }

  key: string;
  sourceTarget: VsphereTarget;
  vinfra: VsphereInfra;
  ready = false;
  readyResolver: (...a: any[]) => any = null;
  readyPromise = null as Promise<boolean>;
  connections: VsphereConnection[] = [];
  connectionPickIndex = 0;
  disconnected = false;
  inventoryPending: boolean | Promise<boolean> = true;
  inventoryStubPendingResolver: any;
  inventoryStubPending: Promise<boolean>;
  inventoryStubHunts: {[iid: string]: boolean | Promise<boolean>} = {};
  inventoryInProgress = false;
  inventoryCheckInProgress = 0;
  inventoryCheckInterval = 30;
  inventoryCheckLast = 0;
  inventoryChangedLast = 0;
  inventoryWatchStarted = false;
  inventoryWatchChecker: ix.Timer = null;
  inventorySerialized: string;
  inventory = {
    byIID: {} as {[key: string]: Mo.Base },
    byGUID: {} as {[key: string]: Mo.Base },
    folder: {} as {[key: string]: Mo.Folder },
    hostSystem: {} as {[key: string]: Mo.HostSystem },
    datacenter: {} as {[key: string]: Mo.Datacenter },
    network: {} as {[key: string]: Mo.Network },
    datastore: {} as {[key: string]: Mo.Datastore },
    computeResource: {} as {[key: string]: Mo.ComputeResource },
    resourcePool: {} as { [key: string]: Mo.ResourcePool },
    virtualApp: {} as {[key: string]: Mo.VirtualApp },
    virtualMachine: {} as {[key: string]: Mo.VirtualMachine },
    virtualSwitch: {} as {[key: string]: Mo.VirtualSwitch },
    deleted: [] as Mo.Base[],
  };
  index = {
    byName: {} as {[key: string]: moref[] },
    byIP: {} as {[key: string]: moref[] },
  };

  utilization: VsphereDatacenterUtilizationSummary = null;
  u9nSerialized: string;
  u9nCheckInterval = 60;
  u9nCheckLast = 0;
  u9nHistory: VsphereDatacenterUtilizationSnapshotData[] = [];
  u9nHistoryMaxKept = 1440;
  u9nWithHistory = true;
  u9nWatchStarted = false;
  u9nWatchChecker: ix.Timer = null;

  readonlyMode = false;
  queriedIIDs: string[] = [];
  queriedStubs: {iid: moref, name: string}[] = [];
  verbose = false;
  jsonLogs = false;

  webhooks: VsphereEventWebhook[] = [];

  constructor(key: string, vinfra: VsphereInfra) {
    super(`vsphere-${key}`);
    this.key = key;
    this.vinfra = vinfra;
    this.ix.reconn.on.defunct = () => {
      this.debugOutput(`This vCenter (${this.key}) class is going defunct after too many errors.`);
    };
    this.ix.reconn.on.restore = () => {
      this.debugOutput(`This vCenter (${this.key}) has come back from defunct state.`);
    };
    this.ix.reconn.actions.reset = async () => {
      this.debugOutput(`Errors heating up above ${this.ix.reconn.config.resetThreshold}; `
                    + `resetting main connection... (${this.ix.reconn.defunctHeat.value.toPrecision(2)})`);
      await this.resetConnection(true);
    };
    this.ix.reconn.actions.attemptToRestore = async () => {
      return await this.resetConnection() ? true : false;
    };
    if (this.vinfra?.behavior?.reconnect) {
      this.ix.reconn.setConfigSource(this.vinfra.behavior.reconnect);
    }
    vinfra.lifecycle.manage(this);
    this.readyResetState();
  }

  get datacenterGeneric$() { return this.rx<VsphereDatacenterEvent>('datacenter_generic').obs(); }
  get connectionReset$() { return this.rx<VsphereDatacenterEvent>('connection:reset').obs(); }
  get connectionDefunct$() { return this.rx<VsphereDatacenterEvent>('connection:defunct').obs(); }
  get connectionRecover$() { return this.rx<VsphereDatacenterEvent>('connection:recover').obs(); }
  get sessionNotValid$() { return this.rx<VsphereDatacenterEvent>('session:not_valid').obs(); }
  get entryRegister$() { return this.rx<VsphereDatacenterEvent>('entry:register').obs(); }
  get entryRefresh$() { return this.rx<VsphereDatacenterEvent>('entry:refresh').obs(); }
  get entryMove$() { return this.rx<VsphereDatacenterEvent>('entry:move').obs(); }
  get entryNew$() { return this.rx<VsphereDatacenterEvent>('entry:new').obs(); }
  get entryDelete$() { return this.rx<VsphereDatacenterEvent>('entry:delete').obs(); }
  get entryNotFound$() { return this.rx<VsphereDatacenterEvent>('entry:not_found').obs(); }
  get u9nUpdate$() { return this.rx<VsphereDatacenterEvent<VsphereDatacenterUtilizationSummary>>('u9n:update').obs(); }

  get u9n() { return this.utilization; }

  async procureConnection(timeout = this.vinfra.behavior.connection.timeout, reuseClient = false) {
    if (!timeout && timeout !== 0) { timeout = this.vinfra.behavior.connection.timeout; }
    let connection: VsphereConnection;
    if (reuseClient && this.connections[0]) {
      connection = this.connections[0];
      const loggedIn = await retry(timeout,
        async () => await connection.login(this.sourceTarget.username, this.sourceTarget.password));
      if (!loggedIn) {
        connection = await this.procureConnection(timeout, false);
      } else {
        connection.api.resetData();
      }
    } else {
      connection = new VsphereConnection(this, this.sourceTarget.url, {} as any, VsphereInfra.granted);
      try {
        const connected = await retry(timeout,
            async () => await connection.connect());
        if (!connected) {
          this.debugOutput(`unable to connect to ${this.sourceTarget.url}`);
          return null;
        }
        const loggedIn = await retry(timeout,
            async () => await connection.login(this.sourceTarget.username, this.sourceTarget.password));
        if (!loggedIn) { return null; }
      } catch (e) {
        this.debugOutput(e);
        return null;
      }
      this.debugOutput(`successfully logged in as ${this.sourceTarget.username}`);
    }
    return connection;
  }

  async resetConnection(reuseClient = false) {
    this.readyResetState();
    const conn = await this.procureConnection(null, reuseClient);
    if (!conn) {
      this.ix.pushError(new Error(`unable to procure authenticated connection`), 10);
      this.readyFulfill(false);
      return null;
    }
    this.connections.length = 0;
    this.connections.push(conn);
    this.readyFulfill(true);
    return conn;
  }

  async initializeConnection() {
    return await this.resetConnection();
  }

  pickConnection() {
    if (this.connections.length === 0) { return null; }
    const idx = (++this.connectionPickIndex) % this.connections.length;
    return this.connections[idx];
  }

  lockStubs() {
    this.unlockStubs();
    this.inventoryStubPending = new Promise(resolve => { this.inventoryStubPendingResolver = resolve; });
  }

  unlockStubs() {
    if (this.inventoryStubPendingResolver) { this.inventoryStubPendingResolver(false); }
    this.inventoryStubPendingResolver = null;
  }

  async getInventory() {
    if (this.inventoryInProgress) { return this; }
    try {
      this.inventoryInProgress = true;
      const conn = this.pickConnection();
      if (!conn) { return this.errorNoConnection(); }
      await conn.api.getInventory(true);
    } catch (e) {  this.ix.pushError(e); }
    this.inventoryInProgress = false;
    return this;
  }

  trimDeleted() {
    this.inventory.deleted = this.inventory.deleted.filter(mo => {
      return (Date.now() - mo.$deletedTime < 3600000); // purge all deleted mo older than 1 hour
    });
  }

  async cleanInventory() {
    this.debugOutput(`cleaning inventory...`);
    for (const iid of Object.keys(this.inventory.byIID)) {
      this.inventory.byIID[iid].unregister(); // remove and mark defunct
    }
    const deletedListSaved = this.inventory.deleted;
    this.inventoryPending = true;
    this.inventoryStubPendingResolver = null;
    this.inventoryStubPending = null;
    this.inventoryStubHunts = {};
    this.inventoryCheckInProgress = 0;
    this.inventory = {
      byIID: {} as {[key: string]: Mo.Base },
      byGUID: {} as {[key: string]: Mo.Base },
      folder: {} as {[key: string]: Mo.Folder },
      hostSystem: {} as {[key: string]: Mo.HostSystem },
      datacenter: {} as {[key: string]: Mo.Datacenter },
      network: {} as {[key: string]: Mo.Network },
      datastore: {} as {[key: string]: Mo.Datastore },
      computeResource: {} as {[key: string]: Mo.ComputeResource },
      resourcePool: {} as { [key: string]: Mo.ResourcePool },
      virtualApp: {} as {[key: string]: Mo.VirtualApp },
      virtualMachine: {} as {[key: string]: Mo.VirtualMachine },
      virtualSwitch: {} as {[key: string]: Mo.VirtualSwitch },
      deleted: deletedListSaved
    };
    this.index = {
      byName: {} as {[key: string]: moref[] },
      byIP: {} as {[key: string]: moref[] },
    };
    this.debugOutput(`unregistered all inventory entries.`);
  }

  async getInventoryIIDs() {
    try {
      const conn = this.pickConnection();
      if (!conn) { return this.errorNoConnection(); }
      const moStubs = await conn.api.getInventoryStubs(true);
      if (!moStubs) { return null; }
      this.queriedStubs = moStubs;
      this.queriedIIDs = this.queriedStubs.map(item => item.iid);
    } catch (e) { this.ix.pushError(e); }
    return this.queriedIIDs;
  }

  async checkInventory() {
    const conn = this.pickConnection();
    if (!conn) { return this.errorNoConnection(); }
    if (this.inventoryCheckInProgress) {
      if (Date.now() - this.inventoryCheckInProgress > 30000) {
        this.ix.pushError(new Error(`inventory check was taking too long; attempting a retry.`));
      } else {
        this.ix.pushError(new Error(`inventory check is already in progress.`), 0.5);
        return;
      }
    }
    this.inventoryCheckInProgress = Date.now();
    const iids = await this.getInventoryIIDs();
    if (!iids) {
      this.inventoryCheckInProgress = 0;
      this.ix.pushError(new Error(`inventory check failed; entities stub list fetch failed.`), 0.5);
      return;
    }
    const newEntries = [];
    const deletedEntries = [];
    const iidsMap = {};
    for (const iid of iids) {
      iidsMap[iid] = true;
      if (!this.inventory.byIID[iid]) { newEntries.push(iid); }
    }
    for (const iid of Object.keys(this.inventory.byIID)) {
      if (iid === '$mo:Folder:group-d1') { continue; }
      if (!iidsMap[iid]) { deletedEntries.push(iid); }
    }
    if (newEntries.length > 0 || deletedEntries.length > 0) {
      this.debugOutput(`inventory change: ${newEntries.length} new entries and ${deletedEntries.length} deleted entries.`);
      if (deletedEntries.length > 0) {
        for (const iid of deletedEntries) {
          const mo = this.inventory.byIID[iid];
          if (!mo) { continue; }
          mo.unregister();
          this.emitEvent(this.entryDelete$, { target: mo.entitySummaryHeaders() });
        }
      }
      if (newEntries.length > 0) {
        const entries = await this.getMany(newEntries);
        for (const mo of entries) { this.emitEvent(this.entryNew$, { target: mo.entitySummaryHeaders() }); }
      }
      this.inventoryChangedLast = Date.now();
      this.preserializeInventory();
    } else {
      // nothing changed
    }
    this.trimDeleted();
    this.inventoryCheckInProgress = 0;
  }

  preserializeInventory() {
    const serializedInv: {[key: string]: string} = {};
    for (const guid of Object.keys(this.inventory.byGUID)) {
      const mo = this.inventory.byGUID[guid];
      if (!mo) { continue; }
      serializedInv[guid] = mo.entitySummaryHeaders();
    }
    this.inventorySerialized = JSON.stringify(serializedInv);
  }

  async postInventoryDataFetch() {
    const dsList = Object.keys(this.inventory.datastore).map(key => this.inventory.datastore[key]);
    for (const ds of dsList) {
      if (ds.info.aliasOf === ds.info.containerId) { continue; }
      ds.aliasOf = dsList.filter(ds2 => {
        if (ds2.info.containerId === ds.info.aliasOf) {
          if (ds2.aliases.indexOf(ds) === -1) { ds2.aliases.push(ds); }
          return true;
        }
      })[0];
    }
    const resPoolList = Object.keys(this.inventory.resourcePool).map(key => this.inventory.resourcePool[key]);
    for (const resPool of resPoolList) {
      if (resPool.parent) { continue; }
      resPool.parent = resPool.owner;
    }
    const vAppList = Object.keys(this.inventory.virtualApp).map(key => this.inventory.virtualApp[key]);
    for (const vApp of vAppList) {
      if (vApp.parent) { continue; }
      vApp.parent = vApp.owner;
    }
  }

  async fetchNewMo(iid, moTypeClass?, stubLock = true, ignoreError = false) {
    iid = iidParse(iid);
    if (this.inventoryStubHunts[iid]) { return this.inventoryStubHunts[iid]; }
    this.inventoryStubHunts[iid] = promise<boolean>(async resolve => {
      try {
        const conn = this.pickConnection();
        if (!conn) { return this.errorNoConnection(); }
        const data = await conn.api.getManagedObjectData(iid);
        if (!data) {
          delete this.inventoryStubHunts[iid];
          return resolve(false);
        }
        if (stubLock) { this.lockStubs(); }
        if (!moTypeClass) { moTypeClass = getMoTypeByName(ref(iid).attributes.type); }
        const mo = new moTypeClass(this, data);
        if (stubLock) { this.unlockStubs(); }
        delete this.inventoryStubHunts[iid];
        resolve(false);
      } catch (e) {
        if (ignoreError) { resolve(false); }
        this.ix.pushError(e);
      }
    });
    return this.inventoryStubHunts[iid];
  }

  async get(iid: string, moTypeClass?) {
    iid = iidParse(iid);
    if (this.inventory.byIID[iid]) { return this.inventory.byIID[iid]; }
    let mo = null;
    const moResolver = this.fetchNewMo(iid, moTypeClass);
    await Promise.resolve(moResolver);
    mo = this.inventory.byIID[iid];
    if (!mo) { return null; }
    return mo;
  }

  async getMany(iids: string[]): Promise<Mo.Base[]> {
    const newMos: Mo.Base[] = [];
    const allProms = [];
    this.lockStubs();
    for (let iid of iids) {
      iid = iidParse(iid);
      if (this.inventory.byIID[iid]) {
        newMos.push(this.inventory.byIID[iid]);
        continue;
      }
      const moResolver = this.fetchNewMo(iid, getMoTypeByName(ref(iid).attributes.type), false, true);
      allProms.push(moResolver);
    }
    await Promise.all(allProms);
    this.unlockStubs();
    return newMos;
  }

  getByName(name: string) {
    const list = this.index.byName[name];
    if (!list || !list[0]) { return null; }
    return this.inventory.byIID[list[0]];
  }

  getManyByName(name: string) {
    const list = this.index.byName[name];
    return list ? list : [];
  }

  updateInventoryInterval(seconds: number) {
    this.inventoryCheckInterval = seconds;
    if (this.inventoryCheckInterval < 5) { this.inventoryCheckInterval = 5; }
    return this.inventoryCheckInterval;
  }

  startInventoryWatch(checkIntervalSeconds: number = 10) {
    this.updateInventoryInterval(checkIntervalSeconds);
    if (this.inventoryWatchStarted) { return; }
    this.inventoryWatchStarted = true;
    const checker = async () => {
      if (this.destroyed) { return; }
      if (!this.inventoryWatchStarted) { return; }
      if (this.ix.reconn.defunctState) { return; }
      if (Date.now() - this.inventoryCheckLast > this.inventoryCheckInterval * 1000) {
        const pendingKeys = Object.keys(this.inventoryStubHunts);
        if (pendingKeys.length > 0) {
          const proms = [];
          for (const pendingKey of pendingKeys) { proms.push(this.inventoryStubHunts[pendingKey]); }
          await PromUtil.allSettled(proms);
        }
        this.inventoryCheckLast = Date.now();
        try { this.checkInventory(); } catch (e) { this.ix.pushError(e); }
      }
    };
    Tasks.addForeground(this, `inventory-watch`, checker, 1000);
    // this.inventoryWatchChecker = interval(1000, checker);
    // this.inventoryWatchChecker.lifecycle.managedBy(this);
  }

  stopInventoryWatch() {
    if (!this.inventoryWatchStarted) { return; }
    this.inventoryWatchStarted = false;
    this.inventoryWatchChecker.destroy();
    this.inventoryWatchChecker = null;
  }

  async refreshComputeResourceInfo() {
    const allProms = [];
    for (const clusterId of Object.keys(this.inventory.computeResource)) {
      allProms.push(this.inventory.computeResource[clusterId].refresh());
      break;
    }
    for (const hostId of Object.keys(this.inventory.hostSystem)) {
      allProms.push(this.inventory.hostSystem[hostId].refresh());
    }
    const results = await PromUtil.allSettled(allProms);
    return results.filter(r => r).length === results.length;
  }

  async recalculateUtilization(noRefresh = false): Promise<VsphereDatacenterUtilizationSummary> {
    if (!noRefresh) {
      const refreshAllSuccess = await this.refreshComputeResourceInfo();
      if (!refreshAllSuccess) { return null; }
    }
    const hostCount = Object.keys(this.inventory.hostSystem).length;
    const vmCount = Object.keys(this.inventory.virtualMachine).length;
    const u9n: VsphereDatacenterUtilizationSummary = {
      t: Date.now(), hostCount, vmCount,
      percent: { cpu: 0, mem: 0, disk: 0 },
      overview: {
        totalCpu: 0, consumedCpu: 0, percentCpu: 0,
        totalMem: 0, consumedMem: 0, percentMem: 0,
        totalStorage: 0, consumedStorage: 0, percentStorage: 0,
      },
      clusterSummary: null,
      hostStats: [],
      storageStats: [],
      history: this.u9nWithHistory ? this.u9nHistory : null
    };
    // for (const clusterId of Object.keys(this.inventory.computeResource)) {
    //   const cluster = this.inventory.computeResource[clusterId];
    //   u9n.clusterSummary = cluster.summary;
    //   if (u9n.clusterSummary) {
    //     u9n.overview.totalCpu = parseInt(u9n.clusterSummary.totalCpu + '', 10) / 1000; // in GHz
    //     u9n.overview.totalMem = parseInt(u9n.clusterSummary.totalMemory + '', 10) / Unit.GiB; // in GiB
    //   }
    //   break;
    // }
    for (const hostId of Object.keys(this.inventory.hostSystem)) {
      const host = this.inventory.hostSystem[hostId];
      if (!host || !host.summary) { continue; }
      const hw = host.summary.hardware;
      u9n.overview.totalCpu += hw.cpuMhz * hw.numCpuCores / 1000;
      u9n.overview.totalMem += parseInt(hw.memorySize, 10) / Unit.GiB;
      if (host.summary.quickStats) {
        u9n.overview.consumedCpu += host.summary.quickStats.overallCpuUsage / 1000;
        u9n.overview.consumedMem += host.summary.quickStats.overallMemoryUsage / 1024;
      }
      u9n.hostStats.push({ iid: hostId, name: host.name, networksCount: host.network.length, stats: host.summary.quickStats });
    }
    for (const dsId of Object.keys(this.inventory.datastore)) {
      const ds = this.inventory.datastore[dsId];
      if (!ds || !ds.info) { continue; }
      if (ds.info.containerId === ds.info.aliasOf) {
        const capBytes = parseInt(ds.summary.capacity, 10);
        const freeBytes = parseInt(ds.summary.freeSpace, 10);
        const usedBytes = capBytes - freeBytes;
        u9n.overview.totalStorage += capBytes / Unit.GiB;
        u9n.overview.consumedStorage += usedBytes / Unit.GiB;
      }
      u9n.storageStats.push({ iid: dsId, name: ds.name, info: ds.info, summary: ds.summary });
    }
    let cpuPercent = u9n.overview.consumedCpu / u9n.overview.totalCpu * 100; if (isNaN(cpuPercent)) { cpuPercent = 0; }
    let memPercent = u9n.overview.consumedMem / u9n.overview.totalMem * 100; if (isNaN(memPercent)) { memPercent = 0; }
    let diskPercent = u9n.overview.consumedStorage / u9n.overview.totalStorage * 100; if (isNaN(diskPercent)) { diskPercent = 0; }
    u9n.percent.cpu = u9n.overview.percentCpu = cpuPercent;
    u9n.percent.mem = u9n.overview.percentMem = memPercent;
    u9n.percent.disk = u9n.overview.percentStorage = diskPercent;
    const snapshot: VsphereDatacenterUtilizationSnapshotData = [
      Math.floor(Date.now() / 1000) % 2592000, /** ts */
      u9n.hostCount, /** h */
      u9n.vmCount, /** v */
      parseFloat(numFormat(u9n.percent.cpu)), /** c */
      parseFloat(numFormat(u9n.percent.mem)), /** m */
      parseFloat(numFormat(u9n.percent.disk)), /** s */
    ];
    if (snapshot[3] === null || snapshot[4] === null || snapshot[5] === null) {
      this.ix.pushError(new Error(`utilization snapshot has null values`), 2);
      return null;
    }
    this.u9nHistory.push(snapshot);
    if (this.u9nHistory.length > this.u9nHistoryMaxKept) { this.u9nHistory.shift(); }
    this.utilization = u9n;
    this.u9nSerialized = JSON.stringify(u9n);
    this.emitEvent(this.u9nUpdate$, u9n);
    return u9n;
  }

  updateU9nCheckInterval(seconds: number) {
    this.u9nCheckInterval = seconds;
    if (this.u9nCheckInterval < 5) { this.u9nCheckInterval = 5; }
    return this.u9nCheckInterval;
  }

  startUtilizationWatch(checkIntervalSeconds: number = 60) {
    this.updateU9nCheckInterval(checkIntervalSeconds);
    if (this.u9nWatchStarted) { return; }
    this.u9nWatchStarted = true;
    const checker = async () => {
      if (this.destroyed) { return; }
      if (!this.u9nWatchStarted) { return; }
      if (this.ix.reconn.defunctState) { return; }
      if (Date.now() - this.u9nCheckLast > this.u9nCheckInterval * 1000) {
        this.u9nCheckLast = Date.now();
        let u9nResult;
        try { u9nResult = this.recalculateUtilization(); } catch (e) { this.ix.pushError(e); }
        if (!u9nResult) { this.ix.pushError(new Error(`utilization calculation has errors`), 2); }
      }
    };
    Tasks.addForeground(this, 'utilization-watch', checker, 1000);
  }

  stopUtilizationWatch() {
    if (!this.u9nWatchStarted) { return; }
    this.u9nWatchStarted = false;
    this.u9nWatchChecker.destroy();
    this.u9nWatchChecker = null;
  }

  errorNoConnection(resolve?: (...a: any[]) => any) {
    this.ix.pushError(new Error(`vCenter '${this.key}' has no usable connections`), 2);
    if (resolve) { resolve(null); }
    return null;
  }

  emitEvent<T = any>(obs: Observable<VsphereDatacenterEvent<T>>, eventData?: any) {
    const rxObj = this.rx<VsphereDatacenterEvent<T>>(null, { findNameFromThisObservable: obs });
    if (!rxObj) {
      this.debugOutput('Unrecognized event type from: ' + new Error('').stack);
      return false;
    }
    const eventType = rxObj.name;
    if (!eventData) { eventData = {}; }
    const sourceMo = eventData.target && eventData.target instanceof Mo.Base ? eventData.target : null;
    const e: VsphereDatacenterEvent = {
      eventTime: Date.now(),
      eventType,
      sourceKey: this.key,
      data: eventData,
      moData: sourceMo ? sourceMo.$src : null,
    };
    rxObj.next(e);
    if (eventType !== 'datacenter_generic') { this.rx<any>('datacenter_generic')?.next(e); }
    for (const hook of this.webhooks) {
      if (!hook.events) { continue; }
      let matched = false;
      if (hook.events.all || hook.events[eventType]) {
        matched = true;
      } else {
        const eventMatchers = Object.keys(hook.events);
        for (const eventMatcher of eventMatchers) { if (eventType.startsWith(eventMatcher)) { matched = true; break; } }
      }
      if (!matched) { continue; }
      const postObj = { headers: null };
      if (hook.headers) { postObj.headers = hook.headers; }
      if (hook.data) { e.extra = hook.data; }
      try {
        axios.post(hook.url, e, postObj).then(_ => {}).catch(_ => {});
      } catch (e) { this.debugOutput(e); }
    }
    return true;
  }

  debugOutput(message: string | Error, keepOnly = false) {
    if (this.verbose) {
      if (this.jsonLogs) {
        if (message instanceof Error) {
          const d = {t: Date.now(), src: this.ix.base, key: this.key, msg: `${message.stack ? message.stack : message.message}`};
          if (keepOnly) { VsphereDatacenter.keptData.push({type: 'error', data: d}); } else {
            // tslint:disable-next-line: no-console
            console.log(JSON.stringify(d));
          }
        } else {
          const d = {t: Date.now(), src: this.ix.base, key: this.key, msg: `${message}`};
          if (keepOnly) { VsphereDatacenter.keptData.push({type: 'error', data: d}); } else {
            // tslint:disable-next-line: no-console
            console.log(JSON.stringify(d));
          }
        }
      } else {
        const date = new Date();
        const dateStr = date.toISOString();
        const locTime = localTime(date);
        if (message instanceof Error) {
          const d = `[${dateStr}] [${locTime}] [${this.ix.base}] [${this.key}] :: ${message.stack ? message.stack : message.message}`;
          if (keepOnly) { VsphereDatacenter.keptData.push({type: 'message', data: d}); } else {
            // tslint:disable-next-line: no-console
            console.log(d);
          }
        } else {
          const d = `[${dateStr}] [${locTime}] [${this.ix.base}] [${this.key}] :: ${message}`;
          if (keepOnly) { VsphereDatacenter.keptData.push({type: 'message', data: d}); } else {
            // tslint:disable-next-line: no-console
            console.log(d);
          }
        }
      }
    }
  }

  debugKeep(message: string | Error) { this.debugOutput(message, true); }

  readyResetState() {
    this.ready = false;
    if (!this.readyResolver) {
      this.readyPromise = new Promise<boolean>(resolve =>  this.readyResolver = resolve);
    }
  }

  readyFulfill(value: boolean) {
    if (this.readyResolver) {
      this.readyResolver(value);
      this.readyResolver = null;
    }
  }

}

function localTime(date: Date) {
  let hours = date.getHours();
  const minutes = date.getMinutes();
  const ampm = hours >= 12 ? 'PM' : 'AM';
  hours = hours % 12;
  hours = hours ? hours : 12;
  const hourStr = hours < 10 ? '0' + hours : hours;
  const minuteStr = minutes < 10 ? '0' + minutes : minutes;
  return hourStr + ':' + minuteStr + ' ' + ampm;
}


export class VsphereInfra extends ix.Entity {
  static granted = false;
  static grant() { VsphereInfra.granted = true; }

  src: string; // metadata info on how/why this was craeted
  datacenterData: {[key: string]: VsphereDatacenter} = {};
  connections: [];
  behavior: VsphereBehaviorController;

  constructor(lcParent: ix.Entity) {
    super('vinfra');
    if (lcParent) { lcParent.lifecycle.manage(this); }
    this.behavior = new VsphereBehaviorController(this);
  }

  getDatacenter(target: VsphereTarget): Promise<VsphereDatacenter> {
    return promise<VsphereDatacenter>(async resolve => {
      const vcenter = await VsphereDatacenter.fromTargetInfo(target, this);
      resolve(vcenter);
      if (target.watch) {
        try {
          vcenter.startInventoryWatch();
          vcenter.startUtilizationWatch();
        } catch (e) {
          // no error output since first listen will fail with no connection present.
        }
      }
      try { await vcenter.initializeConnection(); } catch (e) { vcenter.ix.pushError(e); }
      if (!target.skipFetch) {
        try {
          await vcenter.getInventory();
          await vcenter.recalculateUtilization(true);
          vcenter.u9nCheckLast = Date.now();
        } catch (e) { vcenter.ix.pushError(e); }
      }
    });
  }

}

export class VsphereBehaviorController extends ix.Entity {
  target: VsphereInfra;
  verbose = false;
  jsonLogs = false;
  readonlyMode = false;
  useSdkCache = true;
  connection = {
    timeout: 30
  };
  reconnect = new ix.ConfigSource(ix.ReconnEntityBehavior, { restoreCheckInterval: 300 });
  webhooks: VsphereEventWebhook[] = [];
  constructor(target: VsphereInfra) {
    super('vinfra-controller');
    this.target = target;
    this.lifecycle.managedBy(target);
  }
  get dcMap() { return VsphereDatacenter.registry; }
  setVerbose(value = true) {
    value = value ? true : false;
    this.verbose = value;
    for (const key of Object.keys(this.dcMap)) { this.dcMap[key].verbose = value; }
  }
  setJsonLogs(value = true) {
    value = value ? true : false;
    this.jsonLogs = value;
    for (const key of Object.keys(this.dcMap)) { this.dcMap[key].jsonLogs = value; }
  }
  setReadonlyMode(value = true) {
    value = value ? true : false;
    this.readonlyMode = value;
    for (const key of Object.keys(this.dcMap)) { this.dcMap[key].readonlyMode = value; }
  }
  globallySetTlsUnchecked() { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; }
  globallySetTlsChecked() { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1'; }
  addWebhook(webhookDef: VsphereEventWebhook) { this.webhooks.push(webhookDef); }
  removeWebhook(webhookName: string) {
    let i = -1; for (const hook of this.webhooks) { ++i; if (hook.name === webhookName) { break; } }
    if (i >= 0) { this.webhooks.splice(i, 1); }
  }
}

function numFormat(n: number) {
  if (n < 1) {
    return n.toFixed(2);
  } else if (n < 10) {
    return n.toFixed(1);
  } else {
    return n.toFixed(0);
  }
}

async function retry(timeout: number, fn: (...a) => Promise<any>) {
  return promise<boolean>(async (resolve, reject) => {
    let finished = false;
    fn().then(r => {
      if (finished) { return; }
      finished = true;
      resolve(true);
    }).catch(e => {
      if (finished) { return; }
      finished = true;
      resolve(false);
    });
    setTimeout(() => {
      if (finished) { return; }
      finished = true;
      resolve(false);
    }, timeout * 1000);
  });
}