@causalfoundry/js-sdk/src/core/CfSender.ts

Causal Foundry WEB SDK (JS/TS)

import {NetworkProxy} from "../drivers/network/typings";
import {debug, Debug} from "../logging";

import {v4 as uuidv4} from "uuid";
import isBrowser from "is-in-browser";

import {
    Event,
    ExceptionObject,
    InternalInfoObject,
    LogIngestorOptions,
    MainExceptionBody,
    QueueCatalogItem,
    QueueItem,
} from "./typings";
import {getAppVersionDetails, getDeviceOSDetails, getNetworkInformation,} from "../drivers/CfSystem";
import {version} from "../../package.json";
import {offsetToISO8601} from "../utils";
import {CatalogItemModel} from "../modules/typings";

export interface ISenderRepository {
    add(userId: string, deviceId: string, event: Event, forceSend: boolean): void;

    flush();
}

export default class CfSender implements ISenderRepository {
    private flushing = false;
    private flushIntervalId = 0;
    private cfNetwork;
    private debug;
    private options;
    private endpointURL;
    private catalogEndpointURL;
    private exceptionsEndPoint = "/ingest/sdk/crash";

    private userId = "";
    private deviceId = "";
    private usingAsInternalWrapper = false;

    private queue: QueueItem[];
    private catalogQueue: QueueCatalogItem[];

    public constructor(cfNetwork: NetworkProxy, options: LogIngestorOptions) {
        this.queue = [];
        this.catalogQueue = [];
        this.cfNetwork = cfNetwork;
        this.options = options;
        this.debug = debug(this.options.debug);

        // to avoid short timeout to save cpu usage
        if (this.options.flushInterval < 1000) {
            this.options.flushInterval = 1000;
        }

        if (
            this.options.cacheEventsInLocalstorage &&
            typeof Storage !== "undefined"
        ) {
            try {
                this.queue = JSON.parse(
                    localStorage.getItem(this.options.cacheEventsKey) || "[]"
                ) as QueueItem[];

                this.catalogQueue = JSON.parse(
                    localStorage.getItem(this.options.cacheCatalogKey) || "[]"
                ) as QueueCatalogItem[];
            } catch (e) {
                this.debug(
                    Debug.Error,
                    `Error getting queue items from cache ${e.message || ""}`
                );
            }
        }

        this.endpointURL = `${this.options.baseUrl}${this.options.ingestPath}`;
        this.catalogEndpointURL = `${this.options.baseUrl}${this.options.dimensionPath}`;

        this.updateQueueCache();

        this.flushIntervalId = +setInterval(() => {
            this.flush();
        }, this.options.flushInterval);
    }

    public flush() {
        if (this.options.pauseSDK) {
            return;
        }

        const flushItems = [...this.queue];
        const flushCatalogItems = [...this.catalogQueue];

        if (flushItems.length === 0 || flushCatalogItems.length === 0) {
            this.debug(Debug.Log, "Skip sending empty list of logs ");
            return;
        }

        const uuids: string[] = flushItems.map((item: QueueItem) => item.uuid);
        const data = flushItems.map((item: QueueItem) => item.event);


        const catalogUUIDS: string[] = flushCatalogItems.map((item: QueueCatalogItem) => item.uuid);
        const catalogData = flushCatalogItems.map((item: QueueCatalogItem) => item.catalog);

        this.flushing = true;

        this.send(data, uuids);
        this.sendCatalog(catalogData, catalogUUIDS);
    }

    public add(
        userId: string,
        deviceId: string,
        event: Event,
        forceSend = false,
        isUsingAsInternalWrapper = false
    ) {
        if (this.options.pauseSDK) {
            return;
        }

        const item: QueueItem = {
            uuid: uuidv4(),
            event,
            time: +new Date(),
            retries: 0,
        };
        this.userId = userId;
        this.deviceId = deviceId;
        this.usingAsInternalWrapper = isUsingAsInternalWrapper;

        const isDuplicate = this.queue.some(
            (queueItem) => queueItem.event === item.event
        );
        if (!isDuplicate) {
            this.queue.push(item);
            this.updateQueueCache();
            // send immediately. In case of error, it was already enqueued
            // so will be tried again in interval tick
            if (forceSend) {
                this.send([item.event], [item.uuid]);
            }
        }
    }

    public addCatalogItem(
        catalog: CatalogItemModel,
        isUsingAsInternalWrapper = false
    ) {
        if (this.options.pauseSDK) {
            return;
        }

        const item: QueueCatalogItem = {
            uuid: uuidv4(),
            catalog,
            retries: 0,
        };
        this.usingAsInternalWrapper = isUsingAsInternalWrapper;

        const isDuplicate = this.catalogQueue.some(
            (queueCatalogItem) => queueCatalogItem.catalog === item.catalog
        );
        if (!isDuplicate) {
            this.catalogQueue.push(item);
            this.updateCatalogQueueCache();
        }
    }

    private removeQueueItemsExceedMaxRetries() {
        this.queue = this.queue.filter(
            (item: QueueItem) => item.retries <= this.options.flushMaxRetries
        );
        this.updateQueueCache();
    }

    private removeQueueCatalogItemsExceedMaxRetries() {
        this.catalogQueue = this.catalogQueue.filter(
            (item: QueueCatalogItem) => item.retries <= this.options.flushMaxRetries
        );
        this.updateCatalogQueueCache();
    }

    private removeFromQueue(uuids: string[]) {
        this.queue = this.queue.filter(
            (item: QueueItem) => uuids.indexOf(item.uuid) === -1
        );
        this.updateQueueCache();
    }

    private removeFromCatalogQueue(uuids: string[]) {
        this.catalogQueue = this.catalogQueue.filter(
            (item: QueueCatalogItem) => uuids.indexOf(item.uuid) === -1
        );
        this.updateCatalogQueueCache();
    }

    private removeIndexFromQueue(indexToRemove: number) {
        this.queue.splice(indexToRemove, 1);
        this.updateQueueCache();
    }

    private removeIndexFromCatalogQueue(indexToRemove: number) {
        this.catalogQueue.splice(indexToRemove, 1);
        this.updateCatalogQueueCache();
    }

    private updateQueueCache() {
        if (
            this.options.cacheEventsInLocalstorage &&
            typeof Storage !== "undefined"
        ) {
            localStorage.setItem(
                this.options.cacheEventsKey,
                JSON.stringify(this.queue)
            );
        }
    }

    private updateCatalogQueueCache() {
        if (
            this.options.cacheEventsInLocalstorage &&
            typeof Storage !== "undefined"
        ) {
            localStorage.setItem(
                this.options.cacheCatalogKey,
                JSON.stringify(this.catalogQueue)
            );
        }
    }

    private addRetries(uuids: string[]) {
        const time = +new Date();
        this.queue
            .filter((item: QueueItem) => uuids.indexOf(item.uuid) !== -1)
            .forEach((item: QueueItem) => {
                item.retries = +item.retries + 1;
                item.lastRetry = time;
            });
        this.updateQueueCache();
    }

    private addCatalogRetries(uuids: string[]) {
        const time = +new Date();
        this.catalogQueue
            .filter((item: QueueCatalogItem) => uuids.indexOf(item.uuid) !== -1)
            .forEach((item: QueueCatalogItem) => {
                item.retries = +item.retries + 1;
                item.lastRetry = time;
            });
        this.updateCatalogQueueCache();
    }

    private send(dataList: Event[], uuids: string[]) {
        if (this.options.pauseSDK) {
            return;
        }
        // Remove duplicates
        const hashEvent = (event: Event) => JSON.stringify(event); // Convert event to string for comparison
        const getUniqueEvents = (data: Event[]): Event[] => {
            const seen = new Set<string>(); // Stores unique event hashes
            return data.filter((event) => {
                const hash = hashEvent(event); // Create a unique identifier
                if (seen.has(hash)) return false; // If duplicate, skip it
                seen.add(hash); // Otherwise, add it to seen
                return true;
            });
        };

        const filteredData = getUniqueEvents(dataList);

        const internalInfoObject: InternalInfoObject = {
            s_id: this.generateSessionID(this.userId),
            sdk: `js/${version}`,
            app_id: isBrowser
                ? document?.location.host || "NO_URL_FOUND"
                : "NO_URL_FOUND",
            app_version: getAppVersionDetails(),
            device_id: this.deviceId,
            device_os: getDeviceOSDetails(),
        };
        return this.cfNetwork
            .send(this.endpointURL, {
                data: filteredData,
                internal: internalInfoObject,
                id: this.userId,
            })
            .then(() => {
                this.debug(Debug.Info, `Flushed ${filteredData.length} events`);
                this.removeFromQueue(uuids);
            })
            .catch((error) => {
                if (error.response.data.data) {
                    // Accessing specific properties from the error response
                    this.removeIndexFromQueue(Number(error.response.data.data));
                }
                this.addRetries(uuids);
                this.debug(Debug.Error, error.message || "");
            })
            .finally(() => {
                this.flushing = false;
                this.removeQueueItemsExceedMaxRetries();
            });
    }

    private sendCatalog(dataList: CatalogItemModel[], uuids: string[]) {
        if (this.options.pauseSDK) {
            return;
        }
        // Remove duplicates
        const hashEvent = (event: CatalogItemModel) => JSON.stringify(event); // Convert event to string for comparison
        const getUniqueEvents = (data: CatalogItemModel[]): CatalogItemModel[] => {
            const seen = new Set<string>(); // Stores unique event hashes
            return data.filter((event) => {
                const hash = hashEvent(event); // Create a unique identifier
                if (seen.has(hash)) return false; // If duplicate, skip it
                seen.add(hash); // Otherwise, add it to seen
                return true;
            });
        };

        const filteredData = getUniqueEvents(dataList);

        return this.cfNetwork
            .send(this.catalogEndpointURL, {
                data: filteredData
            })
            .then(() => {
                this.debug(Debug.Info, `Flushed ${filteredData.length} events`);
                this.removeFromCatalogQueue(uuids);
            })
            .catch((error) => {
                if (error.response.data.data) {
                    // Accessing specific properties from the error response
                    this.removeIndexFromCatalogQueue(Number(error.response.data.data));
                }

                this.addCatalogRetries(uuids);
                this.debug(Debug.Error, error.message || "");
            })
            .finally(() => {
                this.flushing = false;
                this.removeQueueCatalogItemsExceedMaxRetries();
            });
    }

    private generateSessionID(userId: string) {
        // return `${userId}_${startTimeInMillis}_${endTimeInMillis}`;
        return `${userId}_${Date.now()}`;
    }

    private checkDefaultNetworkValue(defaultValue: number): number {
        if (this.usingAsInternalWrapper) {
            return this.getRandomNumberValue(100, 1024);
        }
        return defaultValue;
    }

    private getRandomNumberValue(min: number, max: number): number {
        return Math.floor(Math.random() * (max - min) + min);
    }
}