clarity-js/src/data/upload.ts

An analytics library that uses web page interactions to generate aggregated insights

import type { UploadCallback } from "@clarity-types/core";
import {
    BooleanFlag,
    Check,
    Code,
    Constant,
    type EncodedPayload,
    Event,
    Metric,
    Setting,
    Severity,
    type Token,
    type Transit,
    type UploadData,
    XMLReadyState,
} from "@clarity-types/data";
import * as clarity from "@src/clarity";
import config from "@src/core/config";
import measure from "@src/core/measure";
import { report } from "@src/core/report";
import { time } from "@src/core/time";
import { clearTimeout, setTimeout } from "@src/core/timeout";
import compress from "@src/data/compress";
import encode from "@src/data/encode";
import * as envelope from "@src/data/envelope";
import * as extract from "@src/data/extract";
import * as data from "@src/data/index";
import * as limit from "@src/data/limit";
import * as metadata from "@src/data/metadata";
import * as metric from "@src/data/metric";
import * as ping from "@src/data/ping";
import { signalsEvent } from "@src/data/signal";
import * as internal from "@src/diagnostic/internal";
import * as timeline from "@src/interaction/timeline";
import * as region from "@src/layout/region";
import * as style from "@src/layout/style";

let discoverBytes = 0;
let playbackBytes = 0;
let playback: string[];
let analysis: string[];
let timeout: number = null;
let transit: Transit;
let active: boolean;
let queuedTime = 0;
let leanLimit = false;
export let track: UploadData;

export function start(): void {
    active = true;
    discoverBytes = 0;
    playbackBytes = 0;
    leanLimit = false;
    queuedTime = 0;
    playback = [];
    analysis = [];
    transit = {};
    track = null;
}

export function queue(tokens: Token[], transmit = true): void {
    if (active) {
        const now = time();
        const type = tokens.length > 1 ? tokens[1] : null;
        const event = JSON.stringify(tokens);

        if (!config.lean) {
            leanLimit = false;
        } else if (!leanLimit && playbackBytes + event.length > Setting.PlaybackBytesLimit) {
            internal.log(Code.LeanLimit, Severity.Info);
            leanLimit = true;
        }

        switch (type) {
            // biome-ignore lint/suspicious/noFallthroughSwitchClause: we want discover bytes to also count as playback bytes
            case Event.Discover:
                if (leanLimit) {
                    break;
                }
                discoverBytes += event.length;
            case Event.Box:
            case Event.Mutation:
            case Event.Snapshot:
            case Event.StyleSheetAdoption:
            case Event.StyleSheetUpdate:
                if (leanLimit) {
                    break;
                }
                playbackBytes += event.length;
                playback.push(event);
                break;
            default:
                analysis.push(event);
                break;
        }

        // Increment event count metric
        metric.count(Metric.EventCount);

        // Following two checks are precautionary and act as a fail safe mechanism to get out of unexpected situations.
        // Check 1: If for any reason the upload hasn't happened after waiting for 2x the config.delay time,
        // reset the timer. This allows Clarity to attempt an upload again.
        const gap = delay();
        if (now - queuedTime > gap * 2) {
            clearTimeout(timeout);
            timeout = null;
        }

        // Transmit Check: When transmit is set to true (default), it indicates that we should schedule an upload
        // However, in certain scenarios - like metric calculation - which are triggered as part of an existing upload
        // We enrich the data going out with the existing upload. In these cases, call to upload comes with 'transmit' set to false.
        if (transmit && timeout === null) {
            if (type !== Event.Ping) {
                ping.reset();
            }
            timeout = setTimeout(upload, gap);
            queuedTime = now;
            limit.check(playbackBytes);
        }
    }
}

export function stop(): void {
    clearTimeout(timeout);
    upload(true);
    discoverBytes = 0;
    playbackBytes = 0;
    leanLimit = false;
    queuedTime = 0;
    playback = [];
    analysis = [];
    transit = {};
    track = null;
    active = false;
}

async function upload(final = false): Promise<void> {
    timeout = null;

    // Check if we can send playback bytes over the wire or not
    // For better instrumentation coverage, we send playback bytes from second sequence onwards
    // And, we only send playback metric when we are able to send the playback bytes back to server
    const sendPlaybackBytes =
        config.lean === false && playbackBytes > 0 && (playbackBytes < Setting.MaxFirstPayloadBytes || envelope.data.sequence > 0);
    if (sendPlaybackBytes) {
        metric.max(Metric.Playback, BooleanFlag.True);
    }

    // CAUTION: Ensure "transmit" is set to false in the queue function for following events
    // Otherwise you run a risk of infinite loop.
    region.compute();
    timeline.compute();
    data.compute();
    style.compute();

    // Treat this as the last payload only if final boolean was explicitly set to true.
    // In real world tests, we noticed that certain third party scripts (e.g. https://www.npmjs.com/package/raven-js)
    // could inject function arguments for internal tracking (likely stack traces for script errors).
    // For these edge cases, we want to ensure that an injected object (e.g. {"key": "value"}) isn't mistaken to be true.
    const last = final === true;
    const e = JSON.stringify(envelope.envelope(last));
    const a = `[${analysis.join()}]`;

    const p = sendPlaybackBytes ? `[${playback.join()}]` : Constant.Empty;
    const encoded: EncodedPayload = { e, a, p };

    // Get the payload ready for sending over the wire
    // We also attempt to compress the payload if it is not the last payload and the browser supports it
    // In all other cases, we continue to send back string value
    const payload = stringify(encoded);
    const zipped = last ? null : await compress(payload);
    metric.sum(Metric.TotalBytes, zipped ? zipped.length : payload.length);
    send(payload, zipped, envelope.data.sequence, last);

    // Clear out events now that payload has been dispatched
    analysis = [];
    if (sendPlaybackBytes) {
        playback = [];
        playbackBytes = 0;
        discoverBytes = 0;
        leanLimit = false;
    }
}

function stringify(encoded: EncodedPayload): string {
    return encoded.p.length > 0 ? `{"e":${encoded.e},"a":${encoded.a},"p":${encoded.p}}` : `{"e":${encoded.e},"a":${encoded.a}}`;
}

function send(payload: string, zipped: Uint8Array, sequence: number, beacon = false): void {
    // Upload data if a valid URL is defined in the config
    if (typeof config.upload === "string") {
        const url = config.upload as string;
        let dispatched = false;

        // If it's the last payload, attempt to upload using sendBeacon first.
        // The advantage to using sendBeacon is that browser can decide to upload asynchronously, improving chances of success
        // However, we don't want to rely on it for every payload, since we have no ability to retry if the upload failed.
        // Also, in case of sendBeacon, we do not have a way to alter HTTP headers and therefore can't send compressed payload
        if (beacon && "sendBeacon" in navigator) {
            try {
                // Navigator needs to be bound to sendBeacon before it is used to avoid errors in some browsers
                dispatched = navigator.sendBeacon.bind(navigator)(url, payload);
                if (dispatched) {
                    done(sequence);
                }
            } catch {
                /* do nothing - and we will automatically fallback to XHR below */
            }
        }

        // Before initiating XHR upload, we check if the data has already been uploaded using sendBeacon
        // There are two cases when dispatched could still be false:
        //   a) It's not the last payload, and therefore we didn't attempt sending sendBeacon
        //   b) It's the last payload, however, we failed to queue sendBeacon call and need to now fall back to XHR.
        //      E.g. if data is over 64KB, several user agents (like Chrome) will reject to queue the sendBeacon call.
        if (dispatched === false) {
            // While tracking payload for retry, we only track string value of the payload to err on the safe side
            // Not all browsers support compression API and the support for it in supported browsers is still experimental
            if (sequence in transit) {
                transit[sequence].attempts++;
            } else {
                transit[sequence] = { data: payload, attempts: 1 };
            }
            const xhr = new XMLHttpRequest();
            xhr.open("POST", url, true);
            xhr.timeout = Setting.UploadTimeout;
            xhr.ontimeout = () => {
                report(new Error(`${Constant.Timeout} : ${url}`));
            };
            if (sequence !== null) {
                xhr.onreadystatechange = (): void => {
                    measure(check)(xhr, sequence);
                };
            }
            xhr.withCredentials = true;
            if (zipped) {
                // If we do have valid compressed array, send it with appropriate HTTP headers so server can decode it appropriately
                xhr.setRequestHeader(Constant.Accept, Constant.ClarityGzip);
                xhr.send(zipped);
            } else {
                // In all other cases, continue sending string back to the server
                xhr.send(payload);
            }
        }
    } else if (config.upload) {
        const callback = config.upload as UploadCallback;
        callback(payload);
        done(sequence);
    }
}

function check(xhr: XMLHttpRequest, sequence: number): void {
    const transitData = transit[sequence];
    if (xhr && xhr.readyState === XMLReadyState.Done && transitData) {
        // Attempt send payload again (as configured in settings) if we do not receive a success (2XX) response code back from the server
        if ((xhr.status < 200 || xhr.status > 208) && transitData.attempts <= Setting.RetryLimit) {
            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error
            if (xhr.status >= 400 && xhr.status < 500) {
                // In case of a 4XX response from the server, we bail out instead of trying again
                limit.trigger(Check.Server);
            } else {
                // Browser will send status = 0 when it refuses to put network request over the wire
                // This could happen for several reasons, couple of known ones are:
                //    1: Browsers block upload because of content security policy violation
                //    2: Safari will terminate pending XHR requests with status code 0 if the user navigates away from the page
                // In any case, we switch the upload URL to fallback configuration (if available) before re-trying one more time
                if (xhr.status === 0) {
                    config.upload = config.fallback ? config.fallback : config.upload;
                }
                // In all other cases, re-attempt sending the same data
                // For retry we always fallback to string payload, even though we may have attempted
                // sending zipped payload earlier
                send(transitData.data, null, sequence);
            }
        } else {
            track = { sequence, attempts: transitData.attempts, status: xhr.status };
            // Send back an event only if we were not successful in our first attempt
            if (transitData.attempts > 1) {
                encode(Event.Upload);
            }
            // Handle response if it was a 200 response with a valid body
            if (xhr.status === 200 && xhr.responseText) {
                response(xhr.responseText);
            }
            // If we exhausted our retries then trigger Clarity's shutdown for this page since the data will be incomplete
            if (xhr.status === 0) {
                // And, right before we terminate the session, we will attempt one last time to see if we can use
                // different transport option (sendBeacon vs. XHR) to get this data to the server for analysis purposes
                send(transitData.data, null, sequence, true);
                limit.trigger(Check.Retry);
            }
            // Signal that this request completed successfully
            if (xhr.status >= 200 && xhr.status <= 208) {
                done(sequence);
            }
            // Stop tracking this payload now that it's all done
            delete transit[sequence];
        }
    }
}

function done(sequence: number): void {
    // If we everything went successfully, and it is the first sequence, save this session for future reference
    if (sequence === 1) {
        metadata.save();
        metadata.callback();
    }
}

function delay(): number {
    // Progressively increase delay as we continue to send more payloads from the client to the server
    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value
    const gap = config.lean === false && discoverBytes > 0 ? Setting.MinUploadDelay : envelope.data.sequence * config.delay;
    return typeof config.upload === "string" ? Math.max(Math.min(gap, Setting.MaxUploadDelay), Setting.MinUploadDelay) : config.delay;
}

function response(payload: string): void {
    const lines = payload && payload.length > 0 ? payload.split("\n") : [];
    for (const line of lines) {
        const parts = line && line.length > 0 ? line.split(/ (.*)/) : [Constant.Empty];
        switch (parts[0]) {
            case Constant.End:
                // Clear out session storage and end the session so we can start fresh the next time
                limit.trigger(Check.Server);
                break;
            case Constant.Upgrade:
                // Upgrade current session to send back playback information
                clarity.upgrade(Constant.Auto);
                break;
            case Constant.Action:
                // Invoke action callback, if configured and has a valid value
                if (config.action && parts.length > 1) {
                    config.action(parts[1]);
                }
                break;
            case Constant.Extract:
                if (parts.length > 1) {
                    extract.trigger(parts[1]);
                }
                break;
            case Constant.Signal:
                if (parts.length > 1) {
                    signalsEvent(parts[1]);
                }
                break;
        }
    }
}