supastash/dist/utils/sync/pullFromRemote/pullData.js

Offline-first sync engine for Supabase in React Native using SQLite

import { getSupastashConfig } from "../../../core/config";
import log, { logWarn } from "../../logs";
import { supabaseClientErr } from "../../supabaseClientErr";
import { getLastCreatedInfo, updateLastCreatedInfo, } from "./getLastCreatedInfo";
import { getLastPulledInfo, updateLastPulledInfo } from "./getLastPulledInfo";
import isValidFilter from "./validateFilters";
const DEFAULT_MAX_PULL_ATTEMPTS = 150;
let timesPulled = new Map();
let lastPulled = new Map();
const RANDOM_OLD_DATE = "2000-01-01T00:00:00Z";
/**
 * Pulls data from the remote database for a given table
 * @param table - The table to pull data from
 * @returns The data from the table
 */
export async function pullData(table, filters) {
    const lastSyncedAt = await getLastPulledInfo(table);
    const lastCreatedAt = await getLastCreatedInfo(table);
    const supabase = getSupastashConfig().supabaseClient;
    if (!supabase)
        throw new Error(`No supabase client found: ${supabaseClientErr}`);
    let filteredLastCreatedQuery = supabase
        .from(table)
        .select("*")
        .gte("created_at", lastCreatedAt || RANDOM_OLD_DATE)
        .is("deleted_at", null)
        .order("updated_at", { ascending: false, nullsFirst: false });
    let filteredLastSyncedQuery = supabase
        .from(table)
        .select("*")
        .gte("updated_at", lastSyncedAt || RANDOM_OLD_DATE)
        .is("deleted_at", null)
        .order("updated_at", { ascending: false, nullsFirst: false });
    for (const filter of filters || []) {
        if (!isValidFilter([filter])) {
            throw new Error(`Invalid syncFilter: ${JSON.stringify(filter)} for table ${table}`);
        }
        filteredLastCreatedQuery = filteredLastCreatedQuery[filter.operator](filter.column, filter.value);
        filteredLastSyncedQuery = filteredLastSyncedQuery[filter.operator](filter.column, filter.value);
    }
    // Fetch records where created_at >= lastCreatedAt OR updated_at >= lastSyncedAt
    const { data: lastCreatedData, error: lastCreatedError, } = await filteredLastCreatedQuery;
    const { data: lastSyncedData, error: lastSyncedError, } = await filteredLastSyncedQuery;
    if (lastCreatedError || lastSyncedError) {
        log(`[Supastash] Error fetching from ${table}:`, lastCreatedError?.message || lastSyncedError?.message);
        return null;
    }
    const allRows = [...(lastCreatedData ?? []), ...(lastSyncedData ?? [])];
    const merged = {};
    for (const row of allRows) {
        if (!row.id) {
            logWarn(`[Supastash] Skipped row without id from "${table}":`, row);
            continue;
        }
        const id = row.id;
        merged[id] = row;
    }
    const data = Object.values(merged);
    if (!data || data.length === 0) {
        timesPulled.set(table, (timesPulled.get(table) || 0) + 1);
        if ((timesPulled.get(table) || 0) >= DEFAULT_MAX_PULL_ATTEMPTS) {
            const timeSinceLastPull = Date.now() - (lastPulled.get(table) || 0);
            lastPulled.set(table, Date.now());
            log(`[Supastash] No updates for ${table} at ${lastSyncedAt} (times pulled: ${timesPulled.get(table)}) in the last ${timeSinceLastPull}ms`);
            timesPulled.set(table, 0);
        }
        return null;
    }
    log(`Received ${data.length} updates for ${table}`);
    // Update the sync status tables with the latest timestamps
    const createdMaxDate = getMaxCreatedDate(lastCreatedData ?? []);
    const updatedMaxDate = getMaxUpdatedDate(lastSyncedData ?? []);
    if (updatedMaxDate) {
        await updateLastPulledInfo(table, updatedMaxDate);
    }
    if (createdMaxDate) {
        await updateLastCreatedInfo(table, createdMaxDate);
    }
    return data;
}
export function getMaxCreatedDate(data) {
    if (!data || data.length === 0)
        return null;
    const createdColumn = "created_at";
    let maxDate = RANDOM_OLD_DATE;
    for (const item of data) {
        const createdValue = item[createdColumn];
        if (typeof createdValue === "string" && !isNaN(Date.parse(createdValue))) {
            const createdTime = new Date(createdValue).getTime();
            if (createdTime > new Date(maxDate).getTime()) {
                maxDate = new Date(createdTime).toISOString();
            }
        }
    }
    return maxDate === RANDOM_OLD_DATE ? null : maxDate;
}
export function getMaxUpdatedDate(data) {
    if (!data || data.length === 0)
        return null;
    const updatedColumn = "updated_at";
    let maxDate = RANDOM_OLD_DATE;
    for (const item of data) {
        const updatedValue = item[updatedColumn];
        if (typeof updatedValue === "string" && !isNaN(Date.parse(updatedValue))) {
            const updatedTime = new Date(updatedValue).getTime();
            if (updatedTime > new Date(maxDate).getTime()) {
                maxDate = new Date(updatedTime).toISOString();
            }
        }
    }
    return maxDate === RANDOM_OLD_DATE ? null : maxDate;
}