signalk-parquet/dist/utils/hive-path-builder.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

/**
 * Hive Path Builder
 *
 * Constructs Hive-style partitioned paths for Parquet storage.
 * Target structure: tier=raw/context={ctx}/path={path}/year={year}/day={day}/
 */
export type AggregationTier = 'raw' | '5s' | '60s' | '1h';
export interface HivePath {
    basePath: string;
    tier: AggregationTier;
    context: string;
    signalkPath: string;
    year: number;
    dayOfYear: number;
    fullPath: string;
}
export interface PathParseResult {
    isHive: boolean;
    isFlat: boolean;
    tier?: AggregationTier;
    context?: string;
    signalkPath?: string;
    year?: number;
    dayOfYear?: number;
}
export declare class HivePathBuilder {
    /**
     * Build a Hive-style partitioned path
     */
    buildPath(basePath: string, tier: AggregationTier, context: string, signalkPath: string, timestamp: Date): string;
    /**
     * Build a complete file path including filename
     */
    buildFilePath(basePath: string, tier: AggregationTier, context: string, signalkPath: string, timestamp: Date, filenamePrefix?: string): string;
    /**
     * Parse a path to determine if it's Hive-style or flat
     */
    detectPathStyle(filePath: string): PathParseResult;
    /**
     * Convert a flat-style path to Hive-style
     */
    flatToHive(flatPath: string, basePath: string, tier?: AggregationTier): string | null;
    /**
     * Sanitize a context string for use in a path
     * vessels.urn:mrn:signalk:uuid:xxx -> vessels__urn-mrn-signalk-uuid-xxx
     */
    sanitizeContext(context: string): string;
    /**
     * Unsanitize a context string from a path
     */
    unsanitizeContext(sanitized: string): string;
    /**
     * Sanitize a SignalK path for use in a file path
     * navigation.speedOverGround -> navigation__speedOverGround
     */
    sanitizePath(signalkPath: string): string;
    /**
     * Unsanitize a path from a file path
     */
    unsanitizePath(sanitized: string): string;
    /**
     * Get the day of year (1-366)
     */
    getDayOfYear(date: Date): number;
    /**
     * Create a date from year and day of year
     */
    dateFromDayOfYear(year: number, dayOfYear: number): Date;
    /**
     * Get the glob pattern for finding files in a time range
     */
    getGlobPattern(basePath: string, tier: AggregationTier, context?: string, signalkPath?: string, year?: number, dayOfYear?: number): string;
    /**
     * Get all day directories in a time range
     */
    getDaysInRange(from: Date, to: Date): Array<{
        year: number;
        dayOfYear: number;
    }>;
    /**
     * Build DuckDB-compatible glob pattern for Hive partitions
     */
    buildDuckDBGlob(basePath: string, tier: AggregationTier, context?: string, signalkPath?: string, fromDate?: Date, toDate?: Date): string;
    /**
     * Build S3 URI glob pattern for querying parquet files directly from S3
     * Uses DuckDB's native S3 support with partition pruning
     *
     * @param bucket S3 bucket name
     * @param keyPrefix Optional key prefix (e.g., "marine-data/")
     * @param tier Aggregation tier (raw, 5s, 60s, 1h)
     * @param context SignalK context (e.g., "vessels.urn:mrn:signalk:uuid:xxx")
     * @param signalkPath SignalK path (e.g., "navigation.speedOverGround")
     * @param fromDate Start date for partition pruning
     * @param toDate End date for partition pruning
     * @returns S3 URI pattern for DuckDB read_parquet
     */
    buildS3Glob(bucket: string, keyPrefix: string, tier: AggregationTier, context: string, signalkPath: string, fromDate: Date, toDate: Date): string;
    /**
     * Generate day patterns for S3 partition pruning
     * For short ranges (<= 7 days), lists explicit directories
     * For longer ranges, uses wildcards
     */
    private getDayPatterns;
    /**
     * Build multiple S3 glob patterns for hybrid queries (local + S3)
     * Returns patterns for both before and after a cutoff date
     */
    buildS3GlobsForRange(bucket: string, keyPrefix: string, tier: AggregationTier, context: string, signalkPath: string, fromDate: Date, toDate: Date, cutoffDate: Date): {
        s3Pattern: string | null;
        localPattern: string | null;
    };
    /**
     * Find the earliest date that has local parquet data for a given tier/context/path.
     * Scans year= and day= directories to find the minimum date.
     */
    findEarliestDate(dataDir: string, tier: string, sanitizedContext: string, sanitizedPath: string): Date | null;
}
//# sourceMappingURL=hive-path-builder.d.ts.map