@azure/storage-blob/dist/commonjs/internal-avro/AvroReader.js

Microsoft Azure Storage SDK for JavaScript - Blob

"use strict";
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
Object.defineProperty(exports, "__esModule", { value: true });
exports.AvroReader = void 0;
// TODO: Do a review of non-interfaces
/* eslint-disable @azure/azure-sdk/ts-use-interface-parameters */
const AvroConstants_js_1 = require("./AvroConstants.js");
const AvroParser_js_1 = require("./AvroParser.js");
const utils_common_js_1 = require("./utils/utils.common.js");
class AvroReader {
    _dataStream;
    _headerStream;
    _syncMarker;
    _metadata;
    _itemType;
    _itemsRemainingInBlock;
    // Remembers where we started if partial data stream was provided.
    _initialBlockOffset;
    /// The byte offset within the Avro file (both header and data)
    /// of the start of the current block.
    _blockOffset;
    get blockOffset() {
        return this._blockOffset;
    }
    _objectIndex;
    get objectIndex() {
        return this._objectIndex;
    }
    _initialized;
    constructor(dataStream, headerStream, currentBlockOffset, indexWithinCurrentBlock) {
        this._dataStream = dataStream;
        this._headerStream = headerStream || dataStream;
        this._initialized = false;
        this._blockOffset = currentBlockOffset || 0;
        this._objectIndex = indexWithinCurrentBlock || 0;
        this._initialBlockOffset = currentBlockOffset || 0;
    }
    async initialize(options = {}) {
        const header = await AvroParser_js_1.AvroParser.readFixedBytes(this._headerStream, AvroConstants_js_1.AVRO_INIT_BYTES.length, {
            abortSignal: options.abortSignal,
        });
        if (!(0, utils_common_js_1.arraysEqual)(header, AvroConstants_js_1.AVRO_INIT_BYTES)) {
            throw new Error("Stream is not an Avro file.");
        }
        // File metadata is written as if defined by the following map schema:
        // { "type": "map", "values": "bytes"}
        this._metadata = await AvroParser_js_1.AvroParser.readMap(this._headerStream, AvroParser_js_1.AvroParser.readString, {
            abortSignal: options.abortSignal,
        });
        // Validate codec
        const codec = this._metadata[AvroConstants_js_1.AVRO_CODEC_KEY];
        if (!(codec === undefined || codec === null || codec === "null")) {
            throw new Error("Codecs are not supported");
        }
        // The 16-byte, randomly-generated sync marker for this file.
        this._syncMarker = await AvroParser_js_1.AvroParser.readFixedBytes(this._headerStream, AvroConstants_js_1.AVRO_SYNC_MARKER_SIZE, {
            abortSignal: options.abortSignal,
        });
        // Parse the schema
        const schema = JSON.parse(this._metadata[AvroConstants_js_1.AVRO_SCHEMA_KEY]);
        this._itemType = AvroParser_js_1.AvroType.fromSchema(schema);
        if (this._blockOffset === 0) {
            this._blockOffset = this._initialBlockOffset + this._dataStream.position;
        }
        this._itemsRemainingInBlock = await AvroParser_js_1.AvroParser.readLong(this._dataStream, {
            abortSignal: options.abortSignal,
        });
        // skip block length
        await AvroParser_js_1.AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });
        this._initialized = true;
        if (this._objectIndex && this._objectIndex > 0) {
            for (let i = 0; i < this._objectIndex; i++) {
                await this._itemType.read(this._dataStream, { abortSignal: options.abortSignal });
                this._itemsRemainingInBlock--;
            }
        }
    }
    hasNext() {
        return !this._initialized || this._itemsRemainingInBlock > 0;
    }
    async *parseObjects(options = {}) {
        if (!this._initialized) {
            await this.initialize(options);
        }
        while (this.hasNext()) {
            const result = await this._itemType.read(this._dataStream, {
                abortSignal: options.abortSignal,
            });
            this._itemsRemainingInBlock--;
            this._objectIndex++;
            if (this._itemsRemainingInBlock === 0) {
                const marker = await AvroParser_js_1.AvroParser.readFixedBytes(this._dataStream, AvroConstants_js_1.AVRO_SYNC_MARKER_SIZE, {
                    abortSignal: options.abortSignal,
                });
                this._blockOffset = this._initialBlockOffset + this._dataStream.position;
                this._objectIndex = 0;
                if (!(0, utils_common_js_1.arraysEqual)(this._syncMarker, marker)) {
                    throw new Error("Stream is not a valid Avro file.");
                }
                try {
                    this._itemsRemainingInBlock = await AvroParser_js_1.AvroParser.readLong(this._dataStream, {
                        abortSignal: options.abortSignal,
                    });
                }
                catch {
                    // We hit the end of the stream.
                    this._itemsRemainingInBlock = 0;
                }
                if (this._itemsRemainingInBlock > 0) {
                    // Ignore block size
                    await AvroParser_js_1.AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });
                }
            }
            yield result;
        }
    }
}
exports.AvroReader = AvroReader;
//# sourceMappingURL=AvroReader.js.map