@foxglove/ulog
Version:
PX4 ULog file reader
396 lines • 17.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ULog = void 0;
exports.computeTimetampOffset = computeTimetampOffset;
const ChunkedReader_1 = require("./ChunkedReader");
const definition_1 = require("./definition");
const enums_1 = require("./enums");
const findRange_1 = require("./findRange");
const hex_1 = require("./hex");
const parse_1 = require("./parse");
const readMessage_1 = require("./readMessage");
const MAGIC = [0x55, 0x4c, 0x6f, 0x67, 0x01, 0x12, 0x35];
/**
* The synthetic message id for log messages.
*
* When entering a log message in an IndexEntry we tag it with this message id so we can identify
* log messages in the readMessage function if the `includeLogs` option is true.
*/
const LogMessageId = -1;
class ULog {
#filelike;
#chunkSize;
// These members are only populated after open()
#dataEnd;
#header;
#appendedOffsets;
#subscriptions = new Map();
#timeIndex;
#dataMessageCounts;
#logMessageCount;
#dataTimeRange;
constructor(filelike, opts = {}) {
this.#filelike = filelike;
this.#chunkSize = opts.chunkSize;
}
get header() {
return this.#header;
}
/**
* Return a map of message ids to their corresponding subscription
*/
get subscriptions() {
return this.#subscriptions;
}
async open() {
await this.#filelike.open();
const reader = new ChunkedReader_1.ChunkedReader(this.#filelike, this.#chunkSize);
const data = await reader.readBytes(8);
for (let i = 0; i < MAGIC.length; i++) {
if (data[i] !== MAGIC[i]) {
throw new Error(`Invalid ULog header: ${(0, hex_1.toHex)(data)}`);
}
}
const version = data[7];
const timestamp = await reader.readUint64();
const information = new Map();
const parameters = new Map();
const definitions = new Map();
let flagBits;
while (!(await isDataSectionStart(reader))) {
const message = await (0, readMessage_1.readRawMessage)(reader, this.#dataEnd);
if (!message) {
break;
}
switch (message.type) {
case enums_1.MessageType.FlagBits: {
flagBits = message;
break;
}
case enums_1.MessageType.Information: {
const infoMsg = message;
const field = (0, definition_1.parseFieldDefinition)(infoMsg.key);
if (isValidInfoField(field)) {
const value = infoMsg.value;
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
const parsed = (0, parse_1.parseBasicFieldValue)(field, view);
information.set(field.name, parsed);
}
break;
}
case enums_1.MessageType.InformationMulti: {
const infoMultiMsg = message;
const field = (0, definition_1.parseFieldDefinition)(infoMultiMsg.key);
if (isValidInfoField(field)) {
let array = information.get(infoMultiMsg.key);
if (!Array.isArray(array)) {
array = [];
information.set(infoMultiMsg.key, array);
}
const value = infoMultiMsg.value;
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
const parsed = (0, parse_1.parseBasicFieldValue)(field, view);
array.push(parsed);
}
break;
}
case enums_1.MessageType.FormatDefinition: {
const formatMsg = message;
const msgdef = (0, definition_1.parseMessageDefinition)(formatMsg.format);
if (msgdef) {
definitions.set(msgdef.name, msgdef);
}
else {
throw new Error(`oops: ${formatMsg.format}`);
}
break;
}
case enums_1.MessageType.Parameter: {
const paramMsg = message;
const field = (0, definition_1.parseFieldDefinition)(paramMsg.key);
if (isValidParameter(field)) {
const value = paramMsg.value;
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
const parsed = (0, parse_1.parseBasicFieldValue)(field, view);
parameters.set(field.name, { value: parsed, defaultTypes: 0 });
}
break;
}
case enums_1.MessageType.ParameterDefault: {
const paramMsg = message;
const field = (0, definition_1.parseFieldDefinition)(paramMsg.key);
if (isValidParameter(field)) {
const value = paramMsg.value;
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
const parsed = (0, parse_1.parseBasicFieldValue)(field, view);
if (parsed != undefined) {
parameters.set(field.name, { value: parsed, defaultTypes: paramMsg.defaultTypes });
}
}
break;
}
case enums_1.MessageType.Unknown:
case enums_1.MessageType.AddLogged:
case enums_1.MessageType.RemoveLogged:
case enums_1.MessageType.Data:
case enums_1.MessageType.Log:
case enums_1.MessageType.LogTagged:
case enums_1.MessageType.Synchronization:
case enums_1.MessageType.Dropout:
default:
throw new Error(`Unrecognized message type ${message.type}`);
}
}
// File offsets are stored as 64-bit unsigned integers, but we cast to Number here which safely
// stores up to 53-bit integers. This supports ulogs up to 8192 TB in length
const appendedOffsets = flagBits?.appendedOffsets ?? [0n, 0n, 0n];
this.#appendedOffsets = appendedOffsets.map((n) => Number(n));
const firstOffset = this.#appendedOffsets[0];
this.#dataEnd = reader.size();
if (firstOffset > 0 && firstOffset < this.#dataEnd) {
this.#dataEnd = firstOffset;
}
this.#header = { version, timestamp, flagBits, information, parameters, definitions };
await this.#createIndex(reader);
}
async *readMessages(opts = {}) {
const includeLogs = opts.includeLogs ?? true;
const msgIds = opts.msgIds;
const reader = new ChunkedReader_1.ChunkedReader(this.#filelike, this.#chunkSize);
const timeIndex = this.#timeIndex;
if (timeIndex == undefined) {
throw new Error(`Cannot readMessages before createIndex`);
}
if (timeIndex.length === 0) {
return;
}
const startTime = opts.startTime ?? timeIndex[0][0];
const endTime = opts.endTime ?? timeIndex[timeIndex.length - 1][0];
const range = (0, findRange_1.findRange)(timeIndex, startTime, endTime);
if (range == undefined) {
return;
}
const startIndex = range[0];
const endIndex = range[1];
// The number of records between the start and end index. We will iterate this many times
// reading records. Which record we start from and the increment depends on the reverse flag.
const diffIndex = endIndex - startIndex;
// When reading in reverse we will start reading at endIndex and decrement towards startIndex
const increment = opts.reverse === true ? -1 : 1;
let idx = opts.reverse === true ? endIndex : startIndex;
for (let i = 0; i <= diffIndex; i++, idx += increment) {
const [_timestamp, offset, msgId] = timeIndex[idx];
if (includeLogs && msgId === LogMessageId) {
reader.seekTo(offset);
const msg = await this.#readParsedMessage(reader);
if (msg) {
yield msg;
continue;
}
}
// If there are message ids specified, then only yield if the locator msgId matches
if (msgIds != undefined) {
if (msgId == undefined || !msgIds.has(msgId)) {
continue;
}
}
reader.seekTo(offset);
const msg = await this.#readParsedMessage(reader);
if (msg) {
yield msg;
}
}
}
messageCount() {
return this.#timeIndex?.length;
}
dataMessageCounts() {
return this.#dataMessageCounts;
}
logCount() {
return this.#logMessageCount;
}
timeRange() {
return this.#dataTimeRange;
}
async #createIndex(reader) {
if (!this.#header || this.#dataEnd == undefined) {
throw new Error(`Cannot read before open`);
}
const dataEnd = this.#dataEnd;
const timeIndex = [];
const dataCounts = new Map();
let minTimestamp;
let maxTimestamp = 0n;
let logMessageCount = 0;
// The offset of the timestamp field for a data message. The offset is from the start of the
// message data.
const timestampFieldOffsets = new Map();
for (;;) {
const offset = reader.position();
// If there is less than 3 bytes left in the file, we can't read a message header so we're done
if (dataEnd - offset < 3) {
break;
}
const header = await (0, readMessage_1.readMessageHeader)(reader);
const type = header.type;
// If there's not enough bytes in the file to read the message then we end indexing
if (reader.position() + header.size > dataEnd) {
break;
}
if (type === enums_1.MessageType.AddLogged) {
// read the AddLogged message from the reader data
const addLogged = await (0, readMessage_1.readMessageAddLogged)(reader, header);
this.#handleSubscription(addLogged);
timeIndex.push([maxTimestamp, offset, undefined]);
}
else if (type === enums_1.MessageType.Log) {
const logMsg = await (0, readMessage_1.readMessageLog)(reader, header);
if (minTimestamp == undefined || logMsg.timestamp < minTimestamp) {
minTimestamp = logMsg.timestamp;
}
if (logMsg.timestamp > maxTimestamp) {
maxTimestamp = logMsg.timestamp;
}
timeIndex.push([logMsg.timestamp, offset, LogMessageId]);
logMessageCount++;
}
else if (type === enums_1.MessageType.LogTagged) {
const logMsg = await (0, readMessage_1.readMessageLogTagged)(reader, header);
if (minTimestamp == undefined || logMsg.timestamp < minTimestamp) {
minTimestamp = logMsg.timestamp;
}
if (logMsg.timestamp > maxTimestamp) {
maxTimestamp = logMsg.timestamp;
}
timeIndex.push([logMsg.timestamp, offset, LogMessageId]);
logMessageCount++;
}
else if (type === enums_1.MessageType.Data) {
const dataMsg = await (0, readMessage_1.readMessageData)(reader, header);
let timestampOffset = timestampFieldOffsets.get(dataMsg.msgId);
if (timestampOffset == undefined) {
// We don't yet have a timestamp offset for this message id so we compute it
const definition = this.#subscriptions.get(dataMsg.msgId);
if (!definition) {
const msgPos = reader.position() - header.size - 3;
throw new Error(`Unknown msg_id ${dataMsg.msgId} for ${header.size} byte 'D' message at offset ${msgPos}`);
}
timestampOffset = computeTimetampOffset(definition, this.#header.definitions);
timestampFieldOffsets.set(dataMsg.msgId, timestampOffset);
}
const view = new DataView(dataMsg.data.buffer, dataMsg.data.byteOffset, dataMsg.data.byteLength);
// we know the timestamp offset so we can parse the timestamp
const timestamp = view.getBigUint64(timestampOffset, true);
if (minTimestamp == undefined || timestamp < minTimestamp) {
minTimestamp = timestamp;
}
if (timestamp > maxTimestamp) {
maxTimestamp = timestamp;
}
timeIndex.push([timestamp, offset, dataMsg.msgId]);
dataCounts.set(dataMsg.msgId, (dataCounts.get(dataMsg.msgId) ?? 0) + 1);
}
else {
timeIndex.push([maxTimestamp, offset, undefined]);
// Skip past this message
reader.seek(header.size);
}
}
this.#timeIndex = timeIndex.sort(sortTimeIndex);
this.#dataMessageCounts = dataCounts;
this.#logMessageCount = logMessageCount;
this.#dataTimeRange = minTimestamp != undefined ? [minTimestamp, maxTimestamp] : undefined;
}
async #readParsedMessage(reader) {
if (!this.#header) {
throw new Error(`Cannot read before open`);
}
const rawMessage = await (0, readMessage_1.readRawMessage)(reader, this.#dataEnd);
if (!rawMessage) {
return undefined;
}
if (rawMessage.type !== enums_1.MessageType.Data) {
return rawMessage;
}
const dataMsg = rawMessage;
const definition = this.#subscriptions.get(dataMsg.msgId);
if (!definition) {
const msgPos = reader.position() - rawMessage.size - 3;
throw new Error(`Unknown msg_id ${dataMsg.msgId} for ${rawMessage.size} byte 'D' message at offset ${msgPos}`);
}
const data = dataMsg.data;
const value = (0, parse_1.parseMessage)(definition, this.#header.definitions, reader.view(), data.byteOffset);
const parsed = {
size: dataMsg.size,
type: enums_1.MessageType.Data,
msgId: dataMsg.msgId,
data,
value,
};
return parsed;
}
#handleSubscription(subscribe) {
const definition = this.#header?.definitions.get(subscribe.messageName);
if (!definition) {
throw new Error(`AddLogged unknown message_name: ${subscribe.messageName}`);
}
this.#subscriptions.set(subscribe.msgId, { ...definition, multiId: subscribe.multiId });
}
}
exports.ULog = ULog;
async function isDataSectionStart(reader) {
const type = (await reader.peekUint8(2));
switch (type) {
case enums_1.MessageType.AddLogged:
case enums_1.MessageType.RemoveLogged:
case enums_1.MessageType.Data:
case enums_1.MessageType.Log:
case enums_1.MessageType.LogTagged:
case enums_1.MessageType.Synchronization:
case enums_1.MessageType.Dropout:
return true;
case enums_1.MessageType.Unknown:
case enums_1.MessageType.FlagBits:
case enums_1.MessageType.Information:
case enums_1.MessageType.InformationMulti:
case enums_1.MessageType.FormatDefinition:
case enums_1.MessageType.Parameter:
case enums_1.MessageType.ParameterDefault:
return false;
}
}
function isValidInfoField(field) {
return field?.isComplex === false;
}
function isValidParameter(field) {
return Boolean(field && (field.type === "int32_t" || field.type === "float") && field.arrayLength == undefined);
}
function sortTimeIndex(a, b) {
const timestampA = a[0];
const timestampB = b[0];
// If the timestamps are the same, sort by the offset within the file
if (timestampA === timestampB) {
const indexA = a[1];
const indexB = b[1];
return indexA - indexB;
}
return Number(timestampA - timestampB);
}
function computeTimetampOffset(definition, definitions) {
let curOffset = 0;
for (const field of definition.fields) {
if (field.name.startsWith("_")) {
continue;
}
if (field.name === "timestamp") {
if (field.type !== "uint64_t") {
throw new Error(`Message "${definition.name}" has a timestamp field with a non-uint64_t type`);
}
return curOffset;
}
curOffset += (0, parse_1.fieldSize)(field, definitions) * (field.arrayLength ?? 1);
}
throw new Error(`Message "${definition.name}" is missing a timestamp field`);
}
//# sourceMappingURL=ULog.js.map