Skip to content

Commit

Permalink
Add tests for @mcap/core append mode
Browse files Browse the repository at this point in the history
  • Loading branch information
alexern14 committed Nov 21, 2023
1 parent 946559d commit 7039375
Show file tree
Hide file tree
Showing 2 changed files with 214 additions and 1 deletion.
209 changes: 209 additions & 0 deletions typescript/core/src/McapWriter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -344,4 +344,213 @@ describe("McapWriter", () => {
},
]);
});

it("supports append mode", async () => {
const tempBuffer = new TempBuffer();

const writer = new McapWriter({ writable: tempBuffer, chunkSize: 0 });

await writer.start({ library: "", profile: "" });
const channelId = await writer.registerChannel({
topic: "test",
schemaId: 0,
messageEncoding: "json",
metadata: new Map(),
});
await writer.addMessage({
channelId,
data: new Uint8Array(),
sequence: 0,
logTime: 0n,
publishTime: 0n,
});
await writer.addMessage({
channelId,
data: new Uint8Array(),
sequence: 1,
logTime: 1n,
publishTime: 1n,
});
await writer.end();

const writerAppendMode = await McapWriter.InitializeInAppendMode({
writable: tempBuffer,
readable: tempBuffer,
});

await writerAppendMode.addAttachment({
name: "attachment test",
logTime: 0n,
createTime: 0n,
mediaType: "application/json",
data: new TextEncoder().encode(`{"test": "testValue"}`),
});
await writerAppendMode.addMetadata({
name: "metadata test",
metadata: new Map<string, string>([["test", "testValue"]]),
});
await writerAppendMode.end();

const reader = new McapStreamReader();
reader.append(tempBuffer.get());
const records: TypedMcapRecord[] = [];
for (let rec; (rec = reader.nextRecord()); ) {
records.push(rec);
}

expect(records).toEqual<TypedMcapRecord[]>([
{
type: "Header",
library: "",
profile: "",
},
{
type: "Channel",
id: 0,
messageEncoding: "json",
metadata: new Map(),
schemaId: 0,
topic: "test",
},
{
type: "Message",
channelId: 0,
data: new Uint8Array(),
logTime: 0n,
publishTime: 0n,
sequence: 0,
},
{
type: "MessageIndex",
channelId: 0,
records: [[0n, 33n]],
},
{
type: "Message",
channelId: 0,
data: new Uint8Array(),
logTime: 1n,
publishTime: 1n,
sequence: 1,
},
{
type: "MessageIndex",
channelId: 0,
records: [[1n, 0n]],
},
{
type: "Attachment",
name: "attachment test",
logTime: 0n,
createTime: 0n,
mediaType: "application/json",
data: new TextEncoder().encode(`{"test": "testValue"}`),
},
{
type: "Metadata",
name: "metadata test",
metadata: new Map([["test", "testValue"]]),
},
{
type: "DataEnd",
dataSectionCrc: 3215020142,
},
{
type: "Channel",
id: 0,
messageEncoding: "json",
metadata: new Map(),
schemaId: 0,
topic: "test",
},
{
type: "Statistics",
attachmentCount: 1,
channelCount: 1,
channelMessageCounts: new Map([[0, 2n]]),
chunkCount: 2,
messageCount: 2n,
messageEndTime: 1n,
messageStartTime: 0n,
metadataCount: 1,
schemaCount: 0,
},
{
type: "MetadataIndex",
offset: 377n,
length: 51n,
name: "metadata test",
},
{
type: "AttachmentIndex",
offset: 280n,
length: 97n,
logTime: 0n,
createTime: 0n,
dataSize: 21n,
name: "attachment test",
mediaType: "application/json",
},
{
type: "ChunkIndex",
chunkLength: 113n,
chunkStartOffset: 25n,
compressedSize: 64n,
compression: "",
messageEndTime: 0n,
messageIndexLength: 31n,
messageIndexOffsets: new Map([[0, 138n]]),
messageStartTime: 0n,
uncompressedSize: 64n,
},
{
type: "ChunkIndex",
chunkLength: 80n,
chunkStartOffset: 169n,
compressedSize: 31n,
compression: "",
messageEndTime: 1n,
messageIndexLength: 31n,
messageIndexOffsets: new Map([[0, 249n]]),
messageStartTime: 1n,
uncompressedSize: 31n,
},
{
type: "SummaryOffset",
groupLength: 33n,
groupOpcode: Opcode.CHANNEL,
groupStart: 441n,
},
{
type: "SummaryOffset",
groupLength: 65n,
groupOpcode: Opcode.STATISTICS,
groupStart: 474n,
},
{
type: "SummaryOffset",
groupLength: 42n,
groupOpcode: Opcode.METADATA_INDEX,
groupStart: 539n,
},
{
type: "SummaryOffset",
groupLength: 88n,
groupOpcode: Opcode.ATTACHMENT_INDEX,
groupStart: 581n,
},
{
type: "SummaryOffset",
groupLength: 166n,
groupOpcode: Opcode.CHUNK_INDEX,
groupStart: 669n,
},
{
type: "Footer",
summaryCrc: 2655711257,
summaryOffsetStart: 835n,
summaryStart: 441n,
},
]);
});
});
6 changes: 5 additions & 1 deletion typescript/core/src/TempBuffer.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
import { IAppendWritable } from "./IAppendWritable";
import { IWritable } from "./IWritable";
import { IReadable } from "./types";

/**
* In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable.
*/
export class TempBuffer implements IReadable, IWritable {
export class TempBuffer implements IReadable, IWritable, IAppendWritable {
#buffer = new ArrayBuffer(1024);
#size = 0;

position(): bigint {
return BigInt(this.#size);
}
async seek(position: bigint): Promise<void> {
this.#size = Number(position);
}
async write(data: Uint8Array): Promise<void> {
if (this.#size + data.byteLength > this.#buffer.byteLength) {
const newBuffer = new ArrayBuffer(this.#size + data.byteLength);
Expand Down

0 comments on commit 7039375

Please sign in to comment.