cpp: add build-time flags to make compression dependencies optional (#…

…1013)

### Public-Facing Changes

Adds two new build-time flags to optionally disable `zstd` and `lz4`
support. These are `MCAP_COMPRESSION_NO_ZSTD` and
`MCAP_COMPRESSION_NO_LZ4`.

When reading MCAP files, if the reader encounters a known compression
method but is compiled without support for it, it will yield
`Status::UnsupportedCompression`.


### Description

Based on #1008

A new test binary is added to CI, which exercises these build-time
flags.

---------

Co-authored-by: Alexander Sherikov <alexander@sherikov.net>

cpp/Makefile

@@ -42,11 +42,12 @@ bench-host:
 
 .PHONY: test
 test: dev-image
-	docker run -t --rm -v $(CURDIR):/src mcap_cpp_dev ./test/build/Debug/bin/unit-tests
+	docker run -t --rm -v $(CURDIR):/src mcap_cpp_dev make test-host
 
 .PHONY: test-host
 test-host:
 	./test/build/Debug/bin/unit-tests
+	./test/build/Debug/bin/unit-tests-nocompress
 
 .PHONY: hdoc-build
 hdoc-build:

cpp/README.md

@@ -83,6 +83,9 @@ following dependencies:
 - [lz4](https://lz4.github.io/lz4/) (tested with [lz4/1.9.3](https://conan.io/center/lz4))
 - [zstd](https://facebook.github.io/zstd/) (tested with [zstd/1.5.2](https://conan.io/center/zstd))
 
+If your project does not need `lz4` or `zstd` support, you can optionally disable these by defining
+`MCAP_COMPRESSION_NO_LZ4` or `MCAP_COMPRESSION_NO_ZSTD` respectively.
+
 To simplify installation of dependencies, the [Conan](https://conan.io/) package
 manager can be used with the included
 [conanfile.py](https://github.com/foxglove/mcap/blob/main/cpp/mcap/conanfile.py).

cpp/bench/conanfile.py

@@ -4,7 +4,7 @@
 class McapBenchmarksConan(ConanFile):
     settings = "os", "compiler", "build_type", "arch"
     generators = "cmake"
-    requires = "benchmark/1.7.0", "mcap/1.2.1"
+    requires = "benchmark/1.7.0", "mcap/1.3.0"
 
     def build(self):
         cmake = CMake(self)

cpp/build-docs.sh

@@ -4,7 +4,7 @@ set -e
 
 conan config init
 
-conan editable add ./mcap mcap/1.2.1
+conan editable add ./mcap mcap/1.3.0
 conan install docs --install-folder docs/build/Release \
   -s compiler.cppstd=17 -s build_type=Release --build missing
 

cpp/build.sh

@@ -4,7 +4,7 @@ set -e
 
 conan config init
 
-conan editable add ./mcap mcap/1.2.1
+conan editable add ./mcap mcap/1.3.0
 conan install test --install-folder test/build/Debug \
   -s compiler.cppstd=17 -s build_type=Debug --build missing
 

cpp/docs/conanfile.py

@@ -4,7 +4,7 @@
 class McapDocsConan(ConanFile):
     settings = "os", "compiler", "build_type", "arch"
     generators = "cmake"
-    requires = "mcap/1.2.1"
+    requires = "mcap/1.3.0"
 
     def build(self):
         cmake = CMake(self)

cpp/examples/conanfile.py

@@ -5,7 +5,7 @@ class McapExamplesConan(ConanFile):
     settings = "os", "compiler", "build_type", "arch"
     generators = "cmake"
     requires = [
-        "mcap/1.2.1",
+        "mcap/1.3.0",
         "protobuf/3.21.1",
         "nlohmann_json/3.10.5",
         "catch2/2.13.8",

cpp/mcap/conanfile.py

@@ -3,7 +3,7 @@
 
 class McapConan(ConanFile):
     name = "mcap"
-    version = "1.2.1"
+    version = "1.3.0"
     url = "https://github.com/foxglove/mcap"
     homepage = "https://github.com/foxglove/mcap"
     description = "A C++ implementation of the MCAP file format"

cpp/mcap/include/mcap/errors.hpp

@@ -27,6 +27,7 @@ enum class StatusCode {
   MissingStatistics,
   InvalidMessageReadOptions,
   NoMessageIndexesAvailable,
+  UnsupportedCompression,
 };
 
 /**
@@ -98,6 +99,9 @@ struct [[nodiscard]] Status {
       case StatusCode::NoMessageIndexesAvailable:
         message = "file has no message indices";
         break;
+      case StatusCode::UnsupportedCompression:
+        message = "unsupported compression";
+        break;
       default:
         message = "unknown";
         break;

cpp/mcap/include/mcap/reader.hpp

@@ -150,6 +150,7 @@ class MCAP_PUBLIC BufferReader final : public ICompressedReader {
   uint64_t size_;
 };
 
+#ifndef MCAP_COMPRESSION_NO_ZSTD
 /**
  * @brief ICompressedReader implementation that decompresses Zstandard
  * (https://facebook.github.io/zstd/) data.
@@ -183,7 +184,9 @@ class MCAP_PUBLIC ZStdReader final : public ICompressedReader {
   Status status_;
   ByteArray uncompressedData_;
 };
+#endif
 
+#ifndef MCAP_COMPRESSION_NO_LZ4
 /**
  * @brief ICompressedReader implementation that decompresses LZ4
  * (https://lz4.github.io/lz4/) data.
@@ -222,6 +225,7 @@ class MCAP_PUBLIC LZ4Reader final : public ICompressedReader {
   uint64_t compressedSize_;
   uint64_t uncompressedSize_;
 };
+#endif
 
 struct LinearMessageView;
 
@@ -539,8 +543,12 @@ struct MCAP_PUBLIC TypedChunkReader {
   RecordReader reader_;
   Status status_;
   BufferReader uncompressedReader_;
+#ifndef MCAP_COMPRESSION_NO_LZ4
   LZ4Reader lz4Reader_;
+#endif
+#ifndef MCAP_COMPRESSION_NO_ZSTD
   ZStdReader zstdReader_;
+#endif
 };
 
 /**
@@ -627,7 +635,9 @@ struct MCAP_PUBLIC IndexedMessageReader {
   Status status_;
   McapReader& mcapReader_;
   RecordReader recordReader_;
+#ifndef MCAP_COMPRESSION_NO_LZ4
   LZ4Reader lz4Reader_;
+#endif
   ReadMessageOptions options_;
   std::unordered_set<ChannelId> selectedChannels_;
   std::function<void(const Message&, RecordOffset)> onMessage_;

cpp/mcap/include/mcap/reader.inl

@@ -1,9 +1,13 @@
 #include "internal.hpp"
 #include <algorithm>
 #include <cassert>
-#include <lz4frame.h>
-#include <zstd.h>
-#include <zstd_errors.h>
+#ifndef MCAP_COMPRESSION_NO_LZ4
+#  include <lz4frame.h>
+#endif
+#ifndef MCAP_COMPRESSION_NO_ZSTD
+#  include <zstd.h>
+#  include <zstd_errors.h>
+#endif
 
 namespace mcap {
 
@@ -119,6 +123,7 @@ uint64_t FileStreamReader::read(std::byte** output, uint64_t offset, uint64_t si
 
 // LZ4Reader ///////////////////////////////////////////////////////////////////
 
+#ifndef MCAP_COMPRESSION_NO_LZ4
 LZ4Reader::LZ4Reader() {
   const LZ4F_errorCode_t err =
     LZ4F_createDecompressionContext((LZ4F_dctx**)&decompressionContext_, LZ4F_VERSION);
@@ -206,9 +211,11 @@ Status LZ4Reader::decompressAll(const std::byte* data, uint64_t compressedSize,
   }
   return result;
 }
+#endif
 
 // ZStdReader //////////////////////////////////////////////////////////////////
 
+#ifndef MCAP_COMPRESSION_NO_ZSTD
 void ZStdReader::reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) {
   status_ = DecompressAll(data, size, uncompressedSize, &uncompressedData_);
 }
@@ -255,6 +262,7 @@ Status ZStdReader::DecompressAll(const std::byte* data, uint64_t compressedSize,
   }
   return result;
 }
+#endif
 
 // McapReader //////////////////////////////////////////////////////////////////
 
@@ -1251,10 +1259,27 @@ TypedChunkReader::TypedChunkReader()
     , status_{StatusCode::Success} {}
 
 void TypedChunkReader::reset(const Chunk& chunk, Compression compression) {
-  ICompressedReader* decompressor =
-    (compression == Compression::None)  ? static_cast<ICompressedReader*>(&uncompressedReader_)
-    : (compression == Compression::Lz4) ? static_cast<ICompressedReader*>(&lz4Reader_)
-                                        : static_cast<ICompressedReader*>(&zstdReader_);
+  ICompressedReader* decompressor;
+
+  switch (compression) {
+#ifndef MCAP_COMPRESSION_NO_LZ4
+    case Compression::Lz4:
+      decompressor = static_cast<ICompressedReader*>(&lz4Reader_);
+      break;
+#endif
+#ifndef MCAP_COMPRESSION_NO_ZSTD
+    case Compression::Zstd:
+      decompressor = static_cast<ICompressedReader*>(&zstdReader_);
+      break;
+#endif
+    case Compression::None:
+      decompressor = static_cast<ICompressedReader*>(&uncompressedReader_);
+      break;
+    default:
+      status_ = Status(StatusCode::UnsupportedCompression,
+                       internal::StrCat("unsupported compression: ", chunk.compression));
+      return;
+  }
   decompressor->reset(chunk.records, chunk.compressedSize, chunk.uncompressedSize);
   reader_.reset(*decompressor, 0, decompressor->size());
   status_ = decompressor->status();
@@ -1604,7 +1629,8 @@ LinearMessageView::Iterator::Iterator(LinearMessageView& view)
   }
 }
 
-LinearMessageView::Iterator::Impl::Impl(LinearMessageView& view) : view_(view) {
+LinearMessageView::Iterator::Impl::Impl(LinearMessageView& view)
+    : view_(view) {
   auto dataStart = view.dataStart_;
   auto dataEnd = view.dataEnd_;
   auto readMessageOptions = view.readMessageOptions_;
@@ -1658,16 +1684,18 @@ void LinearMessageView::Iterator::Impl::onMessage(const Message& message, Record
 
   auto& channel = *maybeChannel;
   // make sure the message is on the right topic
-  if (view_.readMessageOptions_.topicFilter && !view_.readMessageOptions_.topicFilter(channel.topic)) {
+  if (view_.readMessageOptions_.topicFilter &&
+      !view_.readMessageOptions_.topicFilter(channel.topic)) {
     return;
   }
   SchemaPtr maybeSchema;
   if (channel.schemaId != 0) {
     maybeSchema = view_.mcapReader_.schema(channel.schemaId);
     if (!maybeSchema) {
-      view_.onProblem_(Status{StatusCode::InvalidSchemaId,
-                        internal::StrCat("channel ", channel.id, " (", channel.topic,
-                                         ") references missing schema id ", channel.schemaId)});
+      view_.onProblem_(
+        Status{StatusCode::InvalidSchemaId,
+               internal::StrCat("channel ", channel.id, " (", channel.topic,
+                                ") references missing schema id ", channel.schemaId)});
       return;
     }
   }
@@ -1840,14 +1868,21 @@ void IndexedMessageReader::decompressChunk(const Chunk& chunk,
   if (*compression == Compression::None) {
     slot.decompressedChunk.insert(slot.decompressedChunk.end(), &chunk.records[0],
                                   &chunk.records[chunk.uncompressedSize]);
-  } else if (*compression == Compression::Lz4) {
+  }
+#ifndef MCAP_COMPRESSION_NO_LZ4
+  else if (*compression == Compression::Lz4) {
     status_ = lz4Reader_.decompressAll(chunk.records, chunk.compressedSize, chunk.uncompressedSize,
                                        &slot.decompressedChunk);
-  } else if (*compression == Compression::Zstd) {
+  }
+#endif
+#ifndef MCAP_COMPRESSION_NO_ZSTD
+  else if (*compression == Compression::Zstd) {
     status_ = ZStdReader::DecompressAll(chunk.records, chunk.compressedSize, chunk.uncompressedSize,
                                         &slot.decompressedChunk);
-  } else {
-    status_ = Status(StatusCode::UnrecognizedCompression,
+  }
+#endif
+  else {
+    status_ = Status(StatusCode::UnsupportedCompression,
                      internal::StrCat("unhandled compression: ", chunk.compression));
   }
 }

cpp/mcap/include/mcap/types.hpp

@@ -13,7 +13,7 @@
 
 namespace mcap {
 
-#define MCAP_LIBRARY_VERSION "1.2.1"
+#define MCAP_LIBRARY_VERSION "1.3.0"
 
 using SchemaId = uint16_t;
 using ChannelId = uint16_t;

cpp/mcap/include/mcap/writer.hpp

@@ -9,7 +9,9 @@
 #include <vector>
 
 // Forward declaration
+#ifndef MCAP_COMPRESSION_NO_ZSTD
 struct ZSTD_CCtx_s;
+#endif
 
 namespace mcap {
 
@@ -251,6 +253,7 @@ class MCAP_PUBLIC BufferWriter final : public IChunkWriter {
   std::vector<std::byte> buffer_;
 };
 
+#ifndef MCAP_COMPRESSION_NO_LZ4
 /**
  * @brief An in-memory IChunkWriter implementation that holds data in a
  * temporary buffer before flushing to an LZ4-compressed buffer.
@@ -273,7 +276,9 @@ class MCAP_PUBLIC LZ4Writer final : public IChunkWriter {
   std::vector<std::byte> compressedBuffer_;
   CompressionLevel compressionLevel_;
 };
+#endif
 
+#ifndef MCAP_COMPRESSION_NO_ZSTD
 /**
  * @brief An in-memory IChunkWriter implementation that holds data in a
  * temporary buffer before flushing to an ZStandard-compressed buffer.
@@ -297,6 +302,7 @@ class MCAP_PUBLIC ZStdWriter final : public IChunkWriter {
   std::vector<std::byte> compressedBuffer_;
   ZSTD_CCtx_s* zstdContext_ = nullptr;
 };
+#endif
 
 /**
  * @brief Provides a write interface to an MCAP file.
@@ -445,8 +451,12 @@ class MCAP_PUBLIC McapWriter final {
   std::unique_ptr<FileWriter> fileOutput_;
   std::unique_ptr<StreamWriter> streamOutput_;
   std::unique_ptr<BufferWriter> uncompressedChunk_;
+#ifndef MCAP_COMPRESSION_NO_LZ4
   std::unique_ptr<LZ4Writer> lz4Chunk_;
+#endif
+#ifndef MCAP_COMPRESSION_NO_ZSTD
   std::unique_ptr<ZStdWriter> zstdChunk_;
+#endif
   std::vector<Schema> schemas_;
   std::vector<Channel> channels_;
   std::vector<AttachmentIndex> attachmentIndex_;