From c7ad54beb0ed703aee2872111474eec725711bd4 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Mon, 20 Jan 2025 22:20:34 -0500 Subject: [PATCH] Use arm64/x86_64 ISA extensions to compute CRC32C --- src/workerd/api/crypto/crc-impl.c++ | 24 ++++++++++++++++++++++++ src/workerd/api/crypto/crypto.c++ | 2 +- src/workerd/api/crypto/endianness.h | 4 ++-- src/workerd/io/BUILD.bazel | 10 ++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++ index 7a3e7f251e6..eb170758759 100644 --- a/src/workerd/api/crypto/crc-impl.c++ +++ b/src/workerd/api/crypto/crc-impl.c++ @@ -55,8 +55,10 @@ constexpr std::array gen_crc_table(T polynomial, bool reflectIn return crcTable; } +#if !(__CRC32__ || __ARM_FEATURE_CRC32) // https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi constexpr auto crc32c_table = gen_crc_table(static_cast(0x1edc6f41), true, true); +#endif // https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme constexpr auto crc64nvme_table = gen_crc_table(static_cast(0xad93d23594c93659), true, true); @@ -67,8 +69,30 @@ uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) { return 0; } crc ^= 0xffffffff; +#if __CRC32__ || __ARM_FEATURE_CRC32 + // Using hardware acceleration, process data in 8-byte chunks. Any remaining bytes are processed + // one-by-one in the main loop. + while (length >= 8) { + // 8-byte unaligned read + uint64_t val = *(uint64_t *)data; +#if __ARM_FEATURE_CRC32 + crc = __builtin_arm_crc32cd(crc, val); +#else + crc = __builtin_ia32_crc32di(crc, val); +#endif + length -= 8; + data += 8; + } +#endif + while (length--) { +#if __ARM_FEATURE_CRC32 + crc = __builtin_arm_crc32cb(crc, *data++); +#elif __CRC32__ + crc = __builtin_ia32_crc32qi(crc, *data++); +#else crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); +#endif } return crc ^ 0xffffffff; } diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++ index a9e20bd4963..f091db9a968 100644 --- a/src/workerd/api/crypto/crypto.c++ +++ b/src/workerd/api/crypto/crypto.c++ @@ -734,7 +734,7 @@ class CRC64NVMEDigestContext final: public DigestContext { kj::Array close() override { auto beValue = htobe64(value); - static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + static_assert(sizeof(value) == sizeof(beValue), "CRC64 digest is not 64 bits?"); auto digest = kj::heapArray(sizeof(beValue)); KJ_DASSERT(digest.size() == sizeof(beValue)); memcpy(digest.begin(), &beValue, sizeof(beValue)); diff --git a/src/workerd/api/crypto/endianness.h b/src/workerd/api/crypto/endianness.h index 09bcb0b3cef..38e19519aa7 100644 --- a/src/workerd/api/crypto/endianness.h +++ b/src/workerd/api/crypto/endianness.h @@ -6,8 +6,8 @@ #include // This file provides cross platform support for endianness conversions. -// It is intended to hide away the poluting includes of system headers to provide the functions -// without poluting the global namespace. +// It is intended to hide away the polluting includes of system headers to provide the functions +// without polluting the global namespace. uint16_t htobe16(uint16_t x); uint16_t htole16(uint16_t x); diff --git a/src/workerd/io/BUILD.bazel b/src/workerd/io/BUILD.bazel index f68bc8cd884..e70079b4860 100644 --- a/src/workerd/io/BUILD.bazel +++ b/src/workerd/io/BUILD.bazel @@ -80,6 +80,16 @@ wd_cc_library( ":set_enable_experimental_webgpu": ["//src/workerd/api/gpu:hdrs"], "//conditions:default": [], }), + # global CPU-specific options are inconvenient to specify with bazel – just set the options we + # need for CRC32C in this target for now. + copts = select({ + "@platforms//cpu:aarch64": [ + "-mcrc", + ], + "@platforms//cpu:x86_64": [ + "-msse4.2", + ], + }), defines = select({ ":set_enable_experimental_webgpu": ["WORKERD_EXPERIMENTAL_ENABLE_WEBGPU"], "//conditions:default": [],