compressor: replace snappy with cramjam

The python-snappy library is no longer maintained (see [1]). Cramjam is a drop-in replacement
that allows for a considerable speedup both in the compression and decompression
tasks. The relevant benchmark for snappy's framed format [2].

[1]: intake/python-snappy#124
[2]: https://github.com/milesgranger/pyrus-cramjam/blob/33c0516374fb9726ddcb82c5dfbe86be96d2cd35/cramjam-python/benchmarks/README.md

.github/workflows/build.yml

@@ -70,7 +70,6 @@ jobs:
 
       - id: dependencies
         run: |
-          sudo apt-get install -y libsnappy-dev
           pip install -r requirements.txt
           pip install -r requirements.dev.txt
 

.pylintrc

@@ -1,3 +1,7 @@
+[MAIN]
+
+extension-pkg-allow-list=cramjam
+
 [MESSAGES CONTROL]
 disable=
     bad-option-value,

mypy.ini

@@ -14,14 +14,13 @@ ignore_missing_imports = True
 [mypy-azure.common.*]
 ignore_missing_imports = True
 
-[mypy-dataclasses.*]
+[mypy-cramjam.*]
 ignore_missing_imports = True
 
-
-[mypy-oauth2client.*]
+[mypy-dataclasses.*]
 ignore_missing_imports = True
 
-[mypy-snappy.*]
+[mypy-oauth2client.*]
 ignore_missing_imports = True
 
 [mypy-swiftclient.*]

rohmu.spec

@@ -13,7 +13,8 @@ Requires:       python3-cryptography >= 1.6
 Requires:       python3-dateutil
 Requires:       python3-pydantic
 Requires:       python3-requests
-Requires:       python3-snappy
+# Requires:       python3-snappy
+# TODO: Create python3-cramjam
 Requires:       python3-zstandard
 BuildRequires:  python3-devel
 BuildRequires:  python3-flake8

rohmu/compressor.py

@@ -14,9 +14,9 @@
 import lzma
 
 try:
-    import snappy
+    import cramjam
 except ImportError:
-    snappy = None  # type: ignore
+    cramjam = None  # type: ignore
 
 try:
     import zstandard as zstd
@@ -50,7 +50,7 @@ def __init__(self, src_fp: HasRead, algorithm: str, level: int = 0) -> None:
         if algorithm == "lzma":
             self._compressor = lzma.LZMACompressor(lzma.FORMAT_XZ, -1, level, None)
         elif algorithm == "snappy":
-            self._compressor = snappy.StreamCompressor()
+            self._compressor = cramjam.snappy
         elif algorithm == "zstd":
             self._compressor = zstd.ZstdCompressor(level=level).compressobj()
         else:
@@ -87,7 +87,7 @@ def __init__(self, next_sink: HasWrite, compression_algorithm: str):
 
     def _create_decompressor(self, alg: str) -> Decompressor:
         if alg == "snappy":
-            return snappy.StreamDecompressor()
+            return cramjam.snappy
         elif alg == "lzma":
             return lzma.LZMADecompressor()
         elif alg == "zstd":

rohmu/snappyfile.py

@@ -12,22 +12,22 @@
 import io
 
 try:
-    import snappy
+    import cramjam
 except ImportError:
-    snappy = None  # type: ignore
+    cramjam = None  # type: ignore
 
 
 class SnappyFile(FileWrap):
     def __init__(self, next_fp: FileLike, mode: str) -> None:
-        if snappy is None:
+        if cramjam is None:
             raise io.UnsupportedOperation("Snappy is not available")
 
         if mode == "rb":
-            self.decr = snappy.StreamDecompressor()
+            self.decr = cramjam.snappy
             self.encr = None
         elif mode == "wb":
             self.decr = None
-            self.encr = snappy.StreamCompressor()
+            self.encr = cramjam.snappy
         else:
             raise io.UnsupportedOperation("unsupported mode for SnappyFile")
 

setup.cfg

@@ -26,14 +26,14 @@ packages = find:
 install_requires =
     azure-storage-blob >= 2.1.0
     botocore
+    cramjam >= 2.7.0
     cryptography
     google-api-python-client
     httplib2
     oauth2client
     paramiko
     pydantic
     python-dateutil
-    python-snappy
     requests
     zstandard
     typing_extensions >= 3.10, < 5