Merge pull request #81 from jeromekelleher/fix-blosc-errors

Fixing blosc encode error handling

Author: alimanfoo

docs/release.rst

@@ -40,6 +40,11 @@ Release notes
   some codecs, and also simplifies the implementation of some codecs, improving
   code readability and maintainability. By :user:`John Kirkham <jakirkham>` and
   :user:`Alistair Miles <alimanfoo>`; :issue:`119`, :issue:`121`, :issue:`128`.
+  
+* Improvements to handling of errors in the :class:`numcodecs.blosc.Blosc` and 
+  :class:`numcodecs.lz4.LZ4` codecs when the maximum allowed size of an input 
+  buffer is exceeded. By :user:`Jerome Kelleher <jeromekelleher>`, :issue:`80`,
+  :issue:`81`.
 
 
 .. _release_0.5.5:

numcodecs/blosc.c

@@ -16,7 +16,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING
 
 from .compat_ext cimport Buffer
 from .compat_ext import Buffer
-from .compat import PY2, text_type
+from .compat import PY2, text_type, ensure_contiguous_ndarray
 from .abc import Codec
 
 
@@ -248,7 +248,8 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
         char *source_ptr
         char *dest_ptr
         Buffer source_buffer
-        size_t nbytes, cbytes, itemsize
+        size_t nbytes, itemsize
+        int cbytes
         bytes dest
 
     # check valid cname early
@@ -365,7 +366,8 @@ def decompress(source, dest=None):
         dest_ptr = PyBytes_AS_STRING(dest)
         dest_nbytes = nbytes
     else:
-        dest_buffer = Buffer(dest, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
+        arr = ensure_contiguous_ndarray(dest)
+        dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
         dest_ptr = dest_buffer.ptr
         dest_nbytes = dest_buffer.nbytes
 
@@ -472,6 +474,7 @@ class Blosc(Codec):
     SHUFFLE = SHUFFLE
     BITSHUFFLE = BITSHUFFLE
     AUTOSHUFFLE = AUTOSHUFFLE
+    max_buffer_size = 2**31 - 1
 
     def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=AUTOBLOCKS):
         self.cname = cname
@@ -484,9 +487,11 @@ class Blosc(Codec):
         self.blocksize = blocksize
 
     def encode(self, buf):
+        buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
         return compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize)
 
     def decode(self, buf, out=None):
+        buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
         return decompress(buf, out)
 
     def __repr__(self):

numcodecs/blosc.pyx

@@ -90,15 +90,20 @@ def ensure_ndarray(buf):
     return arr
 
 
-def ensure_contiguous_ndarray(buf):
+def ensure_contiguous_ndarray(buf, max_buffer_size=None):
     """Convenience function to coerce `buf` to a numpy array, if it is not already a
     numpy array. Also ensures that the returned value exports fully contiguous memory,
-    and supports the new-style buffer interface.
+    and supports the new-style buffer interface. If the optional max_buffer_size is
+    provided, raise a ValueError if the number of bytes consumed by the returned
+    array exceeds this value.
 
     Parameters
     ----------
     buf : array-like or bytes-like
         A numpy array or any object exporting a buffer interface.
+    max_buffer_size : int
+        If specified, the largest allowable value of arr.nbytes, where arr
+        is the retured array.
 
     Returns
     -------
@@ -132,6 +137,10 @@ def ensure_contiguous_ndarray(buf):
     else:
         raise ValueError('an array with contiguous memory is required')
 
+    if max_buffer_size is not None and arr.nbytes > max_buffer_size:
+        msg = "Codec does not support buffers of > {} bytes".format(max_buffer_size)
+        raise ValueError(msg)
+
     return arr