Test error on i386 #48
Description
Two tests of the test suite seems to fail on i386 architectures (see the salsa CI log).
The problem has been spotted on one of the Debian CI systems.
Both issues seems to be linked to the size of the image and the limited resources of the VM used in CI (I do not know details but the amount of memory seems to be quite limited).
Does it make sense to try reduce the size of the image used for the test? ... is it something that can be done easily?
I could try to prepare a patch in the case ...
============================ test session starts ==============================
platform linux -- Python 3.11.1, pytest-7.2.0, pluggy-1.0.0+repack
rootdir: /builds/debian-gis-team/sarsen/debian/output/source_dir/.pybuild/cpython3_3.11/build
collected 30 items
tests/test_10_chunking.py ... [ 10%]
tests/test_10_datamodel.py . [ 13%]
tests/test_10_orbit.py .. [ 20%]
tests/test_10_scene.py ... [ 30%]
tests/test_20_geocoding.py ... [ 40%]
tests/test_20_sentinel1.py FF. [ 50%]
tests/test_30_radiometry.py . [ 53%]
tests/test_50_apps.py ssssssssssss [ 93%]
tests/test_60_main.py . [ 96%]
tests/test_90_sarsen.py . [100%]
=================================== FAILURES ===================================
__________________ test_Sentinel1SarProduct[data_path0-IW/VV] __________________
data_path = PosixPath('/builds/debian-gis-team/sarsen/debian/output/source_dir/.pybuild/cpython3_3.11/build/tests/data/S1B_IW_GRDH_1SDV_20211223T051122_20211223T051147_030148_039993_5371.SAFE')
group = 'IW/VV'
@pytest.mark.parametrize("data_path,group", zip(DATA_PATHS, GROUPS))
def test_Sentinel1SarProduct(data_path: str, group: str) -> None:
res = sentinel1.Sentinel1SarProduct(data_path, group)
assert isinstance(res.measurement, xr.Dataset)
assert isinstance(res.orbit, xr.Dataset)
assert isinstance(res.calibration, xr.Dataset)
if res.product_type == "GRD":
assert isinstance(res.coordinate_conversion, xr.Dataset)
assert res.azimuth_fm_rate is None
assert res.dc_estimate is None
else:
assert res.coordinate_conversion is None
assert isinstance(res.azimuth_fm_rate, xr.Dataset)
assert isinstance(res.dc_estimate, xr.Dataset)
assert res.product_type in {"SLC", "GRD"}
> assert isinstance(res.beta_nought(), xr.DataArray)
tests/test_20_sentinel1.py:39:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sarsen/sentinel1.py:156: in beta_nought
return calibrate_measurement(measurement, self.calibration.betaNought)
sarsen/sentinel1.py:44: in calibrate_measurement
beta_nought = xarray_sentinel.calibrate_intensity(measurement, beta_nought_lut)
/usr/lib/python3/dist-packages/xarray_sentinel/sentinel1.py:842: in calibrate_intensity
amplitude = calibrate_amplitude(digital_number, calibration_lut, **kwargs)
/usr/lib/python3/dist-packages/xarray_sentinel/sentinel1.py:816: in calibrate_amplitude
amplitude = digital_number / calibration
/usr/lib/python3/dist-packages/xarray/core/_typed_ops.py:218: in __truediv__
return self._binary_op(other, operator.truediv)
/usr/lib/python3/dist-packages/xarray/core/dataarray.py:4358: in _binary_op
f(self.variable, other_variable)
/usr/lib/python3/dist-packages/xarray/core/_typed_ops.py:408: in __truediv__
return self._binary_op(other, operator.truediv)
/usr/lib/python3/dist-packages/xarray/core/variable.py:2634: in _binary_op
self_data, other_data, dims = _broadcast_compat_data(self, other)
/usr/lib/python3/dist-packages/xarray/core/variable.py:3123: in _broadcast_compat_data
self_data = new_self.data
/usr/lib/python3/dist-packages/xarray/core/variable.py:435: in data
return self.values
/usr/lib/python3/dist-packages/xarray/core/variable.py:608: in values
return _as_array_or_item(self._data)
/usr/lib/python3/dist-packages/xarray/core/variable.py:314: in _as_array_or_item
data = np.asarray(data)
/usr/lib/python3/dist-packages/xarray/core/indexing.py:653: in __array__
self._ensure_cached()
/usr/lib/python3/dist-packages/xarray/core/indexing.py:650: in _ensure_cached
self.array = NumpyIndexingAdapter(np.asarray(self.array))
/usr/lib/python3/dist-packages/xarray/core/indexing.py:623: in __array__
return np.asarray(self.array, dtype=dtype)
/usr/lib/python3/dist-packages/xarray/core/indexing.py:653: in __array__
self._ensure_cached()
/usr/lib/python3/dist-packages/xarray/core/indexing.py:650: in _ensure_cached
self.array = NumpyIndexingAdapter(np.asarray(self.array))
/usr/lib/python3/dist-packages/xarray/core/indexing.py:623: in __array__
return np.asarray(self.array, dtype=dtype)
/usr/lib/python3/dist-packages/xarray/core/indexing.py:653: in __array__
self._ensure_cached()
/usr/lib/python3/dist-packages/xarray/core/indexing.py:650: in _ensure_cached
self.array = NumpyIndexingAdapter(np.asarray(self.array))
/usr/lib/python3/dist-packages/xarray/core/indexing.py:623: in __array__
return np.asarray(self.array, dtype=dtype)
/usr/lib/python3/dist-packages/xarray/core/indexing.py:524: in __array__
return np.asarray(array[self.key], dtype=None)
/usr/lib/python3/dist-packages/rioxarray/_io.py:423: in __getitem__
return indexing.explicit_indexing_adapter(
/usr/lib/python3/dist-packages/xarray/core/indexing.py:815: in explicit_indexing_adapter
result = raw_indexing_method(raw_key.tuple)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <rioxarray._io.RasterioArrayWrapper object at 0xe665e230>
key = (0, slice(None, None, None), slice(None, None, None))
def _getitem(self, key):
band_key, window, squeeze_axis, np_inds = self._get_indexer(key)
if not band_key or any(start == stop for (start, stop) in window):
# no need to do IO
shape = (len(band_key),) + tuple(stop - start for (start, stop) in window)
out = np.zeros(shape, dtype=self.dtype)
else:
with self.lock:
riods = self.manager.acquire(needs_lock=False)
if self.vrt_params is not None:
riods = WarpedVRT(riods, **self.vrt_params)
out = riods.read(band_key, window=window, masked=self.masked)
if self._unsigned_dtype is not None:
out = out.astype(self._unsigned_dtype)
if self.masked:
> out = np.ma.filled(out.astype(self.dtype), self.fill_value)
E numpy.core._exceptions._ArrayMemoryError: Unable to allocate 1.62 GiB for an array with shape (16705, 26102) and data type float32
/usr/lib/python3/dist-packages/rioxarray/_io.py:404: MemoryError
_________________ test_Sentinel1SarProduct[data_path1-IW1/VV] __________________
data_path = PosixPath('/builds/debian-gis-team/sarsen/debian/output/source_dir/.pybuild/cpython3_3.11/build/tests/data/S1A_IW_SLC__1SDV_20220104T170557_20220104T170624_041314_04E951_F1F1.SAFE')
group = 'IW1/VV'
@pytest.mark.parametrize("data_path,group", zip(DATA_PATHS, GROUPS))
def test_Sentinel1SarProduct(data_path: str, group: str) -> None:
res = sentinel1.Sentinel1SarProduct(data_path, group)
assert isinstance(res.measurement, xr.Dataset)
assert isinstance(res.orbit, xr.Dataset)
assert isinstance(res.calibration, xr.Dataset)
if res.product_type == "GRD":
assert isinstance(res.coordinate_conversion, xr.Dataset)
assert res.azimuth_fm_rate is None
assert res.dc_estimate is None
else:
assert res.coordinate_conversion is None
assert isinstance(res.azimuth_fm_rate, xr.Dataset)
assert isinstance(res.dc_estimate, xr.Dataset)
assert res.product_type in {"SLC", "GRD"}
> assert isinstance(res.beta_nought(), xr.DataArray)
tests/test_20_sentinel1.py:39:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sarsen/sentinel1.py:156: in beta_nought
return calibrate_measurement(measurement, self.calibration.betaNought)
sarsen/sentinel1.py:42: in calibrate_measurement
measurement = xarray_sentinel.mosaic_slc_iw(measurement)
/usr/lib/python3/dist-packages/xarray_sentinel/sentinel1.py:791: in mosaic_slc_iw
return xr.concat(bursts, dim="azimuth_time")
/usr/lib/python3/dist-packages/xarray/core/concat.py:231: in concat
return _dataarray_concat(
/usr/lib/python3/dist-packages/xarray/core/concat.py:657: in _dataarray_concat
ds = _dataset_concat(
/usr/lib/python3/dist-packages/xarray/core/concat.py:584: in _dataset_concat
combined_var = concat_vars(
/usr/lib/python3/dist-packages/xarray/core/variable.py:3186: in concat
return Variable.concat(variables, dim, positions, shortcut, combine_attrs)
/usr/lib/python3/dist-packages/xarray/core/variable.py:2047: in concat
data = duck_array_ops.concatenate(arrays, axis=axis)
/usr/lib/python3/dist-packages/xarray/core/duck_array_ops.py:314: in concatenate
return _concatenate(as_shared_dtype(arrays), axis=axis)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
args = ([array([[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0.+0.j],
[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0... 0.+0.j, 0.+0.j, 0.+0.j],
[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0.+0.j]],
dtype=complex64), ...],)
kwargs = {'axis': 0}
relevant_args = [array([[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0.+0.j],
[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0....., 0.+0.j, 0.+0.j, 0.+0.j],
[0.+0.j, 0.+0.j, 0.+0.j, ..., 0.+0.j, 0.+0.j, 0.+0.j]],
dtype=complex64), ...]
> ???
E ValueError: array is too big; `arr.size * arr.dtype.itemsize` is larger than the maximum possible size.
<__array_function__ internals>:180: ValueError
=============================== warnings summary ===============================
tests/test_20_geocoding.py::test_secant_method
/builds/debian-gis-team/sarsen/debian/output/source_dir/.pybuild/cpython3_3.11/build/sarsen/geocoding.py:46: RuntimeWarning: invalid value encountered in divide
t_prev, t_curr = t_curr, t_curr - np.where(q != 0, p / q, 0) # type: ignore
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV]
/usr/lib/python3/dist-packages/rioxarray/_io.py:1111: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix will be returned.
warnings.warn(str(rio_warning.message), type(rio_warning.message)) # type: ignore
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path0-IW/VV] - numpy.core._exceptions._ArrayMemoryError: Unable to allocate 1.62 GiB for an array with shape (16705, 26102) and data type float32
FAILED tests/test_20_sentinel1.py::test_Sentinel1SarProduct[data_path1-IW1/VV] - ValueError: array is too big; `arr.size * arr.dtype.itemsize` is larger than the maximum possible size.
============ 2 failed, 16 passed, 12 skipped, 8 warnings in 29.65s =============