Skip to content

Commit d2568d5

Browse files
franzpoeschelfranzpoeschel
committed
Use UNDEFINED_EXTENT for this instead of empty extent
1 parent 7c1d707 commit d2568d5

File tree

8 files changed

+75
-17
lines changed

8 files changed

+75
-17
lines changed

‎include/openPMD/Dataset.hpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,21 @@ class Dataset
5454
*/
5555
JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
5656
/**
57-
* Some backends (i.e. JSON and TOML in template mode) support the
58-
* creation of dataset with undefined datatype and extent.
59-
* The extent should be given as {UNDEFINED_EXTENT} for that.
57+
* In some use cases, the extent needs not be specified.
58+
* For these, specify Extent{UNDEFINED_EXTENT}.
59+
* Use cases:
60+
*
61+
* 1. Some backends (i.e. JSON and TOML in template mode) support the
62+
* creation of dataset with undefined datatype and extent.
63+
* The extent should be given as {UNDEFINED_EXTENT} for that.
64+
* 2. With openPMD 2.0, the shape of constant components may be omitted
65+
* in writing if it is defined somewhere else as part
66+
* of the same Mesh / Species.
67+
* (https://github.com/openPMD/openPMD-standard/pull/289)
68+
* When reading such datasets, the openPMD-api will try to fill in
69+
* the missing extents, so the extent for constistently-defined
70+
* datasets should ideally not be reported by the read-side API
71+
* as undefined.
6072
*/
6173
UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
6274
};
@@ -87,5 +99,8 @@ class Dataset
8799

88100
std::optional<size_t> joinedDimension() const;
89101
static std::optional<size_t> joinedDimension(Extent const &);
102+
103+
bool undefinedExtent() const;
104+
static bool undefinedExtent(Extent const &);
90105
};
91106
} // namespace openPMD

‎include/openPMD/IO/IOTask.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include "openPMD/ChunkInfo.hpp"
2424
#include "openPMD/Dataset.hpp"
25+
#include "openPMD/Error.hpp"
2526
#include "openPMD/IterationEncoding.hpp"
2627
#include "openPMD/Streaming.hpp"
2728
#include "openPMD/auxiliary/Export.hpp"

‎include/openPMD/RecordComponent.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ class RecordComponent : public BaseRecordComponent
165165
* * Shrinking any dimension's extent.
166166
* * Changing the number of dimensions.
167167
*
168+
* The dataset extent may be empty to indicate undefined extents.
169+
*
168170
* Backend support for resizing datasets:
169171
* * JSON: Supported
170172
* * ADIOS2: Supported as of ADIOS2 2.7.0

‎src/Dataset.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,13 @@ std::optional<size_t> Dataset::joinedDimension(Extent const &extent)
9595
}
9696
return res;
9797
}
98+
99+
bool Dataset::undefinedExtent() const
100+
{
101+
return undefinedExtent(extent);
102+
}
103+
bool Dataset::undefinedExtent(Extent const &e)
104+
{
105+
return e.size() == 1 && e.at(0) == Dataset::UNDEFINED_EXTENT;
106+
}
98107
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -802,6 +802,12 @@ void ADIOS2IOHandlerImpl::createDataset(
802802
"only is not possible.");
803803
}
804804

805+
if (Dataset::undefinedExtent(parameters.extent))
806+
{
807+
throw error::OperationUnsupportedInBackend(
808+
"ADIOS2", "No support for Datasets with undefined extent.");
809+
}
810+
805811
if (!writable->written)
806812
{
807813
/* Sanitize name */
@@ -962,6 +968,12 @@ void ADIOS2IOHandlerImpl::extendDataset(
962968
VERIFY_ALWAYS(
963969
access::write(m_handler->m_backendAccess),
964970
"[ADIOS2] Cannot extend datasets in read-only mode.");
971+
if (Dataset::undefinedExtent(parameters.extent))
972+
{
973+
throw error::OperationUnsupportedInBackend(
974+
"ADIOS2", "No support for Datasets with undefined extent.");
975+
}
976+
965977
setAndGetFilePosition(writable);
966978
auto file = refreshFileFromParent(writable, /* preferParentFile = */ false);
967979
std::string name = nameOfVariable(writable);

‎src/IO/HDF5/HDF5IOHandler.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,11 @@ void HDF5IOHandlerImpl::createDataset(
475475
error::throwOperationUnsupportedInBackend(
476476
"HDF5", "Joined Arrays currently only supported in ADIOS2");
477477
}
478+
else if (Dataset::undefinedExtent(parameters.extent))
479+
{
480+
throw error::OperationUnsupportedInBackend(
481+
"HDF5", "No support for Datasets with undefined extent.");
482+
}
478483

479484
if (!writable->written)
480485
{
@@ -845,6 +850,11 @@ void HDF5IOHandlerImpl::extendDataset(
845850
error::throwOperationUnsupportedInBackend(
846851
"HDF5", "Joined Arrays currently only supported in ADIOS2");
847852
}
853+
else if (Dataset::undefinedExtent(parameters.extent))
854+
{
855+
throw error::OperationUnsupportedInBackend(
856+
"HDF5", "No support for Datasets with undefined extent.");
857+
}
848858

849859
auto res = getFile(writable);
850860
if (!res)

‎src/RecordComponent.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ or alternatively the JSON option {"verify_homogeneous_extents": false}.
7373
Attributable const &callsite, RecordComponent &rc)
7474
{
7575
auto extent = rc.getExtent();
76-
if (extent.empty())
76+
if (Dataset::undefinedExtent(extent))
7777
{
7878
without_extent.emplace_back(rc);
7979
}
@@ -216,9 +216,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
216216
rc.m_hasBeenExtended = true;
217217
}
218218

219-
// @todo check this while flushing
220-
// if (d.extent.empty())
221-
// throw std::runtime_error("Dataset extent must be at least 1D.");
219+
if (d.extent.empty())
220+
throw std::runtime_error("Dataset extent must be at least 1D.");
222221
if (d.empty())
223222
{
224223
if (d.extent.empty())
@@ -275,7 +274,7 @@ Extent RecordComponent::getExtent() const
275274
}
276275
else
277276
{
278-
return {};
277+
return {Dataset::UNDEFINED_EXTENT};
279278
}
280279
}
281280

@@ -402,7 +401,7 @@ void RecordComponent::flush(
402401
}
403402
auto constant_component_write_shape = [&]() {
404403
auto extent = getExtent();
405-
return !extent.empty() &&
404+
return !Dataset::undefinedExtent(extent) &&
406405
std::none_of(extent.begin(), extent.end(), [](auto val) {
407406
return val == Dataset::JOINED_DIMENSION;
408407
});
@@ -542,7 +541,7 @@ void RecordComponent::readBase(bool require_unit_si)
542541
if (!containsAttribute("shape"))
543542
{
544543
setWritten(false, Attributable::EnqueueAsynchronously::No);
545-
resetDataset(Dataset(dtype, {}));
544+
resetDataset(Dataset(dtype, {Dataset::UNDEFINED_EXTENT}));
546545
setWritten(true, Attributable::EnqueueAsynchronously::No);
547546

548547
return;

‎test/Files_SerialIO/components_without_extent.cpp‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ void particle_offset_without_extent()
2727
position_comp.storeChunk(std::move(data), {0}, {5});
2828

2929
auto offset_comp = e["positionOffset"][comp_id];
30-
offset_comp.resetDataset({openPMD::Datatype::INT, {}});
30+
offset_comp.resetDataset(
31+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
3132
offset_comp.makeConstant(0);
3233
}
3334
write.close();
@@ -57,11 +58,13 @@ void particles_without_any_extent()
5758
for (auto comp_id : {"x", "y", "z"})
5859
{
5960
auto position_comp = e["position"][comp_id];
60-
position_comp.resetDataset({openPMD::Datatype::INT, {}});
61+
position_comp.resetDataset(
62+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
6163
position_comp.makeConstant(0);
6264

6365
auto offset_comp = e["positionOffset"][comp_id];
64-
offset_comp.resetDataset({openPMD::Datatype::INT, {}});
66+
offset_comp.resetDataset(
67+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
6568
offset_comp.makeConstant(0);
6669
}
6770
write.close();
@@ -84,7 +87,9 @@ void particles_without_any_extent()
8487
{
8588
for (auto const &component : record.second)
8689
{
87-
REQUIRE(component.second.getExtent() == openPMD::Extent{});
90+
REQUIRE(
91+
component.second.getExtent() ==
92+
openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
8893
}
8994
}
9095
}
@@ -152,7 +157,8 @@ void meshes_with_incomplete_extent()
152157
for (auto comp_id : {"y", "z"})
153158
{
154159
auto comp = E[comp_id];
155-
comp.resetDataset({openPMD::Datatype::INT, {}});
160+
comp.resetDataset(
161+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
156162
comp.makeConstant(0);
157163
}
158164
write.close();
@@ -220,7 +226,9 @@ void meshes_without_any_extent()
220226
for (auto comp_id : {"x", "y", "z"})
221227
{
222228
auto comp = E[comp_id];
223-
comp.resetDataset({openPMD::Datatype::FLOAT, {}});
229+
comp.resetDataset(
230+
{openPMD::Datatype::FLOAT,
231+
{openPMD::Dataset::UNDEFINED_EXTENT}});
224232
comp.makeConstant<float>(0);
225233
}
226234
write.close();
@@ -241,7 +249,9 @@ void meshes_without_any_extent()
241249
auto E = read.snapshots()[0].meshes["E"];
242250
for (auto const &component : E)
243251
{
244-
REQUIRE(component.second.getExtent() == openPMD::Extent{});
252+
REQUIRE(
253+
component.second.getExtent() ==
254+
openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
245255
}
246256
}
247257
}

0 commit comments

Comments
 (0)