Skip to content

Commit 3e3b745

Browse files
scivisionscivision
committed
fix deflate not enabling, generated test and dedupe
1 parent 002d3a3 commit 3e3b745

File tree

10 files changed

+263
-85
lines changed

10 files changed

+263
-85
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ if(NOT CMAKE_BUILD_TYPE)
44
endif()
55
project(nc4fortran
66
LANGUAGES Fortran
7-
VERSION 1.0.0
7+
VERSION 1.0.1
88
DESCRIPTION "thin, light object-oriented NetCDF4 Fortran interface"
99
HOMEPAGE_URL https://github.com/geospace-code/nc4fortran)
1010

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ type(netcdf_file) :: hf
127127
* gzip compression may be applied for rank ≥ 2 arrays by setting `comp_lvl` to a value between 1 and 9.
128128
Shuffle filter is automatically applied for better compression
129129
* string attributes may be applied to any variable at time of writing or later.
130-
* `chunk_size` option may be set for better compression
131130

132131
`integer, intent(out) :: ierr` is optional.
133132
It will be non-zero if error detected.

‎meson.build‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
project('nc4fortran', 'fortran',
22
meson_version : '>=0.52.0',
3-
version: '1.0.0',
3+
version: '1.0.1',
44
default_options : ['default_library=static', 'buildtype=release', 'warning_level=3'])
55

66
subdir('meson')

‎src/interface.f90‎

Lines changed: 68 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ module nc4fortran
88
NF90_ECHAR, NF90_EEDGE, NF90_ENAMEINUSE, NF90_EBADID, NF90_EINDEFINE, NF90_NOWRITE, NF90_EDIMSIZE, &
99
nf90_open, nf90_close, nf90_estride, nf90_inq_varid, nf90_inq_dimid, nf90_inquire_dimension, &
1010
nf90_def_dim, nf90_def_var, nf90_get_var, nf90_put_var, &
11-
nf90_inq_libvers, nf90_sync
11+
nf90_inq_libvers, nf90_sync, nf90_inquire_variable
1212

1313
use pathlib, only : unlink, get_tempdir, is_absolute_path
1414
use string_utils, only : toLower, strip_trailing_null, truncate_string_null
@@ -39,8 +39,9 @@ module nc4fortran
3939

4040
!> methods used directly without type/rank agnosticism
4141
procedure, public :: initialize => nc_initialize, finalize => nc_finalize, &
42-
shape => get_shape, ndims => get_ndims, write_attribute, read_attribute, flush, &
43-
exist=>nc_check_exist, exists=>nc_check_exist
42+
shape => get_shape, ndims => get_ndims, write_attribute, read_attribute, flush=>nc_flush, &
43+
exist=>nc_check_exist, exists=>nc_check_exist, &
44+
is_chunked, is_contig, chunks=>get_chunk
4445

4546
!> generic procedures mapped over type / rank
4647
generic, public :: write => nc_write_scalar, nc_write_1d, nc_write_2d, nc_write_3d, &
@@ -340,7 +341,7 @@ subroutine nc_finalize(self, ierr)
340341
end subroutine nc_finalize
341342

342343

343-
subroutine flush(self, ierr)
344+
subroutine nc_flush(self, ierr)
344345

345346
class(netcdf_file), intent(in) :: self
346347
integer, intent(out), optional :: ierr
@@ -354,7 +355,69 @@ subroutine flush(self, ierr)
354355
error stop
355356
endif
356357

357-
end subroutine flush
358+
end subroutine nc_flush
359+
360+
361+
logical function is_contig(self, dname)
362+
class(netcdf_file), intent(in) :: self
363+
character(*), intent(in) :: dname
364+
integer :: ier, varid
365+
366+
ier = nf90_inq_varid(self%ncid, dname, varid)
367+
if (ier/=NF90_NOERR) then
368+
write(stderr,*) 'ERROR:nc4fortran:is_contig: cannot find variable: ' // dname
369+
error stop
370+
endif
371+
ier = nf90_inquire_variable(self%ncid, varid, contiguous=is_contig)
372+
if (ier/=NF90_NOERR) then
373+
write(stderr,*) 'ERROR:nc4fortran:is_contig: cannot get variable properties'
374+
error stop
375+
endif
376+
end function is_contig
377+
378+
379+
logical function is_chunked(self, dname)
380+
class(netcdf_file), intent(in) :: self
381+
character(*), intent(in) :: dname
382+
integer :: ier, varid
383+
384+
ier = nf90_inq_varid(self%ncid, dname, varid)
385+
if (ier/=NF90_NOERR) then
386+
write(stderr,*) 'ERROR:nc4fortran:is_chunked: cannot find variable: ' // dname
387+
error stop
388+
endif
389+
ier = nf90_inquire_variable(self%ncid, varid, contiguous=is_chunked)
390+
if (ier/=NF90_NOERR) then
391+
write(stderr,*) 'ERROR:nc4fortran:is_chunked: cannot get variable properties'
392+
error stop
393+
endif
394+
is_chunked = .not.is_chunked
395+
end function is_chunked
396+
397+
398+
subroutine get_chunk(self, dname, chunk_size)
399+
class(netcdf_file), intent(in) :: self
400+
character(*), intent(in) :: dname
401+
integer, intent(out) :: chunk_size(:)
402+
logical :: contig
403+
integer :: i, varid
404+
405+
chunk_size = -1
406+
407+
i = nf90_inq_varid(self%ncid, dname, varid)
408+
if (i/=NF90_NOERR) then
409+
write(stderr,*) 'ERROR:nc4fortran:chunk: cannot find variable: ' // dname
410+
return
411+
endif
412+
i = nf90_inquire_variable(self%ncid, varid, contiguous=contig)
413+
if (i/=NF90_NOERR) then
414+
write(stderr,*) 'nc4fortran:chunk: cannot get variable properties'
415+
return
416+
endif
417+
if(contig) return
418+
i = nf90_inquire_variable(self%ncid, varid, chunksizes=chunk_size)
419+
if (i/=NF90_NOERR) write(stderr,*) 'nc4fortran:chunk: cannot get variable properties'
420+
end subroutine get_chunk
358421

359422

360423
logical function is_netcdf(filename)

‎src/read.f90‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
submodule (nc4fortran) read
22
!! This submodule is for reading NetCDF via submodules
33

4-
use netcdf, only : nf90_inquire_variable
54
implicit none (type, external)
65

76
contains

‎src/tests/CMakeLists.txt‎

Lines changed: 13 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,16 @@
1-
add_executable(test_array test_array.f90)
2-
target_link_libraries(test_array nc4fortran::nc4fortran)
3-
if(CMAKE_Fortran_COMPILER_ID STREQUAL GNU)
4-
target_compile_options(test_array PRIVATE -Wno-compare-reals)
5-
endif()
6-
add_test(NAME nc4fortran:array COMMAND $<TARGET_FILE:test_array>
1+
add_executable(test_minimal test_minimal.f90)
2+
target_link_libraries(test_minimal PRIVATE nc4fortran::nc4fortran)
3+
add_test(NAME nc4fortran:minimal COMMAND $<TARGET_FILE:test_minimal>
74
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
5+
set_tests_properties(nc4fortran:minimal PROPERTIES FIXTURES_SETUP nclib)
86

9-
add_executable(test_attributes test_attributes.f90)
10-
target_link_libraries(test_attributes nc4fortran::nc4fortran)
11-
if(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU)
12-
target_compile_options(test_attributes PRIVATE -Wno-compare-reals)
13-
endif()
14-
add_test(NAME nc4fortran:attributes COMMAND $<TARGET_FILE:test_attributes>
15-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
16-
17-
add_executable(test_error test_error.f90)
18-
target_link_libraries(test_error nc4fortran::nc4fortran)
19-
add_test(NAME nc4fortran:error COMMAND $<TARGET_FILE:test_error>
20-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
21-
22-
add_executable(test_exist test_exist.f90)
23-
target_link_libraries(test_exist PRIVATE nc4fortran::nc4fortran)
24-
add_test(NAME nc4fortran:test_exist COMMAND $<TARGET_FILE:test_exist>
25-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
7+
foreach(t array attributes deflate error exist scalar shape string)
8+
add_executable(test_${t} test_${t}.f90)
9+
target_link_libraries(test_${t} PRIVATE nc4fortran::nc4fortran)
10+
if(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU)
11+
target_compile_options(test_${t} PRIVATE -Wno-compare-reals)
12+
endif()
2613

27-
add_executable(test_scalar test_scalar.f90)
28-
target_link_libraries(test_scalar PRIVATE nc4fortran::nc4fortran)
29-
add_test(NAME nc4fortran:test_scalar COMMAND $<TARGET_FILE:test_scalar>
30-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
31-
32-
add_executable(test_string test_string.f90)
33-
target_link_libraries(test_string nc4fortran::nc4fortran)
34-
add_test(NAME nc4fortran:test_string COMMAND $<TARGET_FILE:test_string>
35-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
36-
37-
add_executable(test_shape test_shape.f90)
38-
target_link_libraries(test_shape PRIVATE nc4fortran::nc4fortran)
39-
add_test(NAME nc4fortran:shape_check COMMAND $<TARGET_FILE:test_shape>
40-
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
14+
add_test(NAME nc4fortran:${t} COMMAND $<TARGET_FILE:test_${t}> WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
15+
set_tests_properties(nc4fortran:${t} PROPERTIES FIXTURES_REQUIRED nclib)
16+
endforeach()

‎src/tests/meson.build‎

Lines changed: 17 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,20 @@
1-
test_array = executable('test_array', 'test_array.f90', dependencies: netcdf_interface)
2-
test('array', test_array, suite: 'nc4basic', timeout: 10)
3-
4-
test_attr = executable('test_attributes', 'test_attributes.f90',
5-
fortran_args: quiet,
6-
dependencies: netcdf_interface)
7-
test('attributes', test_attr, args: meson.current_build_dir(), timeout: 10, suite: 'nc4basic')
8-
9-
test_error = executable('test_error', 'test_error.f90',
10-
fortran_args: quiet,
11-
dependencies: netcdf_interface)
12-
test('errors', test_error, args: meson.current_build_dir(), timeout: 10, suite: 'nc4basic')
13-
14-
test_exist = executable('test_exist', 'test_exist.f90',
15-
fortran_args: quiet,
1+
test_minimal = executable('test_minimal', 'test_minimal.f90',
162
dependencies: netcdf_interface)
17-
test('exist', test_exist, args: meson.current_build_dir(), timeout: 10, suite: 'nc4basic')
18-
19-
test_scalar = executable('test_scalar', 'test_scalar.f90', dependencies: netcdf_interface)
20-
test('scalar', test_scalar, suite: 'nc4basic', timeout: 10)
3+
test('minimal', test_minimal,
4+
args: meson.current_build_dir(),
5+
suite: 'nc4basic',
6+
priority: 100,
7+
timeout: 10)
218

22-
test_shape = executable('test_shape', 'test_shape.f90',
23-
fortran_args: quiet,
24-
dependencies: netcdf_interface)
25-
test('shape', test_shape,
26-
is_parallel : false,
27-
timeout: 10, suite: 'nc4basic')
9+
foreach t : ['array', 'attributes', 'deflate', 'error', 'exist', 'scalar', 'shape', 'string']
2810

29-
test_str = executable('test_string', 'test_string.f90',
30-
fortran_args: quiet,
31-
dependencies: netcdf_interface)
32-
test('string', test_str, args: meson.current_build_dir(), timeout: 10, suite: 'nc4basic')
11+
e = executable('test_'+t, 'test_' + t + '.f90', dependencies: netcdf_interface, fortran_args: quiet)
12+
if t == 'shape'
13+
par = false
14+
pri = 50
15+
else
16+
par = true
17+
pri = 0
18+
endif
19+
test(t, e, suite: 'h5basic', timeout: 10, is_parallel: par, priority: pri)
20+
endforeach

‎src/tests/test_deflate.f90‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
program deflate_test
2+
!! unit tests and registration tests of HDF5 deflate compression write
3+
use, intrinsic:: iso_fortran_env, only: int32, real32, real64, stderr=>error_unit
4+
5+
use nc4fortran, only: netcdf_file, toLower, strip_trailing_null, truncate_string_null
6+
7+
implicit none (type, external)
8+
9+
type(netcdf_file) :: h
10+
integer, parameter :: N=1000
11+
integer :: crat, chunks(3)
12+
integer :: fsize
13+
14+
integer, allocatable :: ibig2(:,:), ibig3(:,:,:)
15+
real(real32), allocatable :: big2(:,:), big3(:,:,:)
16+
character(*), parameter :: fn1='deflate1.nc', fn2='deflate2.nc', fn3='deflate3.nc', fn4='deflate4.nc'
17+
18+
allocate(ibig2(N,N), ibig3(N,N,4), big2(N,N), big3(N,N,4))
19+
20+
ibig2 = 0
21+
ibig3 = 0
22+
big2 = 0
23+
big3 = 0
24+
25+
call h%initialize(fn1, status='replace', comp_lvl=1, debug=.true.)
26+
call h%write('big2', big2, dims=['x','y'])
27+
call h%flush()
28+
!> turn compression off for following variables (must flush first)
29+
h%comp_lvl = 0
30+
call h%write('small_contig', big2(:5,:5), dims=['q','r'])
31+
call h%finalize()
32+
33+
inquire(file=fn1, size=fsize)
34+
crat = (N*N*storage_size(big2)/8) / fsize
35+
print '(A,F6.2,A,I6)','#1 filesize (Mbytes): ',fsize/1e6, ' 2D compression ratio:',crat
36+
if (crat < 10) error stop '#1 2D low compression'
37+
38+
call h%initialize(fn1, status='old', action='r', debug=.false.)
39+
40+
if(.not. h%is_chunked('big2')) error stop '#1 not chunked layout'
41+
42+
call h%chunks('big2', chunks(:2))
43+
if(any(chunks(:2) /= [1000, 1000])) error stop '#1 get_chunk mismatch'
44+
45+
if(.not.h%is_contig('small_contig')) error stop '#1 not contig layout'
46+
call h%chunks('small_contig', chunks(:2))
47+
if(any(chunks(:2) /= -1)) error stop '#1 get_chunk mismatch'
48+
49+
call h%finalize()
50+
51+
! ======================================
52+
53+
call h%initialize(fn2, status='replace',comp_lvl=1, debug=.true.)
54+
call h%write('big3', big3)
55+
56+
call h%write('big3_autochunk', big3)
57+
call h%chunks('big3_autochunk', chunks)
58+
if(any(chunks /= [500,500,2])) then
59+
write(stderr,*) '#2 chunk size', chunks
60+
error stop '#2 auto chunk unexpected chunk size'
61+
endif
62+
call h%finalize()
63+
64+
inquire(file=fn2, size=fsize)
65+
crat = (2*N*N*storage_size(big3)/8) / fsize
66+
print '(A,F6.2,A,I6)','#2 filesize (Mbytes): ',fsize/1e6, ' 3D compression ratio:',crat
67+
if (h%comp_lvl > 0 .and. crat < 10) error stop '#2 3D low compression'
68+
69+
! ======================================
70+
71+
call h%initialize(fn3, status='replace',comp_lvl=1, debug=.true.)
72+
73+
call h%write('ibig3', ibig3(:N-10,:N-20,:))
74+
call h%chunks('ibig3', chunks)
75+
if(any(chunks /= [495,490,2])) then
76+
write(stderr,*) '#3 chunk size', chunks
77+
error stop '#3 auto chunk unexpected chunk size'
78+
endif
79+
call h%finalize()
80+
81+
inquire(file=fn3, size=fsize)
82+
crat = (N*N*storage_size(ibig3)/8) / fsize
83+
print '(A,F6.2,A,I6)','#3 filesize (Mbytes): ',fsize/1e6, ' 3D compression ratio:',crat
84+
if (h%comp_lvl > 0 .and. crat < 10) error stop '#3 3D low compression'
85+
! !======================================
86+
87+
call h%initialize(fn4, status='replace', comp_lvl=1, debug=.true.)
88+
call h%write('ibig2', ibig2)
89+
call h%finalize()
90+
91+
inquire(file=fn4, size=fsize)
92+
crat = (N*N*storage_size(ibig2)/8) / fsize
93+
print '(A,F6.2,A,I6)','#4 filesize (Mbytes): ',fsize/1e6, ' 3D compression ratio:',crat
94+
if (h%comp_lvl > 0 .and. crat < 10) error stop '#4 3D low compression'
95+
96+
end program

‎src/tests/test_minimal.f90‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
program test_minimal
2+
3+
use netcdf, only : nf90_create, nf90_def_var, nf90_put_var, nf90_close, NF90_CLOBBER, NF90_NETCDF4, NF90_INT
4+
5+
implicit none (type, external)
6+
7+
integer :: i, ncid, varid
8+
character(*), parameter :: filename='test_minimal.nc'
9+
10+
i = nf90_create(filename, ior(NF90_CLOBBER, NF90_NETCDF4), ncid)
11+
if (i/=0) error stop 'minimal: could not create file'
12+
print *, 'minimal: created '// filename
13+
14+
i = nf90_def_var(ncid, 'x', NF90_INT, varid=varid)
15+
i = nf90_put_var(ncid, varid, 42)
16+
if (i/=0) error stop 'minimal: could not create variable'
17+
print *, 'minimal: created variable'
18+
19+
i = nf90_close(ncid)
20+
if (i/=0) error stop 'minimal: could not close file'
21+
print *, 'minimal: closed '// filename
22+
23+
! this is a Fortran-standard way to delete files
24+
open(newunit=i, file=filename)
25+
close(i, status='delete')
26+
27+
end program

0 commit comments

Comments
 (0)