[Breaking] Return missing if the field is set but null.

docs/src/considerations.md

@@ -96,3 +96,34 @@ The interface is implemented in [`src/tables.jl`](https://github.com/yeesian/Arc
 * `ArchGDAL.FeatureLayer` meets the criteria for an `AbstractRow`-iterator based on the previous bullet and meeting the criteria for [`Iteration`](https://docs.julialang.org/en/v1/manual/interfaces/#man-interface-iteration) in [`base/iterators.jl`](https://github.com/yeesian/ArchGDAL.jl/blob/a665f3407930b8221269f8949c246db022c3a85c/src/base/iterators.jl#L1-L18).
 * `ArchGDAL.AbstractDataset` might contain multiple layers, and might correspond to multiple tables. The way to construct tables would be to get the layers before forming the corresponding tables.
 
+## Missing and Null Semantics in GDAL
+
+When reading the fields of a feature using `getfield(feature, i)`, ArchGDAL observes the following behavior:
+
+| Field | null      | notnull   |
+|-------|-----------|-----------|
+| set   | `missing` | value     |
+| unset | N/A       | `nothing` |
+
+This reflects that
+* a field that is notnull will never return `missing`: use `isfieldnull(feature, i)` to determine if a field has been set.
+* a field is set will never return `nothing` (and a field that unset will always return `nothing`): use `isfieldset(feature, i)` to determine if a field has been set.
+* a field that is set and not null will always have a concrete value: use `isfieldsetandnotnull(feature, i)` to test for it.
+
+When writing the fields of a feature using `setfield!(feature, i, value)`, ArchGDAL observes the following behavior:
+
+| Field     | nullable | notnullable    |
+|-----------|----------|----------------|
+| `nothing` | unset    | unset          |
+| `missing` | null     | `getdefault()` |
+| value     | value    | value          |
+
+This reflects that
+* writing `nothing` will cause the field to be unset.
+* writing `missing` will cause the field to be null. In the cause of a notnullable field, it will take the default value (see https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html for details). If there is no default value, `getdefault()` will return `nothing`, causing the field to be unset.
+* writing a value will behave in the usual manner.
+
+For additional references, see
+* [JuliaLang: Nothingness and missing values](https://docs.julialang.org/en/v1/manual/faq/#faq-nothing)
+* [GDAL: OGR not-null constraints and default values](https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html)
+* [GDAL: Null values in OGR](https://gdal.org/development/rfc/rfc67_nullfieldvalues.html)

src/base/display.jl

@@ -149,7 +149,7 @@ function Base.show(io::IO, layer::AbstractFeatureLayer)::Nothing
     nfielddisplay = min(n, 5)
     for i in 1:nfielddisplay
         fd = getfielddefn(featuredefn, i - 1)
-        display = "     Field $(i - 1) ($(getname(fd))): [$(gettype(fd))]"
+        display = "     Field $(i - 1) ($(getname(fd))): [$(getfieldtype(fd))]"
         if length(display) > 75
             println(io, "$display[1:70]...")
             continue

src/ogr/feature.jl

@@ -149,6 +149,59 @@ function unsetfield!(feature::Feature, i::Integer)::Feature
     return feature
 end
 
+"""
+    isfieldnull(feature::Feature, i::Integer)
+
+Test if a field is null.
+
+### Parameters
+* `feature`: the feature that owned the field.
+* `i`: the field to test, from 0 to GetFieldCount()-1.
+
+### Returns
+`true` if the field is null, otherwise `false`.
+
+### References
+* https://gdal.org/development/rfc/rfc67_nullfieldvalues.html
+"""
+isfieldnull(feature::Feature, i::Integer)::Bool =
+    Bool(GDAL.ogr_f_isfieldnull(feature.ptr, i))
+
+"""
+    isfieldsetandnotnull(feature::Feature, i::Integer)
+
+Test if a field is set and not null.
+
+### Parameters
+* `feature`: the feature that owned the field.
+* `i`: the field to test, from 0 to GetFieldCount()-1.
+
+### Returns
+`true` if the field is set and not null, otherwise `false`.
+
+### References
+* https://gdal.org/development/rfc/rfc67_nullfieldvalues.html
+"""
+isfieldsetandnotnull(feature::Feature, i::Integer)::Bool =
+    Bool(GDAL.ogr_f_isfieldsetandnotnull(feature.ptr, i))
+
+"""
+    setfieldnull!(feature::Feature, i::Integer)
+
+Clear a field, marking it as null.
+
+### Parameters
+* `feature`: the feature that owned the field.
+* `i`: the field to set to null, from 0 to GetFieldCount()-1.
+
+### References
+* https://gdal.org/development/rfc/rfc67_nullfieldvalues.html
+"""
+function setfieldnull!(feature::Feature, i::Integer)::Feature
+    GDAL.ogr_f_setfieldnull(feature.ptr, i)
+    return feature
+end
+
 # """
 #     OGR_F_GetRawFieldRef(OGRFeatureH hFeat,
 #                          int iField) -> OGRField *
@@ -381,7 +434,7 @@ end
 #           pfSecond,pnTZFlag)
 # end
 
-function getdefault(feature::Feature, i::Integer)::Union{String,Missing}
+function getdefault(feature::Feature, i::Integer)::Union{String,Nothing}
     return getdefault(getfielddefn(feature, i))
 end
 
@@ -402,13 +455,33 @@ const _FETCHFIELD = Dict{OGRFieldType,Function}(
     OFTInteger64List => asint64list,
 )
 
+"""
+    getfield(feature, i)
+
+When the field is unset, it will return `nothing`. When the field is set but
+null, it will return `missing`.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
+* https://gdal.org/development/rfc/rfc67_nullfieldvalues.html
+"""
 function getfield(feature::Feature, i::Integer)
-    return if isfieldset(feature, i)
-        _fieldtype = gettype(getfielddefn(feature, i))
-        _fetchfield = get(_FETCHFIELD, _fieldtype, getdefault)
-        _fetchfield(feature, i)
+    return if !isfieldset(feature, i)
+        nothing
+    elseif isfieldnull(feature, i)
+        missing
     else
-        getdefault(feature, i)
+        _fieldtype = getfieldtype(getfielddefn(feature, i))
+        try
+            _fetchfield = _FETCHFIELD[_fieldtype]
+            _fetchfield(feature, i)
+        catch e
+            if e isa KeyError
+                error("$_fieldtype not implemented in getfield, please report an issue to https://github.com/yeesian/ArchGDAL.jl/issues")
+            else
+                rethrow(e)
+            end
+        end
     end
 end
 
@@ -439,6 +512,20 @@ field types may be unaffected.
 """
 function setfield! end
 
+function setfield!(feature::Feature, i::Integer, value::Nothing)::Feature
+    unsetfield!(feature, i)
+    return feature
+end
+
+function setfield!(feature::Feature, i::Integer, value::Missing)::Feature
+    if isnullable(getfielddefn(feature, i))
+        setfieldnull!(feature, 1)
+    else
+        setfield!(feature, i, getdefault(feature, i))
+    end
+    return feature
+end
+
 function setfield!(feature::Feature, i::Integer, value::Int32)::Feature
     GDAL.ogr_f_setfieldinteger(feature.ptr, i, value)
     return feature
@@ -854,6 +941,9 @@ Fill unset fields with default values that might be defined.
 * `feature`: handle to the feature.
 * `notnull`: if we should fill only unset fields with a not-null constraint.
 * `papszOptions`: unused currently. Must be set to `NULL`.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
 function fillunsetwithdefault!(
     feature::Feature;
@@ -885,6 +975,9 @@ fails, then it will fail for all interpretations).
 
 ### Returns
 `true` if all enabled validation tests pass.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
 validate(feature::Feature, flags::FieldValidation, emiterror::Bool)::Bool =
     Bool(GDAL.ogr_f_validate(feature.ptr, flags, emiterror))

src/ogr/fielddefn.jl

@@ -65,12 +65,38 @@ OGRFeatureDefn.
 ### Parameters
 * `fielddefn`: handle to the field definition to set type to.
 * `subtype`: the new field subtype.
+
+### References
+* https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
 """
 function setsubtype!(fielddefn::FieldDefn, subtype::OGRFieldSubType)::FieldDefn
     GDAL.ogr_fld_setsubtype(fielddefn.ptr, subtype)
     return fielddefn
 end
 
+"""
+    getfieldtype(fielddefn::AbstractFieldDefn)
+
+Returns the type or subtype (if any) of this field.
+
+### Parameters
+* `fielddefn`: handle to the field definition.
+
+### Returns
+The field type or subtype.
+
+### References
+* https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
+"""
+function getfieldtype(fielddefn::AbstractFieldDefn)::Union{OGRFieldType, OGRFieldSubType}
+    fieldsubtype = getsubtype(fielddefn)
+    return if fieldsubtype != OFSTNone
+        fieldsubtype
+    else
+        gettype(fielddefn)
+    end
+end
+
 """
     getjustify(fielddefn::AbstractFieldDefn)
 
@@ -196,6 +222,9 @@ Even if this method returns `false` (i.e not-nullable field), it doesn't mean
 that OGRFeature::IsFieldSet() will necessarily return `true`, as fields can be
 temporarily unset and null/not-null validation is usually done when
 OGRLayer::CreateFeature()/SetFeature() is called.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
 isnullable(fielddefn::AbstractFieldDefn)::Bool =
     Bool(GDAL.ogr_fld_isnullable(fielddefn.ptr))
@@ -210,8 +239,11 @@ to set a not-null constraint.
 
 Drivers that support writing not-null constraint will advertize the
 GDAL_DCAP_NOTNULL_FIELDS driver metadata item.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
-function setnullable!(fielddefn::FieldDefn, nullable::Bool)::FieldDefn
+function setnullable!(fielddefn::T, nullable::Bool)::T where {T <: AbstractFieldDefn}
     GDAL.ogr_fld_setnullable(fielddefn.ptr, nullable)
     return fielddefn
 end
@@ -220,12 +252,15 @@ end
     getdefault(fielddefn::AbstractFieldDefn)
 
 Get default field value
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
-function getdefault(fielddefn::AbstractFieldDefn)::Union{String,Missing}
+function getdefault(fielddefn::AbstractFieldDefn)::Union{String,Nothing}
     result =
         @gdal(OGR_Fld_GetDefault::Cstring, fielddefn.ptr::GDAL.OGRFieldDefnH)
     return if result == C_NULL
-        missing
+        nothing
     else
         unsafe_string(result)
     end
@@ -252,6 +287,9 @@ datetime literal value, format should be 'YYYY/MM/DD HH:MM:SS[.sss]'
 
 Drivers that support writing DEFAULT clauses will advertize the
 GDAL_DCAP_DEFAULT_FIELDS driver metadata item.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
 function setdefault!(fielddefn::T, default)::T where {T<:AbstractFieldDefn}
     GDAL.ogr_fld_setdefault(fielddefn.ptr, default)
@@ -266,6 +304,9 @@ Returns whether the default value is driver specific.
 Driver specific default values are those that are not NULL, a numeric value, a
 literal value enclosed between single quote characters, CURRENT_TIMESTAMP,
 CURRENT_TIME, CURRENT_DATE or datetime literal value.
+
+### References
+* https://gdal.org/development/rfc/rfc53_ogr_notnull_default.html
 """
 isdefaultdriverspecific(fielddefn::AbstractFieldDefn)::Bool =
     Bool(GDAL.ogr_fld_isdefaultdriverspecific(fielddefn.ptr))

src/types.jl

@@ -619,13 +619,19 @@ getname(obj::OGRFieldType)::String = GDAL.ogr_getfieldtypename(obj)
     getname(obj::OGRFieldSubType)
 
 Fetch human readable name for a field subtype.
+
+### References
+* https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
 """
 getname(obj::OGRFieldSubType)::String = GDAL.ogr_getfieldsubtypename(obj)
 
 """
     arecompatible(dtype::OGRFieldType, subtype::OGRFieldSubType)
 
 Return if type and subtype are compatible.
+
+### References
+* https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
 """
 arecompatible(dtype::OGRFieldType, subtype::OGRFieldSubType)::Bool =
     Bool(GDAL.ogr_aretypesubtypecompatible(dtype, subtype))