Skip to content

Add support for Float16 type in substrait #16793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

Add support for Float16 type in substrait #16793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f019239
casting f16 to f32 while producing the substrait plan
jatin510 Jul 15, 2025
a1f60bc
Add support for Float16 type variation in Substrait plan
jatin510 Jul 15, 2025
dc6075a
fmt fix
jatin510 Jul 15, 2025
7b113e6
Refactor Float16 handling in Substrait: use UserDefined type instead …
jatin510 Jul 16, 2025
4932a3f
Merge branch 'main' into feat/16298-substrait-float16
jatin510 Jul 17, 2025
627e3fd
Merge branch 'main' into feat/16298-substrait-float16
jatin510 Jul 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion datafusion/substrait/src/logical_plan/consumer/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use crate::variation_const::{
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION_REF,
DEFAULT_MAP_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DICTIONARY_MAP_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION_REF,
INTERVAL_DAY_TIME_TYPE_REF, INTERVAL_MONTH_DAY_NANO_TYPE_NAME,
FLOAT16_TYPE_REF, INTERVAL_DAY_TIME_TYPE_REF, INTERVAL_MONTH_DAY_NANO_TYPE_NAME,
INTERVAL_MONTH_DAY_NANO_TYPE_REF, INTERVAL_YEAR_MONTH_TYPE_REF,
LARGE_CONTAINER_TYPE_VARIATION_REF, TIMESTAMP_MICRO_TYPE_VARIATION_REF,
TIMESTAMP_MILLI_TYPE_VARIATION_REF, TIMESTAMP_NANO_TYPE_VARIATION_REF,
Expand Down Expand Up @@ -260,6 +260,8 @@ pub fn from_substrait_type(
} else {
#[allow(deprecated)]
match u.type_reference {
// Float16 support via UserDefined type
FLOAT16_TYPE_REF => Ok(DataType::Float16),
// Kept for backwards compatibility, producers should use IntervalYear instead
INTERVAL_YEAR_MONTH_TYPE_REF => {
Ok(DataType::Interval(IntervalUnit::YearMonth))
Expand Down
13 changes: 11 additions & 2 deletions datafusion/substrait/src/logical_plan/producer/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use crate::variation_const::{
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION_REF,
DEFAULT_MAP_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DICTIONARY_MAP_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION_REF,
LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
FLOAT16_TYPE_REF, LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
VIEW_CONTAINER_TYPE_VARIATION_REF,
};
Expand Down Expand Up @@ -96,7 +96,15 @@ pub(crate) fn to_substrait_type(
nullability,
})),
}),
// Float16 is not supported in Substrait
// Float16 is not supported in Substrait, use UserDefined type
DataType::Float16 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::UserDefined(r#type::UserDefined {
type_reference: FLOAT16_TYPE_REF,
type_variation_reference: 0,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we use DEFAULT_TYPE_VARIATION_REF?

nullability,
type_parameters: vec![],
})),
}),
DataType::Float32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Fp32(r#type::Fp32 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
Expand Down Expand Up @@ -375,6 +383,7 @@ mod tests {
round_trip_type(DataType::UInt32)?;
round_trip_type(DataType::Int64)?;
round_trip_type(DataType::UInt64)?;
round_trip_type(DataType::Float16)?;
round_trip_type(DataType::Float32)?;
round_trip_type(DataType::Float64)?;

Expand Down
5 changes: 5 additions & 0 deletions datafusion/substrait/src/variation_const.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,8 @@ pub const INTERVAL_MONTH_DAY_NANO_TYPE_REF: u32 = 3;
note = "Use Substrait `IntervalCompund` type instead"
)]
pub const INTERVAL_MONTH_DAY_NANO_TYPE_NAME: &str = "interval-month-day-nano";

/// For [`DataType::Float16`].
///
/// [`DataType::Float16`]: datafusion::arrow::datatypes::DataType::Float16
pub const FLOAT16_TYPE_REF: u32 = 4;