Add Arrow Variant Extension Type, remove Array impl for VariantArray and ShreddedVariantFieldArray (#8392)

# Which issue does this PR close?

- closes #8319
- closes #8296

# Rationale for this change

This is needed to [read/write the Variant Parquet logical
type](#8370) and work with the
rest of the Arrow Ecosystem

Note, this is broken out the larger PR here: 
- from #8365


We need a way to write Variant encoded data to/from parquet, and the
current way the VariantArray is implemented doesn't work (panics when
writing to parquet). See tickets above

Instead of a `impl Array` it seems the better way to do this is using an
Arrow Extension Type. See
#8319 (comment)
for more details



# What changes are included in this PR?

1. remove the `Array` impl for `VariantArray`, which forces explict
conversions back/forth when reading/writing
2. remove the `Array` impl for `ShreddedVariantFieldArray`, which forces
explicit conversions back/forth when reading/writing
3. Add `VariantType` extension type

# Are these changes tested?

Yes, with new code and tests

# Are there any user-facing changes?

Yes, but this is not yet stable / released, so these changes have no
impact on the releasability of this code

---------

Co-authored-by: Ryan Johnson <[email protected]>
Co-authored-by: Matthijs Brobbel <[email protected]>

Author: 3 people

parquet-variant-compute/benches/variant_kernels.rs

@@ -84,7 +84,7 @@ fn benchmark_batch_json_string_to_variant(c: &mut Criterion) {
 
 pub fn variant_get_bench(c: &mut Criterion) {
     let variant_array = create_primitive_variant_array(8192);
-    let input: ArrayRef = Arc::new(variant_array);
+    let input = ArrayRef::from(variant_array);
 
     let options = GetOptions {
         path: vec![].into(),

parquet-variant-compute/src/lib.rs

@@ -46,7 +46,7 @@ mod variant_array_builder;
 pub mod variant_get;
 mod variant_to_arrow;
 
-pub use variant_array::{ShreddingState, VariantArray};
+pub use variant_array::{ShreddingState, VariantArray, VariantType};
 pub use variant_array_builder::{VariantArrayBuilder, VariantValueArrayBuilder};
 
 pub use cast_to_variant::{cast_to_variant, cast_to_variant_with_options};

parquet-variant-compute/src/shred_variant.rs

@@ -22,7 +22,7 @@ use crate::variant_to_arrow::{
     make_primitive_variant_to_arrow_row_builder, PrimitiveVariantToArrowRowBuilder,
 };
 use crate::{VariantArray, VariantValueArrayBuilder};
-use arrow::array::{Array as _, ArrayRef, BinaryViewArray, NullBufferBuilder};
+use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder};
 use arrow::buffer::NullBuffer;
 use arrow::compute::CastOptions;
 use arrow::datatypes::{DataType, Fields};
@@ -310,7 +310,7 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> {
             let (value, typed_value, nulls) = typed_value_builder.finish()?;
             let array =
                 ShreddedVariantFieldArray::from_parts(Some(value), Some(typed_value), nulls);
-            builder = builder.with_field(field_name, Arc::new(array), false);
+            builder = builder.with_field(field_name, ArrayRef::from(array), false);
         }
         if let Some(nulls) = self.typed_value_nulls.finish() {
             builder = builder.with_nulls(nulls);
@@ -327,7 +327,7 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> {
 mod tests {
     use super::*;
     use crate::VariantArrayBuilder;
-    use arrow::array::{Float64Array, Int64Array};
+    use arrow::array::{Array, Float64Array, Int64Array};
     use arrow::datatypes::{DataType, Field, Fields};
     use parquet_variant::{Variant, VariantBuilder, VariantBuilderExt as _};
     use std::sync::Arc;
@@ -556,18 +556,11 @@ mod tests {
             .unwrap();
 
         // Extract score and age fields from typed_value struct
-        let score_field = typed_value
-            .column_by_name("score")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<crate::variant_array::ShreddedVariantFieldArray>()
-            .unwrap();
-        let age_field = typed_value
-            .column_by_name("age")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<crate::variant_array::ShreddedVariantFieldArray>()
-            .unwrap();
+        let score_field =
+            ShreddedVariantFieldArray::try_new(typed_value.column_by_name("score").unwrap())
+                .unwrap();
+        let age_field =
+            ShreddedVariantFieldArray::try_new(typed_value.column_by_name("age").unwrap()).unwrap();
 
         let score_value = score_field
             .value_field()