Merge branch 'main' into ctty/sort-partition

Author: CTTY

Cargo.lock

@@ -55,6 +55,7 @@ async-std = "1.12"
 async-trait = "0.1.88"
 aws-config = "1.6.1"
 aws-sdk-glue = "1.39"
+aws-sdk-s3tables = "1.28.0"
 backon = "1.5.1"
 base64 = "0.22.1"
 bimap = "0.6"
@@ -73,10 +74,13 @@ faststr = "0.2.31"
 fnv = "1.0.7"
 fs-err = "3.1.0"
 futures = "0.3"
-hive_metastore = "0.1"
+hive_metastore = "0.2.0"
 http = "1.2"
 iceberg = { version = "0.6.0", path = "./crates/iceberg" }
 iceberg-catalog-rest = { version = "0.6.0", path = "./crates/catalog/rest" }
+iceberg-catalog-glue = { version = "0.6.0", path = "./crates/catalog/glue" }
+iceberg-catalog-s3tables = { version = "0.6.0", path = "./crates/catalog/s3tables" }
+iceberg-catalog-hms = { version = "0.6.0", path = "./crates/catalog/hms" }
 iceberg-datafusion = { version = "0.6.0", path = "./crates/integrations/datafusion" }
 indicatif = "0.17"
 itertools = "0.13"
@@ -92,7 +96,7 @@ once_cell = "1.20"
 opendal = "0.54.0"
 ordered-float = "4"
 parquet = "55"
-pilota = "0.11.2"
+pilota = "0.11.10"
 port_scanner = "0.1.5"
 pretty_assertions = "1.4"
 rand = "0.8.5"
@@ -120,5 +124,5 @@ typed-builder = "0.20"
 url = "2.5.4"
 uuid = { version = "1.16", features = ["v7"] }
 volo = "0.10.6"
-volo-thrift = "0.10.6"
+volo-thrift = "0.10.8"
 zstd = "0.13.2"

Cargo.toml

@@ -37,7 +37,6 @@ iceberg = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
-typed-builder = { workspace = true }
 
 [dev-dependencies]
 ctor = { workspace = true }

crates/catalog/glue/Cargo.toml

@@ -0,0 +1 @@
+../../../LICENSE

crates/catalog/glue/LICENSE

@@ -0,0 +1 @@
+../../../NOTICE

crates/catalog/glue/NOTICE

@@ -26,10 +26,9 @@ use iceberg::io::{
 use iceberg::spec::{TableMetadata, TableMetadataBuilder};
 use iceberg::table::Table;
 use iceberg::{
-    Catalog, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit,
-    TableCreation, TableIdent,
+    Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result,
+    TableCommit, TableCreation, TableIdent,
 };
-use typed_builder::TypedBuilder;
 
 use crate::error::{from_aws_build_error, from_aws_sdk_error};
 use crate::utils::{
@@ -40,15 +39,90 @@ use crate::{
     AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, with_catalog_id,
 };
 
-#[derive(Debug, TypedBuilder)]
+/// Glue catalog URI
+pub const GLUE_CATALOG_PROP_URI: &str = "uri";
+/// Glue catalog id
+pub const GLUE_CATALOG_PROP_CATALOG_ID: &str = "catalog_id";
+/// Glue catalog warehouse location
+pub const GLUE_CATALOG_PROP_WAREHOUSE: &str = "warehouse";
+
+/// Builder for [`GlueCatalog`].
+#[derive(Debug)]
+pub struct GlueCatalogBuilder(GlueCatalogConfig);
+
+impl Default for GlueCatalogBuilder {
+    fn default() -> Self {
+        Self(GlueCatalogConfig {
+            name: None,
+            uri: None,
+            catalog_id: None,
+            warehouse: "".to_string(),
+            props: HashMap::new(),
+        })
+    }
+}
+
+impl CatalogBuilder for GlueCatalogBuilder {
+    type C = GlueCatalog;
+
+    fn load(
+        mut self,
+        name: impl Into<String>,
+        props: HashMap<String, String>,
+    ) -> impl Future<Output = Result<Self::C>> + Send {
+        self.0.name = Some(name.into());
+
+        if props.contains_key(GLUE_CATALOG_PROP_URI) {
+            self.0.uri = props.get(GLUE_CATALOG_PROP_URI).cloned()
+        }
+
+        if props.contains_key(GLUE_CATALOG_PROP_CATALOG_ID) {
+            self.0.catalog_id = props.get(GLUE_CATALOG_PROP_CATALOG_ID).cloned()
+        }
+
+        if props.contains_key(GLUE_CATALOG_PROP_WAREHOUSE) {
+            self.0.warehouse = props
+                .get(GLUE_CATALOG_PROP_WAREHOUSE)
+                .cloned()
+                .unwrap_or_default();
+        }
+
+        // Collect other remaining properties
+        self.0.props = props
+            .into_iter()
+            .filter(|(k, _)| {
+                k != GLUE_CATALOG_PROP_URI
+                    && k != GLUE_CATALOG_PROP_CATALOG_ID
+                    && k != GLUE_CATALOG_PROP_WAREHOUSE
+            })
+            .collect();
+
+        async move {
+            if self.0.name.is_none() {
+                return Err(Error::new(
+                    ErrorKind::DataInvalid,
+                    "Catalog name is required",
+                ));
+            }
+            if self.0.warehouse.is_empty() {
+                return Err(Error::new(
+                    ErrorKind::DataInvalid,
+                    "Catalog warehouse is required",
+                ));
+            }
+
+            GlueCatalog::new(self.0).await
+        }
+    }
+}
+
+#[derive(Debug)]
 /// Glue Catalog configuration
-pub struct GlueCatalogConfig {
-    #[builder(default, setter(strip_option(fallback = uri_opt)))]
+pub(crate) struct GlueCatalogConfig {
+    name: Option<String>,
     uri: Option<String>,
-    #[builder(default, setter(strip_option(fallback = catalog_id_opt)))]
     catalog_id: Option<String>,
     warehouse: String,
-    #[builder(default)]
     props: HashMap<String, String>,
 }
 
@@ -71,7 +145,7 @@ impl Debug for GlueCatalog {
 
 impl GlueCatalog {
     /// Create a new glue catalog
-    pub async fn new(config: GlueCatalogConfig) -> Result<Self> {
+    async fn new(config: GlueCatalogConfig) -> Result<Self> {
         let sdk_config = create_sdk_config(&config.props, config.uri.as_ref()).await;
         let mut file_io_props = config.props.clone();
         if !file_io_props.contains_key(S3_ACCESS_KEY_ID) {

crates/catalog/glue/src/catalog.rs

@@ -16,6 +16,30 @@
 // under the License.
 
 //! Iceberg Glue Catalog implementation.
+//!
+//! To build a glue catalog with configurations
+//! # Example
+//!
+//! ```rust, no_run
+//! use std::collections::HashMap;
+//!
+//! use iceberg::CatalogBuilder;
+//! use iceberg_catalog_glue::{GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalogBuilder};
+//!
+//! #[tokio::main]
+//! async fn main() {
+//!     let catalog = GlueCatalogBuilder::default()
+//!         .load(
+//!             "glue",
+//!             HashMap::from([(
+//!                 GLUE_CATALOG_PROP_WAREHOUSE.to_string(),
+//!                 "s3://warehouse".to_string(),
+//!             )]),
+//!         )
+//!         .await
+//!         .unwrap();
+//! }
+//! ```
 
 #![deny(missing_docs)]
 

crates/catalog/glue/src/lib.rs

@@ -24,9 +24,12 @@ use std::sync::RwLock;
 use ctor::{ctor, dtor};
 use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY};
 use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
-use iceberg::{Catalog, Namespace, NamespaceIdent, Result, TableCreation, TableIdent};
+use iceberg::{
+    Catalog, CatalogBuilder, Namespace, NamespaceIdent, Result, TableCreation, TableIdent,
+};
 use iceberg_catalog_glue::{
-    AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GlueCatalog, GlueCatalogConfig,
+    AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI,
+    GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder,
 };
 use iceberg_test_utils::docker::DockerCompose;
 use iceberg_test_utils::{normalize_test_name, set_up};
@@ -112,13 +115,22 @@ async fn get_catalog() -> GlueCatalog {
         retries += 1;
     }
 
-    let config = GlueCatalogConfig::builder()
-        .uri(format!("http://{}", glue_socket_addr))
-        .warehouse("s3a://warehouse/hive".to_string())
-        .props(props.clone())
-        .build();
+    let mut glue_props = HashMap::from([
+        (
+            GLUE_CATALOG_PROP_URI.to_string(),
+            format!("http://{}", glue_socket_addr),
+        ),
+        (
+            GLUE_CATALOG_PROP_WAREHOUSE.to_string(),
+            "s3a://warehouse/hive".to_string(),
+        ),
+    ]);
+    glue_props.extend(props.clone());
 
-    GlueCatalog::new(config).await.unwrap()
+    GlueCatalogBuilder::default()
+        .load("glue", glue_props)
+        .await
+        .unwrap()
 }
 
 async fn set_test_namespace(catalog: &GlueCatalog, namespace: &NamespaceIdent) -> Result<()> {

crates/catalog/glue/tests/glue_catalog_test.rs

@@ -38,7 +38,6 @@ pilota = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
-typed-builder = { workspace = true }
 volo-thrift = { workspace = true }
 
 # Transitive dependencies below

crates/catalog/hms/Cargo.toml

@@ -0,0 +1 @@
+../../../LICENSE