From 4511ae5b0010086c43692efae1433a2cebca7c57 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Thu, 5 Jun 2025 05:57:52 +0000 Subject: [PATCH] feat: implement `clickhouse-user-query` --- Cargo.lock | 331 +++++++++++++- .../common/clickhouse-user-query/Cargo.toml | 16 + .../clickhouse-user-query/src/builder.rs | 207 +++++++++ .../common/clickhouse-user-query/src/error.rs | 24 + .../common/clickhouse-user-query/src/lib.rs | 60 +++ .../common/clickhouse-user-query/src/query.rs | 73 +++ .../clickhouse-user-query/src/schema.rs | 72 +++ .../tests/builder_tests.rs | 211 +++++++++ .../tests/integration_tests.rs | 419 ++++++++++++++++++ .../tests/query_tests.rs | 165 +++++++ .../tests/schema_tests.rs | 27 ++ 11 files changed, 1590 insertions(+), 15 deletions(-) create mode 100644 packages/common/clickhouse-user-query/Cargo.toml create mode 100644 packages/common/clickhouse-user-query/src/builder.rs create mode 100644 packages/common/clickhouse-user-query/src/error.rs create mode 100644 packages/common/clickhouse-user-query/src/lib.rs create mode 100644 packages/common/clickhouse-user-query/src/query.rs create mode 100644 packages/common/clickhouse-user-query/src/schema.rs create mode 100644 packages/common/clickhouse-user-query/tests/builder_tests.rs create mode 100644 packages/common/clickhouse-user-query/tests/integration_tests.rs create mode 100644 packages/common/clickhouse-user-query/tests/query_tests.rs create mode 100644 packages/common/clickhouse-user-query/tests/schema_tests.rs diff --git a/Cargo.lock b/Cargo.lock index f7dd11e002..6ff12d4620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2330,6 +2330,56 @@ dependencies = [ "cipher", ] +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.1.0", + "http-body-util", + "hyper 1.6.0", + "hyper-named-pipe", + "hyper-rustls 0.27.3", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls 0.23.25", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.12", + "tokio", + "tokio-util 0.7.12", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with 3.12.0", +] + [[package]] name = "boxed_error" version = "0.2.3" @@ -3039,7 +3089,7 @@ dependencies = [ "rivet-test-images", "rivet-util", "serde", - "testcontainers", + "testcontainers 0.12.0", "thiserror 1.0.69", "tokio", "tokio-util 0.7.12", @@ -3110,7 +3160,7 @@ dependencies = [ "rivet-test-images", "rivet-util", "serde_json", - "testcontainers", + "testcontainers 0.12.0", "thiserror 1.0.69", "tokio", "tracing", @@ -3220,6 +3270,12 @@ dependencies = [ "inout", ] +[[package]] +name = "cityhash-rs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a719913643003b84bd13022b4b7e703c09342cd03b679c4641c7d2e50dc34d" + [[package]] name = "cjson" version = "0.1.2" @@ -3292,13 +3348,13 @@ checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" dependencies = [ "bstr", "bytes", - "clickhouse-derive", + "clickhouse-derive 0.1.1", "clickhouse-rs-cityhash-sys", "futures", "hyper 0.14.31", "hyper-tls 0.5.0", "lz4", - "sealed", + "sealed 0.4.0", "serde", "static_assertions", "thiserror 1.0.69", @@ -3307,6 +3363,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "clickhouse" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3093f817c4f81c8bd174ed8dd30eac785821a8a7eef27a7dcb7f8cd0d0f6548" +dependencies = [ + "bstr", + "bytes", + "cityhash-rs", + "clickhouse-derive 0.2.0", + "futures", + "futures-channel", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "lz4_flex", + "replace_with", + "sealed 0.5.0", + "serde", + "static_assertions", + "thiserror 1.0.69", + "tokio", + "url", +] + [[package]] name = "clickhouse-derive" version = "0.1.1" @@ -3319,6 +3400,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "clickhouse-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70f3e2893f7d3e017eeacdc9a708fbc29a10488e3ebca21f9df6a5d2b616dbb" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals 0.29.1", + "syn 2.0.90", +] + [[package]] name = "clickhouse-inserter" version = "25.5.2" @@ -3345,6 +3438,18 @@ dependencies = [ "cc", ] +[[package]] +name = "clickhouse-user-query" +version = "25.4.2" +dependencies = [ + "clickhouse 0.12.2", + "serde", + "serde_json", + "testcontainers 0.24.0", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "clipboard-win" version = "5.4.0" @@ -5522,6 +5627,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "document-features" version = "0.2.10" @@ -5920,6 +6036,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "etcetera" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c7b13d0780cb82722fd59f6f57f925e143427e4a75313a6c77243bf5326ae6" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.59.0", +] + [[package]] name = "event-listener" version = "5.3.1" @@ -7601,6 +7728,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.22.1" @@ -7728,6 +7870,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -8227,7 +8384,7 @@ dependencies = [ "chirp-client", "chirp-worker", "chrono", - "clickhouse", + "clickhouse 0.11.6", "prost 0.10.4", "rivet-operation", "serde", @@ -8239,7 +8396,7 @@ version = "25.5.2" dependencies = [ "chirp-client", "chirp-worker", - "clickhouse", + "clickhouse 0.11.6", "reqwest 0.11.27", "rivet-config", "rivet-health-checks", @@ -9623,7 +9780,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -10534,6 +10691,31 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax 0.8.5", + "structmeta", + "syn 2.0.90", +] + [[package]] name = "password-hash" version = "0.5.0" @@ -11726,6 +11908,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -11905,6 +12096,12 @@ dependencies = [ "sqlx", ] +[[package]] +name = "replace_with" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884" + [[package]] name = "reqwest" version = "0.11.27" @@ -12672,7 +12869,7 @@ version = "25.5.2" dependencies = [ "anyhow", "async-nats", - "clickhouse", + "clickhouse 0.11.6", "clickhouse-inserter", "dirs", "divan", @@ -12927,7 +13124,7 @@ dependencies = [ name = "rivet-test-images" version = "25.5.2" dependencies = [ - "testcontainers", + "testcontainers 0.12.0", ] [[package]] @@ -13260,7 +13457,7 @@ dependencies = [ "openssl-probe", "rustls 0.19.1", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -13272,7 +13469,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -13285,7 +13482,19 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", ] [[package]] @@ -13577,6 +13786,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "sealed" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a8caec23b7800fb97971a1c6ae365b6239aaeddfb934d6265f8505e795699d" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "sec1" version = "0.3.0" @@ -13619,11 +13840,24 @@ dependencies = [ "security-framework-sys", ] +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.0", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -14395,7 +14629,7 @@ dependencies = [ "byteorder", "crc", "dotenvy", - "etcetera", + "etcetera 0.8.0", "futures-channel", "futures-core", "futures-util", @@ -14585,6 +14819,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.90", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "strum" version = "0.24.1" @@ -15505,6 +15762,35 @@ dependencies = [ "sha2 0.9.9", ] +[[package]] +name = "testcontainers" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bb7577dca13ad86a78e8271ef5d322f37229ec83b8d98da6d996c588a1ddb1" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera 0.10.0", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with 3.12.0", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util 0.7.12", + "url", +] + [[package]] name = "text_lines" version = "0.6.0" @@ -15853,6 +16139,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-tungstenite" version = "0.21.0" diff --git a/packages/common/clickhouse-user-query/Cargo.toml b/packages/common/clickhouse-user-query/Cargo.toml new file mode 100644 index 0000000000..74ee737d94 --- /dev/null +++ b/packages/common/clickhouse-user-query/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "clickhouse-user-query" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +clickhouse = "0.12" +thiserror = "1.0" +serde = { version = "1.0", features = ["derive"] } + +[dev-dependencies] +serde_json = "1.0" +testcontainers = "0.24" +tokio = { version = "1.0", features = ["full"] } diff --git a/packages/common/clickhouse-user-query/src/builder.rs b/packages/common/clickhouse-user-query/src/builder.rs new file mode 100644 index 0000000000..8b2c854e58 --- /dev/null +++ b/packages/common/clickhouse-user-query/src/builder.rs @@ -0,0 +1,207 @@ +use clickhouse::query::Query; +use clickhouse::sql::Identifier; +use serde::{Deserialize, Serialize}; + +use crate::error::{Result, UserQueryError}; +use crate::query::QueryExpr; +use crate::schema::{PropertyType, Schema}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UserDefinedQueryBuilder { + where_clause: String, + bind_values: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum BindValue { + Bool(bool), + String(String), + Number(f64), + ArrayString(Vec), +} + +impl UserDefinedQueryBuilder { + pub fn new(schema: &Schema, expr: &QueryExpr) -> Result { + let mut builder = QueryBuilder::new(schema); + let where_clause = builder.build_where_clause(expr)?; + + if where_clause.trim().is_empty() { + return Err(UserQueryError::EmptyQuery); + } + + Ok(Self { + where_clause, + bind_values: builder.bind_values, + }) + } + + pub fn bind_to(&self, mut query: Query) -> Query { + for bind_value in &self.bind_values { + query = match bind_value { + BindValue::Bool(v) => query.bind(*v), + BindValue::String(v) => query.bind(v), + BindValue::Number(v) => query.bind(*v), + BindValue::ArrayString(v) => query.bind(v), + }; + } + query + } + + pub fn where_expr(&self) -> &str { + &self.where_clause + } +} + +struct QueryBuilder<'a> { + schema: &'a Schema, + bind_values: Vec, +} + +impl<'a> QueryBuilder<'a> { + fn new(schema: &'a Schema) -> Self { + Self { + schema, + bind_values: Vec::new(), + } + } + + fn build_where_clause(&mut self, expr: &QueryExpr) -> Result { + match expr { + QueryExpr::And { exprs } => { + if exprs.is_empty() { + return Err(UserQueryError::EmptyQuery); + } + let clauses: Result> = exprs + .iter() + .map(|e| self.build_where_clause(e)) + .collect(); + Ok(format!("({})", clauses?.join(" AND "))) + } + QueryExpr::Or { exprs } => { + if exprs.is_empty() { + return Err(UserQueryError::EmptyQuery); + } + let clauses: Result> = exprs + .iter() + .map(|e| self.build_where_clause(e)) + .collect(); + Ok(format!("({})", clauses?.join(" OR "))) + } + QueryExpr::BoolEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Bool)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Bool(*value)); + Ok(format!("{} = ?", column)) + } + QueryExpr::BoolNotEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Bool)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Bool(*value)); + Ok(format!("{} != ?", column)) + } + QueryExpr::StringEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::String)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::String(value.clone())); + Ok(format!("{} = ?", column)) + } + QueryExpr::StringNotEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::String)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::String(value.clone())); + Ok(format!("{} != ?", column)) + } + QueryExpr::ArrayContains { property, subproperty, values } => { + if values.is_empty() { + return Err(UserQueryError::EmptyArrayValues("ArrayContains".to_string())); + } + self.validate_property_access(property, subproperty, &PropertyType::ArrayString)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::ArrayString(values.clone())); + Ok(format!("hasAny({}, ?)", column)) + } + QueryExpr::ArrayDoesNotContain { property, subproperty, values } => { + if values.is_empty() { + return Err(UserQueryError::EmptyArrayValues("ArrayDoesNotContain".to_string())); + } + self.validate_property_access(property, subproperty, &PropertyType::ArrayString)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::ArrayString(values.clone())); + Ok(format!("NOT hasAny({}, ?)", column)) + } + QueryExpr::NumberEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} = ?", column)) + } + QueryExpr::NumberNotEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} != ?", column)) + } + QueryExpr::NumberLess { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} < ?", column)) + } + QueryExpr::NumberLessOrEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} <= ?", column)) + } + QueryExpr::NumberGreater { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} > ?", column)) + } + QueryExpr::NumberGreaterOrEqual { property, subproperty, value } => { + self.validate_property_access(property, subproperty, &PropertyType::Number)?; + let column = self.build_column_reference(property, subproperty)?; + self.bind_values.push(BindValue::Number(*value)); + Ok(format!("{} >= ?", column)) + } + } + } + + fn validate_property_access( + &self, + property: &str, + subproperty: &Option, + expected_type: &PropertyType, + ) -> Result<()> { + let prop = self.schema.get_property(property) + .ok_or_else(|| UserQueryError::PropertyNotFound(property.to_string()))?; + + if subproperty.is_some() && !prop.supports_subproperties { + return Err(UserQueryError::SubpropertiesNotSupported(property.to_string())); + } + + if &prop.ty != expected_type { + return Err(UserQueryError::PropertyTypeMismatch( + property.to_string(), + expected_type.type_name().to_string(), + prop.ty.type_name().to_string(), + )); + } + + Ok(()) + } + + fn build_column_reference(&self, property: &str, subproperty: &Option) -> Result { + let property_ident = Identifier(property); + + match subproperty { + Some(subprop) => { + // For ClickHouse Map access, use string literal syntax + Ok(format!("{}[{}]", property_ident.0, format!("'{}'", subprop.replace("'", "\\'")))) + } + None => Ok(property_ident.0.to_string()), + } + } +} + diff --git a/packages/common/clickhouse-user-query/src/error.rs b/packages/common/clickhouse-user-query/src/error.rs new file mode 100644 index 0000000000..5dbdc30d53 --- /dev/null +++ b/packages/common/clickhouse-user-query/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum UserQueryError { + #[error("Property '{0}' not found in schema")] + PropertyNotFound(String), + + #[error("Property '{0}' does not support subproperties")] + SubpropertiesNotSupported(String), + + #[error("Property '{0}' type mismatch: expected {1}, got {2}")] + PropertyTypeMismatch(String, String, String), + + #[error("Invalid property or subproperty name '{0}': must contain only alphanumeric characters and underscores")] + InvalidPropertyName(String), + + #[error("Empty query expression")] + EmptyQuery, + + #[error("Empty array values in {0} operation")] + EmptyArrayValues(String), +} + +pub type Result = std::result::Result; \ No newline at end of file diff --git a/packages/common/clickhouse-user-query/src/lib.rs b/packages/common/clickhouse-user-query/src/lib.rs new file mode 100644 index 0000000000..be7896fec7 --- /dev/null +++ b/packages/common/clickhouse-user-query/src/lib.rs @@ -0,0 +1,60 @@ +//! Safe ClickHouse user-defined query builder +//! +//! This crate provides a safe way to build ClickHouse queries from user-defined expressions +//! while protecting against SQL injection attacks. All user inputs are properly validated +//! and bound using parameterized queries. +//! +//! # Example +//! +//! ```rust +//! use clickhouse_user_query::*; +//! +//! // Define the schema of allowed properties +//! let schema = Schema::new(vec![ +//! Property::new("user_id".to_string(), false, PropertyType::String).unwrap(), +//! Property::new("metadata".to_string(), true, PropertyType::String).unwrap(), +//! Property::new("active".to_string(), false, PropertyType::Bool).unwrap(), +//! Property::new("tags".to_string(), false, PropertyType::ArrayString).unwrap(), +//! ]).unwrap(); +//! +//! // Build a complex query expression +//! let query_expr = QueryExpr::And { +//! exprs: vec![ +//! QueryExpr::StringEqual { +//! property: "user_id".to_string(), +//! subproperty: None, +//! value: "12345".to_string(), +//! }, +//! QueryExpr::BoolEqual { +//! property: "active".to_string(), +//! subproperty: None, +//! value: true, +//! }, +//! QueryExpr::ArrayContains { +//! property: "tags".to_string(), +//! subproperty: None, +//! values: vec!["premium".to_string(), "verified".to_string()], +//! }, +//! ], +//! }; +//! +//! // Create the safe query builder +//! let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); +//! +//! // Use with ClickHouse client (commented out since clickhouse client not available in tests) +//! // let query = clickhouse::Client::default() +//! // .query("SELECT * FROM users WHERE ?") +//! // .bind(builder.where_expr()); +//! // let final_query = builder.bind_to(query); +//! ``` + +// Re-export all public types for convenience +pub use builder::UserDefinedQueryBuilder; +pub use error::{Result, UserQueryError}; +pub use query::QueryExpr; +pub use schema::{Property, PropertyType, Schema}; + +pub mod builder; +pub mod error; +pub mod query; +pub mod schema; \ No newline at end of file diff --git a/packages/common/clickhouse-user-query/src/query.rs b/packages/common/clickhouse-user-query/src/query.rs new file mode 100644 index 0000000000..c6970e6525 --- /dev/null +++ b/packages/common/clickhouse-user-query/src/query.rs @@ -0,0 +1,73 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum QueryExpr { + And { + exprs: Vec, + }, + Or { + exprs: Vec, + }, + BoolEqual { + property: String, + subproperty: Option, + value: bool, + }, + BoolNotEqual { + property: String, + subproperty: Option, + value: bool, + }, + StringEqual { + property: String, + subproperty: Option, + value: String, + }, + StringNotEqual { + property: String, + subproperty: Option, + value: String, + }, + ArrayContains { + property: String, + subproperty: Option, + values: Vec, + }, + ArrayDoesNotContain { + property: String, + subproperty: Option, + values: Vec, + }, + NumberEqual { + property: String, + subproperty: Option, + value: f64, + }, + NumberNotEqual { + property: String, + subproperty: Option, + value: f64, + }, + NumberLess { + property: String, + subproperty: Option, + value: f64, + }, + NumberLessOrEqual { + property: String, + subproperty: Option, + value: f64, + }, + NumberGreater { + property: String, + subproperty: Option, + value: f64, + }, + NumberGreaterOrEqual { + property: String, + subproperty: Option, + value: f64, + }, +} + diff --git a/packages/common/clickhouse-user-query/src/schema.rs b/packages/common/clickhouse-user-query/src/schema.rs new file mode 100644 index 0000000000..3f45dea3e7 --- /dev/null +++ b/packages/common/clickhouse-user-query/src/schema.rs @@ -0,0 +1,72 @@ +use crate::error::{Result, UserQueryError}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Schema { + pub properties: Vec, +} + +impl Schema { + pub fn new(properties: Vec) -> Result { + // All property validation happens in Property::new() + Ok(Self { properties }) + } + + pub fn get_property(&self, name: &str) -> Option<&Property> { + self.properties.iter().find(|p| p.name == name) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Property { + pub name: String, + pub supports_subproperties: bool, + pub ty: PropertyType, +} + +impl Property { + pub fn new(name: String, supports_subproperties: bool, ty: PropertyType) -> Result { + validate_property_name(&name)?; + Ok(Self { + name, + supports_subproperties, + ty, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum PropertyType { + Bool, + String, + Number, + ArrayString, +} + +impl PropertyType { + pub fn type_name(&self) -> &'static str { + match self { + PropertyType::Bool => "bool", + PropertyType::String => "string", + PropertyType::Number => "number", + PropertyType::ArrayString => "array[string]", + } + } +} + +fn validate_property_name(name: &str) -> Result<()> { + if name.is_empty() { + return Err(UserQueryError::InvalidPropertyName(name.to_string())); + } + + if !name.chars().all(|c| c.is_alphanumeric() || c == '_') { + return Err(UserQueryError::InvalidPropertyName(name.to_string())); + } + + if name.chars().next().unwrap().is_numeric() { + return Err(UserQueryError::InvalidPropertyName(name.to_string())); + } + + Ok(()) +} + diff --git a/packages/common/clickhouse-user-query/tests/builder_tests.rs b/packages/common/clickhouse-user-query/tests/builder_tests.rs new file mode 100644 index 0000000000..71db71ce2f --- /dev/null +++ b/packages/common/clickhouse-user-query/tests/builder_tests.rs @@ -0,0 +1,211 @@ +use clickhouse_user_query::*; + +fn create_test_schema() -> Schema { + Schema::new(vec![ + Property::new("prop_a".to_string(), false, PropertyType::String).unwrap(), + Property::new("prop_b".to_string(), true, PropertyType::String).unwrap(), + Property::new("bool_prop".to_string(), false, PropertyType::Bool).unwrap(), + Property::new("number_prop".to_string(), false, PropertyType::Number).unwrap(), + Property::new("array_prop".to_string(), false, PropertyType::ArrayString).unwrap(), + ]).unwrap() +} + +#[test] +fn test_simple_string_equal() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "prop_a".to_string(), + subproperty: None, + value: "foo".to_string(), + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "prop_a = ?"); +} + +#[test] +fn test_subproperty_access() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "prop_b".to_string(), + subproperty: Some("sub".to_string()), + value: "bar".to_string(), + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "prop_b['sub'] = ?"); +} + +#[test] +fn test_and_query() { + let schema = create_test_schema(); + let query = QueryExpr::And { + exprs: vec![ + QueryExpr::StringEqual { + property: "prop_a".to_string(), + subproperty: None, + value: "foo".to_string(), + }, + QueryExpr::BoolEqual { + property: "bool_prop".to_string(), + subproperty: None, + value: true, + }, + ], + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "(prop_a = ? AND bool_prop = ?)"); +} + +#[test] +fn test_array_contains() { + let schema = create_test_schema(); + let query = QueryExpr::ArrayContains { + property: "array_prop".to_string(), + subproperty: None, + values: vec!["val1".to_string(), "val2".to_string()], + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "hasAny(array_prop, ?)"); +} + +#[test] +fn test_property_not_found() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "nonexistent".to_string(), + subproperty: None, + value: "foo".to_string(), + }; + + let result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(result, Err(UserQueryError::PropertyNotFound(_)))); +} + +#[test] +fn test_type_mismatch() { + let schema = create_test_schema(); + let query = QueryExpr::BoolEqual { + property: "prop_a".to_string(), // This is a string property + subproperty: None, + value: true, + }; + + let result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(result, Err(UserQueryError::PropertyTypeMismatch(_, _, _)))); +} + +#[test] +fn test_subproperties_not_supported() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "prop_a".to_string(), // This doesn't support subproperties + subproperty: Some("sub".to_string()), + value: "foo".to_string(), + }; + + let result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(result, Err(UserQueryError::SubpropertiesNotSupported(_)))); +} + +#[test] +fn test_invalid_property_name() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "prop-with-dashes".to_string(), + subproperty: None, + value: "foo".to_string(), + }; + + // Invalid property names are now caught as "not found" since schema validation + // happens at schema creation time, not query time + let builder_result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(builder_result, Err(UserQueryError::PropertyNotFound(_)))); +} + +#[test] +fn test_subproperty_with_special_chars() { + let schema = create_test_schema(); + let query = QueryExpr::StringEqual { + property: "prop_b".to_string(), // This supports subproperties + subproperty: Some("sub-with-dashes".to_string()), + value: "foo".to_string(), + }; + + // Subproperties with special characters should work fine with Identifier escaping + let builder_result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(builder_result.is_ok()); + + let builder = builder_result.unwrap(); + assert_eq!(builder.where_expr(), "prop_b['sub-with-dashes'] = ?"); +} + +#[test] +fn test_empty_array_values() { + let schema = create_test_schema(); + let query = QueryExpr::ArrayContains { + property: "array_prop".to_string(), + subproperty: None, + values: vec![], + }; + + let result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(result, Err(UserQueryError::EmptyArrayValues(_)))); +} + +#[test] +fn test_number_greater() { + let schema = create_test_schema(); + let query = QueryExpr::NumberGreater { + property: "number_prop".to_string(), + subproperty: None, + value: 42.5, + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "number_prop > ?"); +} + +#[test] +fn test_number_less_or_equal() { + let schema = create_test_schema(); + let query = QueryExpr::NumberLessOrEqual { + property: "number_prop".to_string(), + subproperty: None, + value: 100.0, + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "number_prop <= ?"); +} + +#[test] +fn test_number_with_subproperty() { + let schema = Schema::new(vec![ + Property::new("metrics".to_string(), true, PropertyType::Number).unwrap(), + ]).unwrap(); + + let query = QueryExpr::NumberEqual { + property: "metrics".to_string(), + subproperty: Some("score".to_string()), + value: 95.5, + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query).unwrap(); + assert_eq!(builder.where_expr(), "metrics['score'] = ?"); +} + +#[test] +fn test_number_type_mismatch() { + let schema = create_test_schema(); + let query = QueryExpr::NumberGreater { + property: "prop_a".to_string(), // This is a String type, not Number + subproperty: None, + value: 42.0, + }; + + let result = UserDefinedQueryBuilder::new(&schema, &query); + assert!(matches!(result, Err(UserQueryError::PropertyTypeMismatch(_, _, _)))); +} \ No newline at end of file diff --git a/packages/common/clickhouse-user-query/tests/integration_tests.rs b/packages/common/clickhouse-user-query/tests/integration_tests.rs new file mode 100644 index 0000000000..b612afc190 --- /dev/null +++ b/packages/common/clickhouse-user-query/tests/integration_tests.rs @@ -0,0 +1,419 @@ +use clickhouse::{Client, Row}; +use clickhouse_user_query::*; +use serde::Deserialize; +use serde_json; +use testcontainers::{runners::AsyncRunner, ContainerAsync, GenericImage, core::ContainerPort}; + +#[derive(Row, Deserialize)] +struct UserRow { + id: String, +} + +struct TestSetup { + client: Client, + _container: ContainerAsync, +} + +impl TestSetup { + async fn new() -> Self { + let clickhouse_image = GenericImage::new("clickhouse/clickhouse-server", "23.8-alpine") + .with_exposed_port(ContainerPort::Tcp(8123)) + .with_exposed_port(ContainerPort::Tcp(9000)); + + let container = clickhouse_image.start().await.expect("Failed to start ClickHouse container"); + + let port = container.get_host_port_ipv4(8123).await.expect("Failed to get port"); + let client = Client::default() + .with_url(format!("http://localhost:{}", port)); + + // Wait for ClickHouse to be ready and create test table + let setup = Self { + client, + _container: container, + }; + + // Wait for ClickHouse to fully start up + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + setup.setup_test_data().await; + setup + } + + async fn setup_test_data(&self) { + // Create test table with sample data + self.client + .query("CREATE TABLE IF NOT EXISTS test_users ( + id String, + active Bool, + metadata Map(String, String), + tags Array(String), + age UInt32, + score Float64 + ) ENGINE = Memory") + .execute() + .await + .expect("Failed to create test table"); + + // Insert test data + self.client + .query("INSERT INTO test_users VALUES + ('user1', true, {'region': 'us-east', 'tier': 'premium'}, ['verified', 'premium'], 25, 95.5), + ('user2', false, {'region': 'us-west', 'tier': 'basic'}, ['basic'], 30, 67.2), + ('user3', true, {'region': 'eu', 'tier': 'premium'}, ['verified', 'premium', 'beta'], 22, 88.9)") + .execute() + .await + .expect("Failed to insert test data"); + } +} + +#[tokio::test] +async fn test_simple_query_execution() { + let setup = TestSetup::new().await; + + // Create schema + let schema = Schema::new(vec![ + Property::new("active".to_string(), false, PropertyType::Bool).unwrap(), + ]).unwrap(); + + // Create query + let query_expr = QueryExpr::BoolEqual { + property: "active".to_string(), + subproperty: None, + value: true, + }; + + // Build query + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + // Execute query + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return user1 and user3 (active users) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user1".to_string())); + assert!(result.contains(&"user3".to_string())); +} + +#[tokio::test] +async fn test_subproperty_query_execution() { + let setup = TestSetup::new().await; + + // Create schema with map support + let schema = Schema::new(vec![ + Property::new("metadata".to_string(), true, PropertyType::String).unwrap(), + ]).unwrap(); + + // Query for premium tier users + let query_expr = QueryExpr::StringEqual { + property: "metadata".to_string(), + subproperty: Some("tier".to_string()), + value: "premium".to_string(), + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return user1 and user3 (premium tier) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user1".to_string())); + assert!(result.contains(&"user3".to_string())); +} + +#[tokio::test] +async fn test_array_contains_query_execution() { + let setup = TestSetup::new().await; + + // Create schema with array support + let schema = Schema::new(vec![ + Property::new("tags".to_string(), false, PropertyType::ArrayString).unwrap(), + ]).unwrap(); + + // Query for users with specific tags + let query_expr = QueryExpr::ArrayContains { + property: "tags".to_string(), + subproperty: None, + values: vec!["verified".to_string(), "beta".to_string()], + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return user1 and user3 (have verified) and user3 (has beta) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user1".to_string())); + assert!(result.contains(&"user3".to_string())); +} + +#[tokio::test] +async fn test_complex_and_or_query_execution() { + let setup = TestSetup::new().await; + + // Create comprehensive schema + let schema = Schema::new(vec![ + Property::new("active".to_string(), false, PropertyType::Bool).unwrap(), + Property::new("metadata".to_string(), true, PropertyType::String).unwrap(), + Property::new("tags".to_string(), false, PropertyType::ArrayString).unwrap(), + ]).unwrap(); + + // Complex query: (active = true AND metadata['tier'] = 'premium') OR tags contains 'beta' + let query_expr = QueryExpr::Or { + exprs: vec![ + QueryExpr::And { + exprs: vec![ + QueryExpr::BoolEqual { + property: "active".to_string(), + subproperty: None, + value: true, + }, + QueryExpr::StringEqual { + property: "metadata".to_string(), + subproperty: Some("tier".to_string()), + value: "premium".to_string(), + }, + ], + }, + QueryExpr::ArrayContains { + property: "tags".to_string(), + subproperty: None, + values: vec!["beta".to_string()], + }, + ], + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return: + // - user1 (active=true AND tier=premium) + // - user3 (active=true AND tier=premium AND has beta tag) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user1".to_string())); + assert!(result.contains(&"user3".to_string())); +} + +#[tokio::test] +async fn test_sql_injection_protection() { + let setup = TestSetup::new().await; + + // Create schema + let schema = Schema::new(vec![ + Property::new("metadata".to_string(), true, PropertyType::String).unwrap(), + ]).unwrap(); + + // Attempt SQL injection in subproperty + let query_expr = QueryExpr::StringEqual { + property: "metadata".to_string(), + subproperty: Some("'; DROP TABLE test_users; --".to_string()), + value: "malicious".to_string(), + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + // Verify the query builds safely with proper escaping + let where_clause = builder.where_expr(); + assert!(where_clause.contains("metadata['\\'; DROP TABLE test_users; --']")); + assert!(where_clause.contains("= ?")); + + // Execute the query - it should run safely and return no results + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution should succeed safely") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return no results (not drop the table) + assert_eq!(result.len(), 0); + + // Verify table still exists by running a simple query + let table_check: Vec = setup.client + .query("SELECT id FROM test_users LIMIT 1") + .fetch_all::() + .await + .expect("Table should still exist") + .into_iter() + .map(|user| user.id) + .collect(); + + assert!(!table_check.is_empty(), "Table should not have been dropped"); +} + +#[tokio::test] +async fn test_json_serialization_roundtrip() { + let setup = TestSetup::new().await; + + // Create schema + let schema = Schema::new(vec![ + Property::new("active".to_string(), false, PropertyType::Bool).unwrap(), + Property::new("metadata".to_string(), true, PropertyType::String).unwrap(), + ]).unwrap(); + + // Create complex query + let original_query = QueryExpr::And { + exprs: vec![ + QueryExpr::BoolEqual { + property: "active".to_string(), + subproperty: None, + value: true, + }, + QueryExpr::StringEqual { + property: "metadata".to_string(), + subproperty: Some("tier".to_string()), + value: "premium".to_string(), + }, + ], + }; + + // Serialize to JSON + let json = serde_json::to_string(&original_query).unwrap(); + + // Deserialize from JSON + let deserialized_query: QueryExpr = serde_json::from_str(&json).unwrap(); + + // Build queries from both and verify they're identical + let original_builder = UserDefinedQueryBuilder::new(&schema, &original_query).unwrap(); + let deserialized_builder = UserDefinedQueryBuilder::new(&schema, &deserialized_query).unwrap(); + + assert_eq!(original_builder.where_expr(), deserialized_builder.where_expr()); + + // Execute both queries and verify results are the same + let query1 = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", original_builder.where_expr())); + let query1 = original_builder.bind_to(query1); + + let query2 = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", deserialized_builder.where_expr())); + let query2 = deserialized_builder.bind_to(query2); + + let result1: Vec = query1 + .fetch_all::() + .await + .unwrap() + .into_iter() + .map(|user| user.id) + .collect(); + + let result2: Vec = query2 + .fetch_all::() + .await + .unwrap() + .into_iter() + .map(|user| user.id) + .collect(); + + assert_eq!(result1, result2); + assert_eq!(result1.len(), 2); // user1 and user3 +} + +#[tokio::test] +async fn test_numeric_query_execution() { + let setup = TestSetup::new().await; + + // Create schema with number support + let schema = Schema::new(vec![ + Property::new("score".to_string(), false, PropertyType::Number).unwrap(), + ]).unwrap(); + + // Query for users with score greater than 80 + let query_expr = QueryExpr::NumberGreater { + property: "score".to_string(), + subproperty: None, + value: 80.0, + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return user1 (95.5) and user3 (88.9), but not user2 (67.2) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user1".to_string())); + assert!(result.contains(&"user3".to_string())); + assert!(!result.contains(&"user2".to_string())); +} + +#[tokio::test] +async fn test_numeric_less_or_equal_query() { + let setup = TestSetup::new().await; + + // Create schema with number support + let schema = Schema::new(vec![ + Property::new("score".to_string(), false, PropertyType::Number).unwrap(), + ]).unwrap(); + + // Query for users with score <= 90 + let query_expr = QueryExpr::NumberLessOrEqual { + property: "score".to_string(), + subproperty: None, + value: 90.0, + }; + + let builder = UserDefinedQueryBuilder::new(&schema, &query_expr).unwrap(); + + let query = setup.client.query(&format!("SELECT id FROM test_users WHERE {}", builder.where_expr())); + let query = builder.bind_to(query); + + let result: Vec = query + .fetch_all::() + .await + .expect("Query execution failed") + .into_iter() + .map(|user| user.id) + .collect(); + + // Should return user2 (67.2) and user3 (88.9), but not user1 (95.5) + assert_eq!(result.len(), 2); + assert!(result.contains(&"user2".to_string())); + assert!(result.contains(&"user3".to_string())); + assert!(!result.contains(&"user1".to_string())); +} \ No newline at end of file diff --git a/packages/common/clickhouse-user-query/tests/query_tests.rs b/packages/common/clickhouse-user-query/tests/query_tests.rs new file mode 100644 index 0000000000..a5834df7b5 --- /dev/null +++ b/packages/common/clickhouse-user-query/tests/query_tests.rs @@ -0,0 +1,165 @@ +use clickhouse_user_query::*; + +#[test] +fn test_query_expr_serde() { + let query = QueryExpr::StringEqual { + property: "user_id".to_string(), + subproperty: None, + value: "12345".to_string(), + }; + + // Test serialization + let json = serde_json::to_string_pretty(&query).unwrap(); + + assert!(json.contains(r#""string_equal""#)); + assert!(json.contains(r#""property": "user_id""#)); + assert!(json.contains(r#""value": "12345""#)); + + // Test deserialization + let deserialized: QueryExpr = serde_json::from_str(&json).unwrap(); + match deserialized { + QueryExpr::StringEqual { property, value, .. } => { + assert_eq!(property, "user_id"); + assert_eq!(value, "12345"); + } + _ => panic!("Expected StringEqual"), + } +} + +#[test] +fn test_complex_query_serde() { + let query = QueryExpr::And { + exprs: vec![ + QueryExpr::BoolEqual { + property: "active".to_string(), + subproperty: None, + value: true, + }, + QueryExpr::ArrayContains { + property: "tags".to_string(), + subproperty: Some("category".to_string()), + values: vec!["premium".to_string(), "verified".to_string()], + }, + ], + }; + + let json = serde_json::to_string_pretty(&query).unwrap(); + + assert!(json.contains(r#""and""#)); + assert!(json.contains(r#""bool_equal""#)); + assert!(json.contains(r#""array_contains""#)); + + let deserialized: QueryExpr = serde_json::from_str(&json).unwrap(); + match deserialized { + QueryExpr::And { exprs } => { + assert_eq!(exprs.len(), 2); + } + _ => panic!("Expected And expression"), + } +} + +#[test] +fn test_query_expr_creation() { + let query = QueryExpr::And { + exprs: vec![ + QueryExpr::StringEqual { + property: "user_id".to_string(), + subproperty: None, + value: "12345".to_string(), + }, + QueryExpr::BoolEqual { + property: "active".to_string(), + subproperty: None, + value: true, + }, + ], + }; + + match query { + QueryExpr::And { exprs } => { + assert_eq!(exprs.len(), 2); + } + _ => panic!("Expected And expression"), + } +} + +#[test] +fn test_subproperty_query() { + let query = QueryExpr::StringEqual { + property: "metadata".to_string(), + subproperty: Some("key".to_string()), + value: "value".to_string(), + }; + + match query { + QueryExpr::StringEqual { property, subproperty, value } => { + assert_eq!(property, "metadata"); + assert_eq!(subproperty, Some("key".to_string())); + assert_eq!(value, "value"); + } + _ => panic!("Expected StringEqual expression"), + } +} + +#[test] +fn test_array_query() { + let query = QueryExpr::ArrayContains { + property: "tags".to_string(), + subproperty: None, + values: vec!["premium".to_string(), "verified".to_string()], + }; + + match query { + QueryExpr::ArrayContains { property, values, .. } => { + assert_eq!(property, "tags"); + assert_eq!(values.len(), 2); + assert!(values.contains(&"premium".to_string())); + } + _ => panic!("Expected ArrayContains expression"), + } +} + +#[test] +fn test_numeric_query() { + let query = QueryExpr::NumberGreater { + property: "score".to_string(), + subproperty: None, + value: 85.5, + }; + + match query { + QueryExpr::NumberGreater { property, value, .. } => { + assert_eq!(property, "score"); + assert_eq!(value, 85.5); + } + _ => panic!("Expected NumberGreater expression"), + } +} + +#[test] +fn test_numeric_query_serde() { + let query = QueryExpr::NumberLessOrEqual { + property: "metrics".to_string(), + subproperty: Some("latency".to_string()), + value: 100.0, + }; + + // Test serialization + let json = serde_json::to_string_pretty(&query).unwrap(); + + assert!(json.contains(r#""number_less_or_equal""#)); + assert!(json.contains(r#""property": "metrics""#)); + assert!(json.contains(r#""subproperty": "latency""#)); + assert!(json.contains(r#""value": 100.0"#)); + + // Test deserialization + let deserialized: QueryExpr = serde_json::from_str(&json).unwrap(); + match deserialized { + QueryExpr::NumberLessOrEqual { property, subproperty, value } => { + assert_eq!(property, "metrics"); + assert_eq!(subproperty, Some("latency".to_string())); + assert_eq!(value, 100.0); + } + _ => panic!("Expected NumberLessOrEqual"), + } +} \ No newline at end of file diff --git a/packages/common/clickhouse-user-query/tests/schema_tests.rs b/packages/common/clickhouse-user-query/tests/schema_tests.rs new file mode 100644 index 0000000000..16c4a5bbfb --- /dev/null +++ b/packages/common/clickhouse-user-query/tests/schema_tests.rs @@ -0,0 +1,27 @@ +use clickhouse_user_query::*; + +#[test] +fn test_schema_creation() { + let schema = Schema::new(vec![ + Property::new("valid_name".to_string(), false, PropertyType::String).unwrap(), + Property::new("another_valid_123".to_string(), true, PropertyType::Bool).unwrap(), + ]).unwrap(); + + assert_eq!(schema.properties.len(), 2); + assert!(schema.get_property("valid_name").is_some()); + assert!(schema.get_property("nonexistent").is_none()); +} + +#[test] +fn test_invalid_property_name() { + let result = Property::new("invalid-name".to_string(), false, PropertyType::String); + assert!(result.is_err()); +} + +#[test] +fn test_property_type_names() { + assert_eq!(PropertyType::Bool.type_name(), "bool"); + assert_eq!(PropertyType::String.type_name(), "string"); + assert_eq!(PropertyType::Number.type_name(), "number"); + assert_eq!(PropertyType::ArrayString.type_name(), "array[string]"); +} \ No newline at end of file