From e252637a9898b96774920171e2ab1a15a18c7971 Mon Sep 17 00:00:00 2001 From: Hayden Hung Hoang Date: Wed, 27 Aug 2025 22:35:32 +0700 Subject: [PATCH 1/5] feat: added xtasks `fetch` and `code-gen` to generate code from openapi spec Also added `compose.yml` to start a typesense server locally for testing --- .cargo/config.toml | 2 + .gitignore | 2 + Cargo.toml | 3 +- compose.yml | 9 ++ xtask/Cargo.toml | 11 ++ xtask/src/main.rs | 134 ++++++++++++++++++ xtask/src/preprocess_openapi.rs | 236 ++++++++++++++++++++++++++++++++ 7 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 .cargo/config.toml create mode 100644 compose.yml create mode 100644 xtask/Cargo.toml create mode 100644 xtask/src/main.rs create mode 100644 xtask/src/preprocess_openapi.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..f0ccbc9 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" \ No newline at end of file diff --git a/.gitignore b/.gitignore index e551aa3..e1f7497 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target Cargo.lock .env + +/typesense-data \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index a378f77..23adab9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,8 @@ members = [ "typesense", "typesense_derive", - "typesense_codegen" + "typesense_codegen", + "xtask", ] resolver = "3" diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..c77de65 --- /dev/null +++ b/compose.yml @@ -0,0 +1,9 @@ +services: + typesense: + image: typesense/typesense:29.0 + restart: on-failure + ports: + - '8108:8108' + volumes: + - ./typesense-data:/data + command: '--data-dir /data --api-key=xyz --enable-cors' diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000..a394748 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" + +[dependencies] +reqwest = { version = "0.11", features = ["blocking"] } # "blocking" is simpler for scripts +anyhow = "1.0" +clap = { version = "4.0", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000..c95fe62 --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,134 @@ +use anyhow::{Context, Result}; +use clap::{Parser, ValueEnum}; +use std::env; +use std::fs; +use std::process::Command; +mod preprocess_openapi; +use preprocess_openapi::preprocess_openapi_file; + +const SPEC_URL: &str = + "https://raw.githubusercontent.com/typesense/typesense-api-spec/master/openapi.yml"; + +// Input spec file, expected in the project root. +const INPUT_SPEC_FILE: &str = "openapi.yml"; +const OUTPUT_PREPROCESSED_FILE: &str = "./preprocessed_openapi.yml"; + +// Output directory for the generated code. +const OUTPUT_DIR: &str = "typesense_codegen"; + +#[derive(Parser)] +#[command( + author, + version, + about = "A task runner for the typesense-rust project" +)] +struct Cli { + /// The list of tasks to run in sequence. + #[arg(required = true, value_enum)] + tasks: Vec, +} + +#[derive(ValueEnum, Clone, Debug)] +#[clap(rename_all = "kebab-case")] // Allows us to type `code-gen` instead of `CodeGen` +enum Task { + /// Fetches the latest OpenAPI spec from [the Typesense repository](https://github.com/typesense/typesense-api-spec/blob/master/openapi.yml). + Fetch, + /// Generates client code from the spec file using the Docker container. + CodeGen, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + for task in cli.tasks { + println!("▶️ Running task: {:?}", task); + match task { + Task::Fetch => task_fetch_api_spec()?, + Task::CodeGen => task_codegen()?, + } + } + Ok(()) +} + +fn task_fetch_api_spec() -> Result<()> { + println!("▶️ Running codegen task..."); + + println!(" - Downloading spec from {}", SPEC_URL); + let response = + reqwest::blocking::get(SPEC_URL).context("Failed to download OpenAPI spec file")?; + + if !response.status().is_success() { + anyhow::bail!("Failed to download spec: HTTP {}", response.status()); + } + + let spec_content = response.text()?; + fs::write(INPUT_SPEC_FILE, spec_content) + .context(format!("Failed to write spec to {}", INPUT_SPEC_FILE))?; + println!(" - Spec saved to {}", INPUT_SPEC_FILE); + + println!("✅ Fetch API spec task finished successfully."); + + Ok(()) +} + +/// Task to generate client code from the OpenAPI spec using a Docker container. +fn task_codegen() -> Result<()> { + println!("▶️ Running codegen task via Docker..."); + + println!("Preprocessing the Open API spec file..."); + preprocess_openapi_file(INPUT_SPEC_FILE, OUTPUT_PREPROCESSED_FILE) + .expect("Preprocess failed, aborting!"); + // Get the absolute path to the project's root directory. + // std::env::current_dir() gives us the directory from which `cargo xtask` was run. + let project_root = env::current_dir().context("Failed to get current directory")?; + + // Check if the input spec file exists before trying to run Docker. + let input_spec_path = project_root.join(INPUT_SPEC_FILE); + if !input_spec_path.exists() { + anyhow::bail!( + "Input spec '{}' not found in project root. Please add it before running.", + INPUT_SPEC_FILE + ); + } + + // Construct the volume mount string for Docker. + // Docker needs an absolute path for the volume mount source. + // to_string_lossy() is used to handle potential non-UTF8 paths gracefully. + let volume_mount = format!("{}:/local", project_root.to_string_lossy()); + println!(" - Using volume mount: {}", volume_mount); + + // Set up and run the Docker command. + println!(" - Starting Docker container..."); + let status = Command::new("docker") + .arg("run") + .arg("--rm") // Remove the container after it exits + .arg("-v") + .arg(volume_mount) // Mount the project root to /local in the container + .arg("openapitools/openapi-generator-cli") + .arg("generate") + .arg("-i") + .arg(format!("/local/{}", OUTPUT_PREPROCESSED_FILE)) // Input path inside the container + .arg("-g") + .arg("rust") + .arg("-o") + .arg(format!("/local/{}", OUTPUT_DIR)) // Output path inside the container + .arg("--additional-properties") + .arg("library=reqwest") + .arg("--additional-properties") + .arg("supportMiddleware=true") + .arg("--additional-properties") + .arg("useSingleRequestParameter=true") + // .arg("--additional-properties") + // .arg("useBonBuilder=true") + .status() + .context("Failed to execute Docker command. Is Docker installed and running?")?; + + // Check if the command was successful. + if !status.success() { + anyhow::bail!("Docker command failed with status: {}", status); + } + + println!("✅ Codegen task finished successfully."); + println!(" Generated code is available in '{}'", OUTPUT_DIR); + Ok(()) +} diff --git a/xtask/src/preprocess_openapi.rs b/xtask/src/preprocess_openapi.rs new file mode 100644 index 0000000..4189481 --- /dev/null +++ b/xtask/src/preprocess_openapi.rs @@ -0,0 +1,236 @@ +use serde_yaml::{Mapping, Value}; +use std::fs; + +// --- Main function to orchestrate the file reading, processing, and writing --- +pub fn preprocess_openapi_file( + input_path: &str, + output_path: &str, +) -> Result<(), Box> { + // --- Step 1: Read the OpenAPI spec from the input file --- + println!("Reading OpenAPI spec from {}...", input_path); + let input_content = fs::read_to_string(input_path) + .map_err(|e| format!("Failed to read {}: {}", input_path, e))?; + let mut doc: Value = serde_yaml::from_str(&input_content)?; + + // Ensure the root is a mutable mapping + let doc_root = doc + .as_mapping_mut() + .ok_or("OpenAPI spec root is not a YAML map")?; + + // --- Step 2: Apply all the required transformations --- + println!("Preprocessing the spec..."); + unwrap_search_parameters(doc_root)?; + unwrap_multi_search_parameters(doc_root)?; + unwrap_parameters_by_path( + doc_root, + "/collections/{collectionName}/documents/import", + "post", + "importDocumentsParameters", + Some("ImportDocumentsParameters"), // Copy schema to components + )?; + unwrap_parameters_by_path( + doc_root, + "/collections/{collectionName}/documents/export", + "get", + "exportDocumentsParameters", + Some("ExportDocumentsParameters"), // Copy schema to components + )?; + unwrap_parameters_by_path( + doc_root, + "/collections/{collectionName}/documents", + "patch", + "updateDocumentsParameters", + Some("UpdateDocumentsParameters"), // Copy schema to components + )?; + unwrap_parameters_by_path( + doc_root, + "/collections/{collectionName}/documents", + "delete", + "deleteDocumentsParameters", + Some("DeleteDocumentsParameters"), // Copy schema to components + )?; + println!("Preprocessing complete."); + + // --- Step 3: Serialize the modified spec and write to the output file --- + println!("Writing processed spec to {}...", output_path); + let output_yaml = serde_yaml::to_string(&doc)?; + fs::write(output_path, output_yaml) + .map_err(|e| format!("Failed to write {}: {}", output_path, e))?; + + println!("Successfully created {}.", output_path); + Ok(()) +} + +/// A generic function to: +/// 1. (Optional) Copy an inline parameter schema to `components/schemas`. +/// 2. Unwrap that parameter object into individual query parameters within the `paths` definition. +fn unwrap_parameters_by_path( + doc: &mut Mapping, + path: &str, + method: &str, + param_name_to_unwrap: &str, + new_component_name: Option<&str>, +) -> Result<(), String> { + // --- Step 1 (Optional): Copy the inline schema to components --- + if let Some(component_name) = new_component_name { + println!( + "- Copying inline schema for '{}' to components.schemas.{}...", + param_name_to_unwrap, component_name + ); + + // Find the parameter with the inline schema to copy using a read-only borrow + let params_for_copy = doc + .get("paths") + .and_then(|p| p.get(path)) + .and_then(|p| p.get(method)) + .and_then(|op| op.get("parameters")) + .and_then(|params| params.as_sequence()) + .ok_or_else(|| format!("Could not find parameters for {} {}", method, path))?; + + let param_to_copy = params_for_copy + .iter() + .find(|p| p.get("name").and_then(|n| n.as_str()) == Some(param_name_to_unwrap)) + .ok_or_else(|| format!("Parameter '{}' not found for copying", param_name_to_unwrap))?; + + let inline_schema = param_to_copy + .get("schema") + .cloned() // Clone the schema to avoid borrowing issues + .ok_or_else(|| format!("No schema found for '{}'", param_name_to_unwrap))?; + + // Get a mutable borrow to insert the cloned schema into components + let schemas = doc + .get_mut("components") + .and_then(|c| c.get_mut("schemas")) + .and_then(|s| s.as_mapping_mut()) + .ok_or_else(|| "Could not find components/schemas section".to_string())?; + + schemas.insert(component_name.into(), inline_schema); + } + + // --- Step 2: Unwrap the parameter object into individual parameters --- + println!( + "- Unwrapping parameter object '{}'...", + param_name_to_unwrap + ); + + // Navigate down to the operation's parameters list (mutable) + let params_for_unwrap = doc + .get_mut("paths") + .and_then(|p| p.get_mut(path)) + .and_then(|p| p.get_mut(method)) + .and_then(|op| op.get_mut("parameters")) + .and_then(|params| params.as_sequence_mut()) + .ok_or_else(|| format!("Could not find parameters for {} {}", method, path))?; + + let param_index = params_for_unwrap + .iter() + .position(|p| p.get("name").and_then(|n| n.as_str()) == Some(param_name_to_unwrap)) + .ok_or_else(|| format!("Parameter '{}' not found in {}", param_name_to_unwrap, path))?; + + let param_object = params_for_unwrap.remove(param_index); + let properties = param_object + .get("schema") + .and_then(|s| s.get("properties")) + .and_then(|p| p.as_mapping()) + .ok_or_else(|| { + format!( + "Could not extract properties from '{}'", + param_name_to_unwrap + ) + })?; + + for (key, value) in properties { + let mut new_param = Mapping::new(); + new_param.insert("name".into(), key.clone()); + new_param.insert("in".into(), "query".into()); + new_param.insert("schema".into(), value.clone()); + params_for_unwrap.push(new_param.into()); + } + + Ok(()) +} + +/// Special handler for unwrapping search parameters from `components/schemas`. +fn unwrap_search_parameters(doc: &mut Mapping) -> Result<(), String> { + println!("- Unwrapping searchParameters..."); + // Get the definition of SearchParameters from components + let search_params_props = doc + .get("components") + .and_then(|c| c.get("schemas")) + .and_then(|s| s.get("SearchParameters")) + .and_then(|sp| sp.get("properties")) + .and_then(|p| p.as_mapping()) + .cloned() // Clone to avoid borrowing issues + .ok_or_else(|| "Could not find schema for SearchParameters".to_string())?; + + // Navigate to the operation's parameters list + let params = doc + .get_mut("paths") + .and_then(|p| p.get_mut("/collections/{collectionName}/documents/search")) + .and_then(|p| p.get_mut("get")) + .and_then(|op| op.get_mut("parameters")) + .and_then(|params| params.as_sequence_mut()) + .ok_or_else(|| { + "Could not find parameters for /collections/{collectionName}/documents/search" + .to_string() + })?; + + // Find and remove the old parameter object. + let param_index = params + .iter() + .position(|p| p.get("name").and_then(|n| n.as_str()) == Some("searchParameters")) + .ok_or_else(|| "searchParameters object not found".to_string())?; + params.remove(param_index); + + // Add the new individual parameters. + for (key, value) in search_params_props { + let mut new_param = Mapping::new(); + new_param.insert("name".into(), key.clone()); + new_param.insert("in".into(), "query".into()); + new_param.insert("schema".into(), value.clone()); + params.push(new_param.into()); + } + + Ok(()) +} + +/// Special handler for unwrapping multi-search parameters from `components/schemas`. +fn unwrap_multi_search_parameters(doc: &mut Mapping) -> Result<(), String> { + println!("- Unwrapping multiSearchParameters..."); + // Get the definition of MultiSearchParameters from components + let search_params_props: Mapping = doc + .get("components") + .and_then(|c| c.get("schemas")) + .and_then(|s| s.get("MultiSearchParameters")) + .and_then(|sp| sp.get("properties")) + .and_then(|p| p.as_mapping()) + .cloned() + .ok_or_else(|| "Could not find schema for MultiSearchParameters".to_string())?; + + // Navigate to the operation's parameters list + let params = doc + .get_mut("paths") + .and_then(|p| p.get_mut("/multi_search")) + .and_then(|p| p.get_mut("post")) + .and_then(|op| op.get_mut("parameters")) + .and_then(|params| params.as_sequence_mut()) + .ok_or_else(|| "Could not find parameters for /multi_search".to_string())?; + + // Find and remove the old parameter object. + let param_index = params + .iter() + .position(|p| p.get("name").and_then(|n| n.as_str()) == Some("multiSearchParameters")) + .ok_or_else(|| "multiSearchParameters object not found".to_string())?; + params.remove(param_index); + + // Add the new individual parameters. + for (key, value) in search_params_props { + let mut new_param = Mapping::new(); + new_param.insert("name".into(), key.clone()); + new_param.insert("in".into(), "query".into()); + new_param.insert("schema".into(), value.clone()); + params.push(new_param.into()); + } + + Ok(()) +} From 7fa1e58dd0f8b4345fc9ac374e10a94017654519 Mon Sep 17 00:00:00 2001 From: Hayden Hung Hoang Date: Wed, 27 Aug 2025 23:29:05 +0700 Subject: [PATCH 2/5] fix `Cargo.toml` package metadata --- xtask/Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index a394748..994b250 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "xtask" -version = "0.1.0" -edition = "2021" +publish = false +version = "0.0.0" +edition.workspace = true [dependencies] reqwest = { version = "0.11", features = ["blocking"] } # "blocking" is simpler for scripts From c1739daec879e504f09c7ff838a3d948270f3912 Mon Sep 17 00:00:00 2001 From: Hayden Hung Hoang Date: Wed, 27 Aug 2025 23:30:25 +0700 Subject: [PATCH 3/5] lint fix --- typesense_derive/src/lib.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/typesense_derive/src/lib.rs b/typesense_derive/src/lib.rs index 1f92193..5855394 100644 --- a/typesense_derive/src/lib.rs +++ b/typesense_derive/src/lib.rs @@ -43,8 +43,8 @@ fn impl_typesense_collection(item: ItemStruct) -> syn::Result { } = extract_attrs(attrs)?; let collection_name = collection_name.unwrap_or_else(|| ident.to_string().to_lowercase()); - if let Some(ref sorting_field) = default_sorting_field { - if !fields.iter().any(|field| + if let Some(ref sorting_field) = default_sorting_field + && !fields.iter().any(|field| // At this point we are sure that this field is a named field. field.ident.as_ref().unwrap() == sorting_field) { @@ -55,7 +55,6 @@ fn impl_typesense_collection(item: ItemStruct) -> syn::Result { ), )); } - } let typesense_fields = fields .iter() @@ -98,19 +97,16 @@ fn impl_typesense_collection(item: ItemStruct) -> syn::Result { // Get the inner type for a given wrapper fn ty_inner_type<'a>(ty: &'a syn::Type, wrapper: &'static str) -> Option<&'a syn::Type> { - if let syn::Type::Path(p) = ty { - if p.path.segments.len() == 1 && p.path.segments[0].ident == wrapper { - if let syn::PathArguments::AngleBracketed(ref inner_ty) = p.path.segments[0].arguments { - if inner_ty.args.len() == 1 { + if let syn::Type::Path(p) = ty + && p.path.segments.len() == 1 && p.path.segments[0].ident == wrapper + && let syn::PathArguments::AngleBracketed(ref inner_ty) = p.path.segments[0].arguments + && inner_ty.args.len() == 1 { // len is 1 so this should not fail let inner_ty = inner_ty.args.first().unwrap(); if let syn::GenericArgument::Type(t) = inner_ty { return Some(t); } } - } - } - } None } @@ -231,8 +227,8 @@ fn to_typesense_field_type(field: &Field) -> syn::Result syn::Result>>()?; From ea93b0ab7fbd0a3a22f17f3541cd6de2759f297f Mon Sep 17 00:00:00 2001 From: Hayden Hung Hoang Date: Thu, 28 Aug 2025 00:29:16 +0700 Subject: [PATCH 4/5] fix: avoid compiling xtask for wasm --- xtask/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c95fe62..0fd68d2 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -37,6 +37,10 @@ enum Task { CodeGen, } +#[cfg(target_family = "wasm")] +fn main() {} + +#[cfg(not(target_family = "wasm"))] fn main() -> Result<()> { let cli = Cli::parse(); @@ -50,6 +54,7 @@ fn main() -> Result<()> { Ok(()) } +#[cfg(not(target_family = "wasm"))] fn task_fetch_api_spec() -> Result<()> { println!("▶️ Running codegen task..."); From 741b11aa05b41e015e49557426c8e61ebff7262c Mon Sep 17 00:00:00 2001 From: Hayden Hung Hoang Date: Thu, 28 Aug 2025 00:30:44 +0700 Subject: [PATCH 5/5] fix fmt and typesense/lib.rs docs test --- typesense/src/lib.rs | 2 +- typesense_derive/src/lib.rs | 84 ++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/typesense/src/lib.rs b/typesense/src/lib.rs index 286c959..0e6c763 100644 --- a/typesense/src/lib.rs +++ b/typesense/src/lib.rs @@ -8,7 +8,7 @@ //! # Examples //! //! ``` -//! #[cfg(any(feature = "tokio_test", target_arch = "wasm32"))] +//! #[cfg(not(target_family = "wasm"))] //! { //! use serde::{Deserialize, Serialize}; //! use typesense::document::Document; diff --git a/typesense_derive/src/lib.rs b/typesense_derive/src/lib.rs index 5855394..4b1d70f 100644 --- a/typesense_derive/src/lib.rs +++ b/typesense_derive/src/lib.rs @@ -47,14 +47,14 @@ fn impl_typesense_collection(item: ItemStruct) -> syn::Result { && !fields.iter().any(|field| // At this point we are sure that this field is a named field. field.ident.as_ref().unwrap() == sorting_field) - { - return Err(syn::Error::new_spanned( - item_ts, - format!( - "defined default_sorting_field = \"{sorting_field}\" does not match with any field." - ), - )); - } + { + return Err(syn::Error::new_spanned( + item_ts, + format!( + "defined default_sorting_field = \"{sorting_field}\" does not match with any field." + ), + )); + } let typesense_fields = fields .iter() @@ -98,15 +98,17 @@ fn impl_typesense_collection(item: ItemStruct) -> syn::Result { // Get the inner type for a given wrapper fn ty_inner_type<'a>(ty: &'a syn::Type, wrapper: &'static str) -> Option<&'a syn::Type> { if let syn::Type::Path(p) = ty - && p.path.segments.len() == 1 && p.path.segments[0].ident == wrapper - && let syn::PathArguments::AngleBracketed(ref inner_ty) = p.path.segments[0].arguments - && inner_ty.args.len() == 1 { - // len is 1 so this should not fail - let inner_ty = inner_ty.args.first().unwrap(); - if let syn::GenericArgument::Type(t) = inner_ty { - return Some(t); - } - } + && p.path.segments.len() == 1 + && p.path.segments[0].ident == wrapper + && let syn::PathArguments::AngleBracketed(ref inner_ty) = p.path.segments[0].arguments + && inner_ty.args.len() == 1 + { + // len is 1 so this should not fail + let inner_ty = inner_ty.args.first().unwrap(); + if let syn::GenericArgument::Type(t) = inner_ty { + return Some(t); + } + } None } @@ -227,42 +229,40 @@ fn to_typesense_field_type(field: &Field) -> syn::Result { - if i != "facet" { - return Some(Err(syn::Error::new_spanned( - i, - format!("Unexpected token {i}. Did you mean `facet`?"), - ))); - } - } - Some(ref tt) => { - return Some(Err(syn::Error::new_spanned( - tt, - format!("Unexpected token {tt}. Did you mean `facet`?"), - ))); - } - None => { + { + let mut tokens = g.stream().into_iter(); + match tokens.next() { + Some(proc_macro2::TokenTree::Ident(ref i)) => { + if i != "facet" { return Some(Err(syn::Error::new_spanned( - attr, - "expected `facet`", + i, + format!("Unexpected token {i}. Did you mean `facet`?"), ))); } } - - if let Some(ref tt) = tokens.next() { + Some(ref tt) => { return Some(Err(syn::Error::new_spanned( tt, - "Unexpected token. Expected )", + format!("Unexpected token {tt}. Did you mean `facet`?"), ))); } - return Some(Ok(())); + None => { + return Some(Err(syn::Error::new_spanned(attr, "expected `facet`"))); + } } + + if let Some(ref tt) = tokens.next() { + return Some(Err(syn::Error::new_spanned( + tt, + "Unexpected token. Expected )", + ))); + } + return Some(Ok(())); + } None }) .collect::>>()?;