feat: add json schema generation for CasConfig via schemars

Author: Pegasust

.gitignore

@@ -23,3 +23,4 @@ nixos.bazelrc
 rust-project.json
 darwin.bazelrc
 nativelink.bazelrc
+nativelink_config.schema.json

Cargo.lock

@@ -9,7 +9,17 @@ humantime = "2.1.0"
 serde = { version = "1.0.218", default-features = false, features = ["derive"] }
 serde_json5 = "0.2.1"
 shellexpand = { version = "3.1.0", default-features = false, features = ["base-0"] }
+schemars = { version = "0.8.22", features = ["derive"], optional = true }
+serde_json = { version = "1.0.139", default-features = false, optional = true }
 
 [dev-dependencies]
 pretty_assertions = { version = "1.4.1", features = ["std"] }
 serde_json = { version = "1.0.139", default-features = false }
+
+[features]
+dev-schema = ["schemars", "serde_json"]
+
+[[bin]]
+name = "build-schema"
+path = "src/bin/build_schema.rs"
+required-features=["dev-schema"]

nativelink-config/Cargo.toml

@@ -0,0 +1,24 @@
+//! ```sh
+//! cargo run --bin build-schema --features dev-schema --package nativelink-config
+//! ```
+
+#[cfg(feature = "dev-schema")]
+fn main() {
+    use std::fs::File;
+
+    use nativelink_config::cas_server::CasConfig;
+    use schemars::schema_for;
+    use serde_json::to_writer_pretty;
+    const FILE: &str = "nativelink_config.schema.json";
+
+    let schema = schema_for!(CasConfig);
+    to_writer_pretty(File::create(FILE).expect("to create file"), &schema)
+        .expect("to export schema");
+
+    println!("Wrote schema to {FILE}");
+}
+
+#[cfg(not(feature = "dev-schema"))]
+fn main() {
+    eprintln!("Enable with --features dev-schema");
+}

nativelink-config/src/bin/build_schema.rs

@@ -14,6 +14,8 @@
 
 use std::collections::HashMap;
 
+#[cfg(feature = "dev-schema")]
+use schemars::JsonSchema;
 use serde::Deserialize;
 
 use crate::serde_utils::{
@@ -34,6 +36,7 @@ pub type InstanceName = String;
 
 #[allow(non_camel_case_types)]
 #[derive(Deserialize, Debug, Default, Clone, Copy)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum HttpCompressionAlgorithm {
     /// No compression.
     #[default]
@@ -54,6 +57,7 @@ pub enum HttpCompressionAlgorithm {
 /// and cloud-clients to use another.
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct HttpCompressionConfig {
     /// The compression algorithm that the server will use when sending
     /// responses to clients. Enabling this will likely save a lot of
@@ -77,6 +81,7 @@ pub struct HttpCompressionConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct AcStoreConfig {
     /// The store name referenced in the `stores` map in the main config.
     /// This store name referenced here may be reused multiple times.
@@ -91,6 +96,7 @@ pub struct AcStoreConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct CasStoreConfig {
     /// The store name referenced in the `stores` map in the main config.
     /// This store name referenced here may be reused multiple times.
@@ -100,6 +106,7 @@ pub struct CasStoreConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct CapabilitiesRemoteExecutionConfig {
     /// Scheduler used to configure the capabilities of remote execution.
     #[serde(deserialize_with = "convert_string_with_shellexpand")]
@@ -108,6 +115,7 @@ pub struct CapabilitiesRemoteExecutionConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct CapabilitiesConfig {
     /// Configuration for remote execution capabilities.
     /// If not set the capabilities service will inform the client that remote
@@ -117,6 +125,7 @@ pub struct CapabilitiesConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct ExecutionConfig {
     /// The store name referenced in the `stores` map in the main config.
     /// This store name referenced here may be reused multiple times.
@@ -131,6 +140,7 @@ pub struct ExecutionConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct ByteStreamConfig {
     /// Name of the store in the "stores" configuration.
     pub cas_stores: HashMap<InstanceName, StoreRefName>,
@@ -161,6 +171,7 @@ pub struct ByteStreamConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct WorkerApiConfig {
     /// The scheduler name referenced in the `schedulers` map in the main config.
     #[serde(deserialize_with = "convert_string_with_shellexpand")]
@@ -169,6 +180,7 @@ pub struct WorkerApiConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct PrometheusConfig {
     /// Path to register prometheus metrics. If path is "/metrics", and your
     /// domain is "example.com", you can reach the endpoint with:
@@ -181,6 +193,7 @@ pub struct PrometheusConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct AdminConfig {
     /// Path to register the admin API. If path is "/admin", and your
     /// domain is "example.com", you can reach the endpoint with:
@@ -193,6 +206,7 @@ pub struct AdminConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct HealthConfig {
     /// Path to register the health status check. If path is "/status", and your
     /// domain is "example.com", you can reach the endpoint with:
@@ -204,6 +218,7 @@ pub struct HealthConfig {
 }
 
 #[derive(Deserialize, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct BepConfig {
     /// The store to publish build events to.
     /// The store name referenced in the `stores` map in the main config.
@@ -212,6 +227,7 @@ pub struct BepConfig {
 }
 
 #[derive(Deserialize, Clone, Debug, Default)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct IdentityHeaderSpec {
     /// The name of the header to look for the identity in.
     /// Default: "x-identity"
@@ -224,6 +240,7 @@ pub struct IdentityHeaderSpec {
 }
 
 #[derive(Deserialize, Clone, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct OriginEventsPublisherSpec {
     /// The store to publish nativelink events to.
     /// The store name referenced in the `stores` map in the main config.
@@ -232,6 +249,7 @@ pub struct OriginEventsPublisherSpec {
 }
 
 #[derive(Deserialize, Clone, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct OriginEventsSpec {
     /// The publisher configuration for origin events.
     pub publisher: OriginEventsPublisherSpec,
@@ -247,6 +265,7 @@ pub struct OriginEventsSpec {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct ServicesConfig {
     /// The Content Addressable Storage (CAS) backend config.
     /// The key is the `instance_name` used in the protocol and the
@@ -301,6 +320,7 @@ pub struct ServicesConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct TlsConfig {
     /// Path to the certificate file.
     #[serde(deserialize_with = "convert_string_with_shellexpand")]
@@ -329,6 +349,7 @@ pub struct TlsConfig {
 /// specified.
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct HttpServerConfig {
     /// Interval to send keep-alive pings via HTTP2.
     /// Note: This is in seconds.
@@ -396,13 +417,15 @@ pub struct HttpServerConfig {
 
 #[allow(non_camel_case_types)]
 #[derive(Deserialize, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum ListenerConfig {
     /// Listener for HTTP/HTTPS/HTTP2 sockets.
     http(HttpListener),
 }
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct HttpListener {
     /// Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen
     /// to all IPs.
@@ -427,6 +450,7 @@ pub struct HttpListener {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct ServerConfig {
     /// Name of the server. This is used to help identify the service
     /// for telemetry and logs.
@@ -449,6 +473,7 @@ pub struct ServerConfig {
 
 #[allow(non_camel_case_types)]
 #[derive(Deserialize, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum WorkerProperty {
     /// List of static values.
     /// Note: Generally there should only ever be 1 value, but if the platform
@@ -464,6 +489,7 @@ pub enum WorkerProperty {
 /// Generic config for an endpoint and associated configs.
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct EndpointConfig {
     /// URI of the endpoint.
     #[serde(deserialize_with = "convert_string_with_shellexpand")]
@@ -479,6 +505,7 @@ pub struct EndpointConfig {
 
 #[allow(non_camel_case_types)]
 #[derive(Copy, Clone, Deserialize, Debug, Default)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum UploadCacheResultsStrategy {
     /// Only upload action results with an exit code of 0.
     #[default]
@@ -496,6 +523,7 @@ pub enum UploadCacheResultsStrategy {
 
 #[allow(non_camel_case_types)]
 #[derive(Clone, Deserialize, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum EnvironmentSource {
     /// The name of the platform property in the action to get the value from.
     property(String),
@@ -543,6 +571,7 @@ pub enum EnvironmentSource {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct UploadActionResultConfig {
     /// Underlying AC store that the worker will use to publish execution results
     /// into. Objects placed in this store should be reachable from the
@@ -603,6 +632,7 @@ pub struct UploadActionResultConfig {
 
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct LocalWorkerConfig {
     /// Name of the worker. This is give a more friendly name to a worker for logging
     /// and metric publishing. This is also the prefix of the worker id
@@ -695,13 +725,15 @@ pub struct LocalWorkerConfig {
 
 #[allow(non_camel_case_types)]
 #[derive(Deserialize, Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub enum WorkerConfig {
     /// A worker type that executes jobs locally on this machine.
     local(LocalWorkerConfig),
 }
 
 #[derive(Deserialize, Debug, Clone, Copy)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct GlobalConfig {
     /// Maximum number of open files that can be opened at one time.
     /// This value is not strictly enforced, it is a best effort. Some internal libraries
@@ -747,6 +779,7 @@ pub struct GlobalConfig {
 
 #[derive(Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct CasConfig {
     /// List of stores available to use in this config.
     /// The keys can be used in other configs when needing to reference a store.

nativelink-config/src/cas_server.rs

@@ -22,10 +22,12 @@ use std::collections::HashMap;
 use std::fmt;
 use std::marker::PhantomData;
 
+#[cfg(feature = "dev-schema")]
+use schemars::JsonSchema;
 use serde::de::{MapAccess, SeqAccess, Visitor};
 use serde::{Deserialize, Deserializer};
-
 #[derive(Debug, Clone, Deserialize)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct NamedConfig<Spec> {
     pub name: String,
     #[serde(flatten)]
@@ -41,6 +43,7 @@ pub type StoreConfigs = NamedConfigs<crate::stores::StoreSpec>;
 pub type SchedulerConfigs = NamedConfigs<crate::schedulers::SchedulerSpec>;
 
 #[derive(Debug)]
+#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
 pub struct NamedConfigs<T>(pub Vec<NamedConfig<T>>);
 
 impl<T> NamedConfigs<T> {