Add ability to configure carp via env var (#139)

Author: gostkin

Cargo.lock

@@ -14,11 +14,7 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: postgres # name of postgres container
-    port: 5432 # internal postgres port
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@postgres:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:

deployment/config/indexer/cardano_node_docker.yml

@@ -7,11 +7,7 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: postgres # name of postgres container
-    port: 5432 # internal postgres port
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@postgres:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:

deployment/config/indexer/oura_docker.yml

@@ -43,6 +43,7 @@ NETWORK=mainnet
 
 # can be cardano_node_docker.yml or oura_docker.yml or 
 # any custom filename located in deployment/config/indexer folder 
+# alternative to CARP_CONFIG env variable
 CONFIG_FILE=cardano_node_docker.yml
 
 CARP_VERSION=2.2.1
@@ -65,7 +66,7 @@ DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${POSTGRES_HOST}:${POSTGRES_PO
 PGURI=$DATABASE_URL
 ```
 
-Please set `NETWORK`, `CONFIG_FILE` and postgres variables carefully. `CONFIG_FILE` should be located in `deployment/config/indexer` folder.
+Please set `NETWORK`, `CONFIG_FILE` and postgres variables carefully. `CONFIG_FILE` should be located in `deployment/config/indexer` folder. You can use `CARP_CONFIG` env variable instead of `CONFIG_FILE` if you want to use just env variables for configuration.
 
 You can also add non-mandatory variables that are useful for backuper:
 
@@ -83,7 +84,7 @@ AWS_SECRET_ACCESS_KEY=<Secret for given account>
 
 ## Carp configuration
 
-We mentioned config files above, they should be stored in `deployment/config/indexer` folder, and you should set `CONFIG_FILE` env variable with the config name.
+We mentioned config files above, they should be stored in `deployment/config/indexer` folder, and you should set `CONFIG_FILE` env variable with the config name. Alternatively you can set `CARP_CONFIG` env variable and not use the configuration file
 
 The difference from non-docker configuration is in the networking mainly.
 

docs/docs/indexer/docker.md

@@ -8,7 +8,9 @@ To run carp you need to configure carp itself, set up / configure cardano-node (
 
 ## Configuration & concepts
 
-Carp itself uses special file for configuration, the examples are stored [there](https://github.com/dcSpark/carp/blob/main/indexer/configs/). Let's dive into configuration a little further:
+Carp itself uses special file for configuration, the examples are stored [there](https://github.com/dcSpark/carp/blob/main/indexer/configs/). Besides, carp's config can be set up through `CARP_CONFIG` env variable in json format. Let's dive into configuration a little further:
+
+File format:
 ```yaml
 source:
   type: oura
@@ -19,16 +21,17 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: localhost
-    port: 5432
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@localhost:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:
 ```
 
+Json format:
+```json
+{"source":{"type":"oura","socket":"relays-new.cardano-mainnet.iohk.io:3001","bearer":"Tcp"},"sink":{"type":"cardano","db":{"type": "postgres","database_url":"postgresql://carp:1234@localhost:5432/carp_mainnet"},"network":"mainnet"},"start_block":null}
+```
+
 As you might see there are several key sections: source and sink. For sink there's only one option at the moment: `cardano` sink. For source there are two options with different configurations: `oura` and `cardano_net`.
 
 ### Sink configuration

docs/docs/indexer/run.md

@@ -24,10 +24,11 @@ async-trait = { version = "0.1.64" }
 base64 = { version = "0.21.0" }
 cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" }
 clap = { version = "3.1", features = ["derive"] }
-ctrlc = {version = "3.2.4", features = ["termination"] }
+ctrlc = { version = "3.2.4", features = ["termination"] }
+dotenv = { version = "0.15.0" }
 hex = { version = "0.4.3" }
 oura = { version = "1.8.0" }
-pallas = {version = "0.17.0" }
+pallas = { version = "0.17.0" }
 serde = { version = "1.0.152", features = ["derive", "rc"] }
 serde_json = { version = "1.0.92" }
 serde_yaml = { version = "0.9.17" }

indexer/Cargo.toml

@@ -12,11 +12,7 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: localhost
-    port: 5432
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@localhost:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:

indexer/configs/cardano_node.yml

@@ -7,11 +7,7 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: localhost
-    port: 5432
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@localhost:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:

indexer/configs/default.yml

@@ -7,11 +7,7 @@ sink:
   type: cardano
   db:
     type: postgres
-    host: localhost
-    port: 5432
-    user: carp
-    password: "1234"
-    db: carp_mainnet
+    database_url: postgresql://carp:1234@localhost:5432/carp_mainnet
   network: mainnet # preview / preprod / testnet
 
 start_block:

indexer/configs/oura.yml

@@ -33,20 +33,14 @@ pub struct Cli {
 
     /// path to config file
     #[clap(long, value_parser)]
-    config_path: PathBuf,
+    config_path: Option<PathBuf>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(tag = "type", rename_all = "snake_case")]
 #[serde(deny_unknown_fields)]
 pub enum DbConfig {
-    Postgres {
-        host: String,
-        port: u64,
-        user: String,
-        password: String,
-        db: String,
-    },
+    Postgres { database_url: String },
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -114,19 +108,31 @@ async fn main() -> anyhow::Result<()> {
     tracing::info!("Execution plan {}", plan);
     let exec_plan = Arc::new(ExecutionPlan::load_from_file(&plan)?);
 
-    tracing::info!("Config file {:?}", config_path);
-    let file = File::open(&config_path).with_context(|| {
-        format!(
-            "Cannot open config file {path}",
-            path = config_path.display()
-        )
-    })?;
-    let config: Config = serde_yaml::from_reader(file).with_context(|| {
-        format!(
-            "Cannot parse config file {path}",
-            path = config_path.display()
-        )
-    })?;
+    let config = if let Some(config_path) = config_path {
+        tracing::info!("Config file {:?}", config_path);
+        let file = File::open(&config_path).with_context(|| {
+            format!(
+                "Cannot open config file {path}",
+                path = config_path.display()
+            )
+        })?;
+        let config: Config = serde_yaml::from_reader(file).with_context(|| {
+            format!(
+                "Cannot parse config file {path}",
+                path = config_path.display()
+            )
+        })?;
+        config
+    } else {
+        dotenv::dotenv().ok();
+
+        let carp_config = std::env::var("CARP_CONFIG")
+            .expect("env CARP_CONFIG not found and --config-path not specified");
+        let config: Config = serde_json::from_str(&carp_config)
+            .with_context(|| format!("Cannot parse config string {carp_config}"))?;
+
+        config
+    };
 
     let (network, mut sink) = match config.sink {
         SinkConfig::Cardano { ref network, .. } => (