diff --git a/holesky/.env.example b/holesky/.env.example index 55f4a2b..fb42f94 100644 --- a/holesky/.env.example +++ b/holesky/.env.example @@ -1,4 +1,4 @@ -MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0-rc.5 +MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0 NETWORK_NAME=eigenda-network MAIN_SERVICE_NAME=eigenda-native-node @@ -35,7 +35,6 @@ NODE_INTERNAL_RETRIEVAL_PORT=${NODE_RETRIEVAL_PORT} NODE_INTERNAL_V2_DISPERSAL_PORT=${NODE_V2_DISPERSAL_PORT} NODE_INTERNAL_V2_RETRIEVAL_PORT=${NODE_V2_RETRIEVAL_PORT} - # Reachability Check NODE_REACHABILITY_POLL_INTERVAL=60 NODE_DATAAPI_URL=https://dataapi-holesky.eigenda.xyz diff --git a/holesky/README.md b/holesky/README.md index 5c26dc0..14a834e 100644 --- a/holesky/README.md +++ b/holesky/README.md @@ -20,7 +20,7 @@ NODE_INTERNAL_V2_RETRIEVAL_PORT=${NODE_V2_RETRIEVAL_PORT} ### 2. Update `MAIN_SERVICE_IMAGE` ``` -MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0-rc.5 +MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0 ``` ### 3. Update socket registration @@ -81,3 +81,34 @@ This port is intended for Nginx reverse proxy use. It is not required if the ope ### `EIGENDA_INTERNAL_V2_RETRIEVAL_PORT` This port is intended for Nginx reverse proxy use. It is not required if the operator is not using a reverse proxy. + +## Advanced - Multi drive support for V2 LittDB + +LittDB is capable of partitioning the chunks DB across multiple drives. See https://github.com/Layr-Labs/eigenda/blob/master/node/database-paths.md for more details. + +### Example .env that defines 2 littDB partitions +``` +NODE_LITT_DB_STORAGE_HOST_PATH_1=${NODE_DB_PATH_HOST}/chunk_v2_litt_1 +NODE_LITT_DB_STORAGE_HOST_PATH_2=${NODE_DB_PATH_HOST}/chunk_v2_litt_2 +NODE_LITT_DB_STORAGE_PATHS=/data/operator/db/chunk_v2_litt_1,/data/operator/db/chunk_v2_litt_2 +``` +The `NODE_LITT_DB_STORAGE_HOST_PATH_X` should map to a separately mounted drive folder for each partition. +The `NODE_LITT_DB_STORAGE_PATHS` needs to map to the docker volume mounts defined in `docker-compose.yaml` +#### Example docker-compose volume mounts +``` + volumes: + - "${NODE_ECDSA_KEY_FILE_HOST}:/app/operator_keys/ecdsa_key.json:readonly" + - "${NODE_BLS_KEY_FILE_HOST}:/app/operator_keys/bls_key.json:readonly" + - "${NODE_G1_PATH_HOST}:/app/g1.point:readonly" + - "${NODE_G2_PATH_HOST}:/app/g2.point.powerOf2:readonly" + - "${NODE_CACHE_PATH_HOST}:/app/cache:rw" + - "${NODE_LOG_PATH_HOST}:/app/logs:rw" + - "${NODE_DB_PATH_HOST}:/data/operator/db:rw" + - "${NODE_LITT_DB_STORAGE_HOST_PATH_1}:/data/operator/db/chunk_v2_litt_1:rw" + - "${NODE_LITT_DB_STORAGE_HOST_PATH_2}:/data/operator/db/chunk_v2_litt_2:rw" +``` +#### Example node output after loading both partitions +``` +eigenda-native-node | Jun 2 18:11:47.430 INF node/validator_store.go:108 Using littDB at paths /data/operator/db/chunk_v2_litt_1, /data/operator/db/chunk_v2_litt_2 +eigenda-native-node | Jun 2 18:11:47.430 INF littbuilder/db_impl.go:118 LittDB started, current data size: 0 +eigenda-native-node | Jun 2 18:11:47.430 INF littbuilder/db_impl.go:169 creating table chunks \ No newline at end of file diff --git a/holesky/docker-compose.yml b/holesky/docker-compose.yml index a6d08aa..92020af 100644 --- a/holesky/docker-compose.yml +++ b/holesky/docker-compose.yml @@ -38,6 +38,7 @@ services: - "${NODE_CACHE_PATH_HOST}:/app/cache:rw" - "${NODE_LOG_PATH_HOST}:/app/logs:rw" - "${NODE_DB_PATH_HOST}:/data/operator/db:rw" +# TODO: Uncomment and update if using remote BLS signer https://github.com/Layr-Labs/cerberus # - "${NODE_BLS_SIGNER_CERT_FILE_HOST}:/app/cerberus/cerberus.crt:readonly" restart: unless-stopped networks: diff --git a/holesky/run.sh b/holesky/run.sh index 5b1554c..4fcb542 100755 --- a/holesky/run.sh +++ b/holesky/run.sh @@ -3,7 +3,7 @@ . ./.env -node_plugin_image="ghcr.io/layr-labs/eigenda/opr-nodeplugin:0.9.0-rc.5" +node_plugin_image="ghcr.io/layr-labs/eigenda/opr-nodeplugin:0.9.0" # Check if V2 ports are defined if [ -z "$NODE_V2_DISPERSAL_PORT" ]; then @@ -83,8 +83,7 @@ listQuorums() { } updateSocket() { - # we have to pass a dummy quorum-id-list as it is required by the plugin - echo "You are about to update your socket to: $socket" + echo "You are about to update your socket registration to: $socket" echo "Confirm? [Y/n] " read -r answer if [ "$answer" = "n" ] || [ "$answer" = "N" ]; then @@ -92,6 +91,7 @@ updateSocket() { exit 1 fi + # we have to pass a dummy quorum-id-list as it is required by the plugin docker run --env-file .env \ --rm \ --volume "${NODE_ECDSA_KEY_FILE_HOST}":/app/operator_keys/ecdsa_key.json \ diff --git a/mainnet/.env.example b/mainnet/.env.example index e64422b..7b5c3bc 100644 --- a/mainnet/.env.example +++ b/mainnet/.env.example @@ -1,4 +1,4 @@ -MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.8.6 +MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0 NETWORK_NAME=eigenda-network MAIN_SERVICE_NAME=eigenda-native-node @@ -102,6 +102,13 @@ NODE_CACHE_PATH_HOST=${USER_HOME}/eigenda-operator-setup/resources/cache # TODO: Operators need to update this to their own keys NODE_ECDSA_KEY_FILE_HOST=${EIGENLAYER_HOME}/operator_keys/opr.ecdsa.key.json NODE_BLS_KEY_FILE_HOST=${EIGENLAYER_HOME}/operator_keys/opr.bls.key.json +# TODO: Uncomment and update if using remote BLS signer https://github.com/Layr-Labs/cerberus +#NODE_BLS_SIGNER_CERT_FILE=/app/cerberus/cerberus.crt # DO NOT CHANGE THIS +#NODE_BLS_REMOTE_SIGNER_ENABLED=true +#NODE_BLS_REMOTE_SIGNER_URL= +#NODE_BLS_SIGNER_API_KEY= +#NODE_BLS_PUBLIC_KEY_HEX= +#NODE_BLS_SIGNER_CERT_FILE_HOST= # TODO: The ip provider service used to obtain a node's public IP [seeip (default), ipify) NODE_PUBLIC_IP_PROVIDER=seeip diff --git a/mainnet/README.md b/mainnet/README.md index 96b98df..1d771e2 100644 --- a/mainnet/README.md +++ b/mainnet/README.md @@ -1,3 +1,114 @@ ## Mainnet -Head over to our [EigenDA operator guides](https://docs.eigenlayer.xyz/eigenda/operator-guides/overview) for installation instructions and more details. \ No newline at end of file +Head over to our [EigenDA operator guides](https://docs.eigenlayer.xyz/eigenda/operator-guides/overview) for installation instructions and more details. + +## Blazar (EigenDA V2) Migration +Operators running node version `<=v0.8.6` will need to define new v2 specific environment variables, expose 2 new ports, and update their socket registration as part of the migration to v2. + +## Migration Steps +### 1. Update `.env` with v2 specific environment variables +``` +NODE_V2_RUNTIME_MODE=v1-and-v2 + +NODE_V2_DISPERSAL_PORT=32006 +NODE_V2_RETRIEVAL_PORT=32007 + +# Internal ports for Nginx reverse proxy +NODE_INTERNAL_V2_DISPERSAL_PORT=${NODE_V2_DISPERSAL_PORT} +NODE_INTERNAL_V2_RETRIEVAL_PORT=${NODE_V2_RETRIEVAL_PORT} +``` + +### 2. Update `MAIN_SERVICE_IMAGE` +``` +MAIN_SERVICE_IMAGE=ghcr.io/layr-labs/eigenda/opr-node:0.9.0 +``` + +### 3. Update socket registration +EigenDA v2 adds new ports to the socket registration. Socket registration update is required to receive v2 traffic. + +Ensure that you are using the latest version of the [eigenda-operator-setup](https://github.com/Layr-Labs/eigenda-operator-setup/releases) before updating the socket. +``` +(eigenda-operator-setup) > ./run.sh update-socket +You are about to update your socket to: 23.93.87.155:32005;32004;32006;32007 +Confirm? [Y/n] +``` + +### 4. Restart the node and monitor for reachability checks +The node will check reachability of v1 & v2 sockets. If reachability checks are failing, check that the new ports are open and accessible. +``` +Feb 20 19:47:07.861 INF node/node.go:743 Reachability check v1 - dispersal socket ONLINE component=Node status="node.Dispersal is available" socket=operator.eigenda.xyz:32001 +Feb 20 19:47:07.861 INF node/node.go:750 Reachability check v1 - retrieval socket ONLINE component=Node status="node.Retrieval is available" socket=operator.eigenda.xyz:32002 +Feb 20 19:47:07.867 INF node/node.go:743 Reachability check v2 - dispersal socket ONLINE component=Node status="validator.Dispersal is available" socket=operator.eigenda.xyz:32003 +Feb 20 19:47:07.867 INF node/node.go:750 Reachability check v2 - retrieval socket ONLINE component=Node status="validator.Retrieval is available" socket=operator.eigenda.xyz:32005 +``` + +### 5. Confirm v2 StoreChunks requests are being served +``` +Feb 20 19:50:36.741 INF grpc/server_v2.go:140 new StoreChunks request batchHeaderHash=873ac1c7faeec0f1e5c886142d0b364a94b3e906f1b4b4f1b0466a5f79cecefb numBlobs=14 referenceBlockNumber=3393054 +Feb 20 19:50:41.765 INF grpc/server_v2.go:140 new StoreChunks request batchHeaderHash=76873d64609d50aaf90e1c435c9278c588f1a174a4c0b4a721438a7d44bb2f1e numBlobs=18 referenceBlockNumber=3393054 +Feb 20 19:50:46.760 INF grpc/server_v2.go:140 new StoreChunks request batchHeaderHash=8182f31c9b58e04f0a09dfbf1634a73e47a660b441f65c7a35ef9e7afd064493 numBlobs=16 referenceBlockNumber=3393054 + +``` + +## V2 Remote BLS Signer API Migration +__Breaking Change:__ With the release of EigenDA Blazar (v2), the the node software has been upgraded to use the latest [cerberus](https://github.com/Layr-Labs/cerberus) remote BLS signer release. + +This change requires the operator to define the `NODE_BLS_SIGNER_API_KEY` environment variable. + +Follow the steps from the [cerberus setup guide](https://github.com/Layr-Labs/cerberus?tab=readme-ov-file#remote-signer-implementation-of-cerberus-api) to create an API key. + + +## V2 Environment Variable Reference + +### `EIGENDA_RUNTIME_MODE` +This environment variable will be used to determine the runtime mode of the EigenDA node. + +- `v1-and-v2`: The node will serve both v1 and v2 traffic (default) +- `v2-only`: The node will serve v2 traffic only +- `v1-only`: The node will serve v1 traffic only + +The `v1-only` & `v2-only` modes are intended for isolating traffic to separate validator instances - where 1 instance serves v1 traffic and a second instance serves v2 traffic. + +### `EIGENDA_V2_DISPERSAL_PORT` +Operators must publically expose this port. This port will be used to listen for dispersal requests from the EigenDA v2 API. IP whitelisting is no longer required with v2. + +### `EIGENDA_V2_RETRIEVAL_PORT` +Operators must publically expose this port. This port will be used to listen for retrieval requests from the EigenDA v2 API. + +### `EIGENDA_INTERNAL_V2_DISPERSAL_PORT` +This port is intended for Nginx reverse proxy use. It is not required if the operator is not using a reverse proxy. + +### `EIGENDA_INTERNAL_V2_RETRIEVAL_PORT` +This port is intended for Nginx reverse proxy use. It is not required if the operator is not using a reverse proxy. + + +## Advanced - Multi drive support for V2 LittDB + +LittDB is capable of partitioning the chunks DB across multiple drives. See https://github.com/Layr-Labs/eigenda/blob/master/node/database-paths.md for more details. + +### Example .env that defines 2 littDB partitions +``` +NODE_LITT_DB_STORAGE_HOST_PATH_1=${NODE_DB_PATH_HOST}/chunk_v2_litt_1 +NODE_LITT_DB_STORAGE_HOST_PATH_2=${NODE_DB_PATH_HOST}/chunk_v2_litt_2 +NODE_LITT_DB_STORAGE_PATHS=/data/operator/db/chunk_v2_litt_1,/data/operator/db/chunk_v2_litt_2 +``` +The `NODE_LITT_DB_STORAGE_HOST_PATH_X` should map to a separately mounted drive folder for each partition. +The `NODE_LITT_DB_STORAGE_PATHS` needs to map to the docker volume mounts defined in `docker-compose.yaml` +#### Example docker-compose volume mounts +``` + volumes: + - "${NODE_ECDSA_KEY_FILE_HOST}:/app/operator_keys/ecdsa_key.json:readonly" + - "${NODE_BLS_KEY_FILE_HOST}:/app/operator_keys/bls_key.json:readonly" + - "${NODE_G1_PATH_HOST}:/app/g1.point:readonly" + - "${NODE_G2_PATH_HOST}:/app/g2.point.powerOf2:readonly" + - "${NODE_CACHE_PATH_HOST}:/app/cache:rw" + - "${NODE_LOG_PATH_HOST}:/app/logs:rw" + - "${NODE_DB_PATH_HOST}:/data/operator/db:rw" + - "${NODE_LITT_DB_STORAGE_HOST_PATH_1}:/data/operator/db/chunk_v2_litt_1:rw" + - "${NODE_LITT_DB_STORAGE_HOST_PATH_2}:/data/operator/db/chunk_v2_litt_2:rw" +``` +#### Example node output after loading both partitions +``` +eigenda-native-node | Jun 2 18:11:47.430 INF node/validator_store.go:108 Using littDB at paths /data/operator/db/chunk_v2_litt_1, /data/operator/db/chunk_v2_litt_2 +eigenda-native-node | Jun 2 18:11:47.430 INF littbuilder/db_impl.go:118 LittDB started, current data size: 0 +eigenda-native-node | Jun 2 18:11:47.430 INF littbuilder/db_impl.go:169 creating table chunks \ No newline at end of file diff --git a/mainnet/docker-compose.yml b/mainnet/docker-compose.yml index 6b5164a..92020af 100644 --- a/mainnet/docker-compose.yml +++ b/mainnet/docker-compose.yml @@ -38,6 +38,8 @@ services: - "${NODE_CACHE_PATH_HOST}:/app/cache:rw" - "${NODE_LOG_PATH_HOST}:/app/logs:rw" - "${NODE_DB_PATH_HOST}:/data/operator/db:rw" +# TODO: Uncomment and update if using remote BLS signer https://github.com/Layr-Labs/cerberus +# - "${NODE_BLS_SIGNER_CERT_FILE_HOST}:/app/cerberus/cerberus.crt:readonly" restart: unless-stopped networks: eigenda: diff --git a/mainnet/run.sh b/mainnet/run.sh index eca1f7f..4fcb542 100755 --- a/mainnet/run.sh +++ b/mainnet/run.sh @@ -3,9 +3,20 @@ . ./.env -socket="$NODE_HOSTNAME":"${NODE_DISPERSAL_PORT}"\;"${NODE_RETRIEVAL_PORT}" +node_plugin_image="ghcr.io/layr-labs/eigenda/opr-nodeplugin:0.9.0" -node_plugin_image="ghcr.io/layr-labs/eigenda/opr-nodeplugin:0.8.6" +# Check if V2 ports are defined +if [ -z "$NODE_V2_DISPERSAL_PORT" ]; then + echo "ERROR: NODE_V2_DISPERSAL_PORT is not defined!" +fi +if [ -z "$NODE_V2_RETRIEVAL_PORT" ]; then + echo "ERROR: NODE_V2_RETRIEVAL_PORT is not defined!" +fi +if [ -z "$NODE_V2_DISPERSAL_PORT" ] || [ -z "$NODE_V2_RETRIEVAL_PORT" ]; then + echo "ERROR: Please update your .env file. See .env.example for reference." + exit 1 +fi +socket="$NODE_HOSTNAME":"${NODE_DISPERSAL_PORT}"\;"${NODE_RETRIEVAL_PORT}"\;"${NODE_V2_DISPERSAL_PORT}"\;"${NODE_V2_RETRIEVAL_PORT}" # In all commands, We have to explicitly set the password again here because # when docker run loads the `.env` file, it keeps the quotes around the password diff --git a/monitoring/dashboards/eigenda-metrics.json b/monitoring/dashboards/eigenda-metrics.json index cd78f6d..0363b8e 100644 --- a/monitoring/dashboards/eigenda-metrics.json +++ b/monitoring/dashboards/eigenda-metrics.json @@ -2034,14 +2034,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "avg(eigenda_node_store_chunks_latency_ms{}) by (quantile)", + "expr": "(\n sum by (stage) (\n rate(\n eigenda_node_store_chunks_stage_latency_ms_sum{}[5m]\n )\n )\n)\n/\n(\n sum by (stage) (\n rate(\n eigenda_node_store_chunks_stage_latency_ms_count{}[5m]\n )\n )\n)\n", "instant": false, - "legendFormat": "p{{quantile}}", + "legendFormat": "latency", "range": true, "refId": "A" } ], - "title": "EigenDA V2 Store Chunks Latency", + "title": "EigenDA V2 Average Store Chunks Latency by Stage", "type": "timeseries" }, { @@ -2110,7 +2110,7 @@ "x": 12, "y": 75 }, - "id": 29, + "id": 28, "options": { "legend": { "calcs": [], @@ -2130,14 +2130,15 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "avg(eigenda_node_get_chunks_latency_ms{\n}) by (quantile)", + "expr": "sum by (stage) (\n eigenda_node_store_chunks_stage_latency_ms{\n quantile=\"0.5\"\n }\n)\n", + "hide": false, "instant": false, - "legendFormat": "p{{quantile}}", + "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "EigenDA V2 Get Chunks Latency", + "title": "EigenDA V2 Store Chunks p50 Latency By Stage", "type": "timeseries" }, { @@ -2206,7 +2207,7 @@ "x": 0, "y": 82 }, - "id": 28, + "id": 31, "options": { "legend": { "calcs": [], @@ -2226,14 +2227,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by(stage) (eigenda_node_store_chunks_latency_ms{quantile=\"0.5\"})", + "expr": "sum by (stage) (\n eigenda_node_store_chunks_stage_latency_ms{\n quantile=\"0.9\"\n }\n)\n", "instant": false, - "legendFormat": "p50 {{stage}}", + "legendFormat": "{{stage}}", "range": true, "refId": "A" } ], - "title": "EigenDA V2 Store Chunks p50 Latency By Stage", + "title": "EigenDA V2 Store Chunks p90 Latency By Stage", "type": "timeseries" }, { @@ -2302,7 +2303,7 @@ "x": 12, "y": 82 }, - "id": 31, + "id": 32, "options": { "legend": { "calcs": [], @@ -2322,15 +2323,927 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "avg by(stage) (eigenda_node_store_chunks_latency_ms{quantile=\"0.99\"})", + "expr": "sum by (stage) (\n eigenda_node_store_chunks_stage_latency_ms{\n quantile=\"0.99\"\n }\n)\n", "instant": false, - "legendFormat": "p99 {{stage}}", + "legendFormat": "{{stage}}", "range": true, "refId": "A" } ], "title": "EigenDA V2 Store Chunks p99 Latency By Stage", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(eigenda_node_get_chunks_latency_ms{\n}) by (quantile)", + "instant": false, + "legendFormat": "p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "EigenDA V2 Get Chunks Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 97 + }, + "id": 33, + "panels": [], + "title": "LittDB", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size of LittDB data on disk.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(node_littdb_table_size_bytes{})\n", + "instant": false, + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "Disk Footprint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The memory footprint of the read cache.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(node_littdb_chunk_read_cache_weight{})\n", + "instant": false, + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "Read Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The memory footprint of the write cache.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(node_littdb_chunk_write_cache_weight{})\n", + "instant": false, + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "Write Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of bytes read from LittDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(node_littdb_bytes_read{}[3m])) by (service)", + "instant": false, + "legendFormat": "reads", + "range": true, + "refId": "A" + } + ], + "title": "Bytes Read / Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of bytes written to LittDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(node_littdb_bytes_written{}[3m])) by (service)", + "instant": false, + "legendFormat": "writes", + "range": true, + "refId": "A" + } + ], + "title": "Bytes Written / Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The average latency of LittDB read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(node_littdb_read_latency_ms{}) by (quantile)\n", + "instant": false, + "legendFormat": "{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(node_littdb_read_latency_ms_sum{}[5m]))\n/\nsum(rate(node_littdb_read_latency_ms_count{}[5m]))\n", + "hide": false, + "instant": false, + "legendFormat": "average", + "range": true, + "refId": "B" + } + ], + "title": "Read Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The average latency of LittDB write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 119 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(node_littdb_write_latency_ms{}) by (quantile)\n", + "instant": false, + "legendFormat": "{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(node_littdb_write_latency_ms{}[5m]))\n/\nsum(rate(node_littdb_write_latency_ms_count{}[5m]))\n", + "hide": false, + "instant": false, + "legendFormat": "average", + "range": true, + "refId": "B" + } + ], + "title": "Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of cache hits per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 119 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(node_littdb_cache_hits{}[5m]))\n", + "hide": false, + "instant": false, + "legendFormat": "hits", + "range": true, + "refId": "B" + } + ], + "title": "Cache Hits", + "type": "timeseries" } ], "refresh": "", @@ -2349,4 +3262,4 @@ "uid": "dcd9d495-ff1e-4794-b2b6-62634e4b40d1", "version": 17, "weekStart": "" -} \ No newline at end of file +}