diff --git a/platform-enterprise_docs/enterprise-sidebar.json b/platform-enterprise_docs/enterprise-sidebar.json index 21509dca0..35f4c920e 100644 --- a/platform-enterprise_docs/enterprise-sidebar.json +++ b/platform-enterprise_docs/enterprise-sidebar.json @@ -58,7 +58,8 @@ "enterprise/advanced-topics/custom-launch-container", "enterprise/advanced-topics/firewall-configuration", "enterprise/advanced-topics/seqera-container-images", - "enterprise/advanced-topics/content-security-policy" + "enterprise/advanced-topics/content-security-policy", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting" diff --git a/platform-enterprise_docs/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_docs/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_docs/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/docker/tower.env b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/docker/tower.env index 7adc905ec..5d201aa09 100644 --- a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/docker/tower.env +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/docker/tower.env @@ -4,9 +4,6 @@ TOWER_JWT_SECRET= TOWER_LICENSE= -# Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.1/enterprise/configuration/overview#backend-memory-requirements -JAVA_OPTS="-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" - # Compute environment settings TOWER_ENABLE_PLATFORMS=altair-platform,awsbatch-platform,azbatch-platform,eks-platform,gke-platform,googlebatch-platform,k8s-platform,lsf-platform,moab-platform,slurm-platform,uge-platform diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-cron.yml b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-cron.yml index 981636d90..94754c867 100644 --- a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-cron.yml +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-cron.yml @@ -43,9 +43,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,cron" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.1/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" ports: - containerPort: 8080 resources: diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-svc.yml b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-svc.yml index 7aeb36d78..411baa263 100644 --- a/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-svc.yml +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/_templates/k8s/tower-svc.yml @@ -36,9 +36,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,ha" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.1/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" # TLS certificate for Studios #- name: TOWER_OIDC_PEM_PATH # value: '/data/certs/oidc.pem' diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_versioned_docs/version-24.1/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/configuration/overview.mdx b/platform-enterprise_versioned_docs/version-24.1/enterprise/configuration/overview.mdx index 357b24eff..107e75495 100644 --- a/platform-enterprise_versioned_docs/version-24.1/enterprise/configuration/overview.mdx +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/configuration/overview.mdx @@ -411,50 +411,6 @@ services: These default memory allocation limits are included in your Kubernetes manifests ([tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml)) and Docker Compose ([docker-compose.yml](../_templates/docker/docker-compose.yml)) configuration templates. ::: -### JVM memory tuning - -For production deployments, configure JVM memory parameters via the `JAVA_OPTS` environment variable. The following baseline configuration is suitable for most deployments: - -```bash -JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -:::note -These default JVM memory settings are included in the configuration templates provided in these docs: -- Kubernetes: [tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml) -- Docker Compose: [tower.env](../_templates/docker/tower.env) -::: - -**Parameter descriptions:** -- **Heap memory** (`-Xms`/`-Xmx`): Memory pool for Java objects. Set initial (`Xms`) and maximum (`Xmx`) heap size. -- **Direct memory** (`MaxDirectMemorySize`): Off-heap memory used for NIO operations, network buffers, and file I/O. Critical for handling concurrent workflow API operations. -- **Netty memory accounting** (`io.netty.maxDirectMemory=0`): Disables Netty's internal tracking; relies on JVM direct memory limits instead. -- **Buffer caching** (`jdk.nio.maxCachedBufferSize`): Limits size of cached NIO buffers to prevent excessive memory retention. - -**When to adjust these values:** - -Increase `MaxDirectMemorySize` if you observe: -- `OutOfMemoryError: Direct buffer memory` in logs -- High concurrent workflow launch rates (>100 simultaneous workflows) -- Large configuration payloads or extensive API usage - -Increase heap memory (`-Xmx`) if you observe: -- `OutOfMemoryError: Java heap space` in logs -- Garbage collection pauses affecting performance -- Growing memory usage under sustained load - -**Example: High-concurrency deployment** -For deployments running 200+ concurrent workflows: -```bash -JAVA_OPTS: -Xms1000M -Xmx3000M -XX:MaxDirectMemorySize=1600m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -Ensure container/pod memory limits are set higher than JVM limits to accommodate non-heap memory usage. - -:::warning -These are starting recommendations. Monitor your deployment's actual memory usage and adjust based on your specific workload patterns. Undersized memory allocation can cause OOM failures and service instability. -::: - ## Compute environments Configuration values to enable computing platforms and customize Batch Forge resource naming. diff --git a/platform-enterprise_versioned_docs/version-24.1/enterprise/upgrade.md b/platform-enterprise_versioned_docs/version-24.1/enterprise/upgrade.md index 223b4fdc9..75a310a12 100644 --- a/platform-enterprise_versioned_docs/version-24.1/enterprise/upgrade.md +++ b/platform-enterprise_versioned_docs/version-24.1/enterprise/upgrade.md @@ -15,17 +15,6 @@ The database volume is persistent on the local machine by default if you use the 1. Download the latest versions of your deployment templates and update your Seqera container versions: - [docker-compose.yml](./_templates/docker/docker-compose.yml) for Docker Compose deployments - [tower-cron.yml](./_templates/k8s/tower-cron.yml) and [tower-svc.yml](./_templates/k8s/tower-svc.yml) for Kubernetes deployments -1. **JVM memory configuration defaults (recommended)**: The following `JAVA_OPTS` environment variable is included in the deployment templates downloaded in the preceding step, to optimize JVM memory settings: - - ```bash - JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 - ``` - - These baseline values are suitable for most deployments running moderate concurrent workflow loads. - - :::tip - These are starting recommendations that may require tuning based on your deployment's workload. See [Backend memory requirements](./configuration/overview.mdx#backend-memory-requirements) for detailed guidance on when and how to adjust these values for your environment. - ::: 1. Restart the application. 1. If you're using a containerized database as part of your implementation: 1. Stop the application. diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/docker/tower.env b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/docker/tower.env index bc54d11f4..5d201aa09 100644 --- a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/docker/tower.env +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/docker/tower.env @@ -4,9 +4,6 @@ TOWER_JWT_SECRET= TOWER_LICENSE= -# Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.2/enterprise/configuration/overview#backend-memory-requirements -JAVA_OPTS="-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" - # Compute environment settings TOWER_ENABLE_PLATFORMS=altair-platform,awsbatch-platform,azbatch-platform,eks-platform,gke-platform,googlebatch-platform,k8s-platform,lsf-platform,moab-platform,slurm-platform,uge-platform diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-cron.yml b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-cron.yml index 0f75b3e90..50cf1a80e 100644 --- a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-cron.yml +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-cron.yml @@ -43,9 +43,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,cron" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" ports: - containerPort: 8080 resources: diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-svc.yml b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-svc.yml index baafe9d20..454b21a8e 100644 --- a/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-svc.yml +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/_templates/k8s/tower-svc.yml @@ -36,9 +36,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,ha" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/24.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" # TLS certificate for Studios #- name: TOWER_OIDC_PEM_PATH # value: '/data/certs/oidc.pem' diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_versioned_docs/version-24.2/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/configuration/overview.mdx b/platform-enterprise_versioned_docs/version-24.2/enterprise/configuration/overview.mdx index f6d8f6422..59fce3059 100644 --- a/platform-enterprise_versioned_docs/version-24.2/enterprise/configuration/overview.mdx +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/configuration/overview.mdx @@ -416,50 +416,6 @@ services: These default memory allocation limits are included in your Kubernetes manifests ([tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml)) and Docker Compose ([docker-compose.yml](../_templates/docker/docker-compose.yml)) configuration templates. ::: -### JVM memory tuning - -For production deployments, configure JVM memory parameters via the `JAVA_OPTS` environment variable. The following baseline configuration is suitable for most deployments: - -```bash -JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -:::note -These default JVM memory settings are included in the configuration templates provided in these docs: -- Kubernetes: [tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml) -- Docker Compose: [tower.env](../_templates/docker/tower.env) -::: - -**Parameter descriptions:** -- **Heap memory** (`-Xms`/`-Xmx`): Memory pool for Java objects. Set initial (`Xms`) and maximum (`Xmx`) heap size. -- **Direct memory** (`MaxDirectMemorySize`): Off-heap memory used for NIO operations, network buffers, and file I/O. Critical for handling concurrent workflow API operations. -- **Netty memory accounting** (`io.netty.maxDirectMemory=0`): Disables Netty's internal tracking; relies on JVM direct memory limits instead. -- **Buffer caching** (`jdk.nio.maxCachedBufferSize`): Limits size of cached NIO buffers to prevent excessive memory retention. - -**When to adjust these values:** - -Increase `MaxDirectMemorySize` if you observe: -- `OutOfMemoryError: Direct buffer memory` in logs -- High concurrent workflow launch rates (>100 simultaneous workflows) -- Large configuration payloads or extensive API usage - -Increase heap memory (`-Xmx`) if you observe: -- `OutOfMemoryError: Java heap space` in logs -- Garbage collection pauses affecting performance -- Growing memory usage under sustained load - -**Example: High-concurrency deployment** -For deployments running 200+ concurrent workflows: -```bash -JAVA_OPTS: -Xms1000M -Xmx3000M -XX:MaxDirectMemorySize=1600m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -Ensure container/pod memory limits are set higher than JVM limits to accommodate non-heap memory usage. - -:::warning -These are starting recommendations. Monitor your deployment's actual memory usage and adjust based on your specific workload patterns. Undersized memory allocation can cause OOM failures and service instability. -::: - ## Compute environments Configuration values to enable computing platforms and customize Batch Forge resource naming. diff --git a/platform-enterprise_versioned_docs/version-24.2/enterprise/upgrade.md b/platform-enterprise_versioned_docs/version-24.2/enterprise/upgrade.md index 52cdf65a3..3f0fcff60 100644 --- a/platform-enterprise_versioned_docs/version-24.2/enterprise/upgrade.md +++ b/platform-enterprise_versioned_docs/version-24.2/enterprise/upgrade.md @@ -15,17 +15,6 @@ The database volume is persistent on the local machine by default if you use the 1. Download the latest versions of your deployment templates and update your Seqera container versions: - [docker-compose.yml](./_templates/docker/docker-compose.yml) for Docker Compose deployments - [tower-cron.yml](./_templates/k8s/tower-cron.yml) and [tower-svc.yml](./_templates/k8s/tower-svc.yml) for Kubernetes deployments -1. **JVM memory configuration defaults (recommended)**: The following `JAVA_OPTS` environment variable is included in the deployment templates downloaded in the preceding step, to optimize JVM memory settings: - - ```bash - JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 - ``` - - These baseline values are suitable for most deployments running moderate concurrent workflow loads. - - :::tip - These are starting recommendations that may require tuning based on your deployment's workload. See [Backend memory requirements](./configuration/overview.mdx#backend-memory-requirements) for detailed guidance on when and how to adjust these values for your environment. - ::: 1. Restart the application. 1. If you're using a containerized database as part of your implementation: 1. Stop the application. diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/docker/tower.env b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/docker/tower.env index e40b2c5c8..b8d34bbd6 100644 --- a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/docker/tower.env +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/docker/tower.env @@ -4,9 +4,6 @@ TOWER_JWT_SECRET= TOWER_LICENSE= -# Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.1/enterprise/configuration/overview#backend-memory-requirements -JAVA_OPTS="-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" - # Compute environment settings TOWER_ENABLE_PLATFORMS=altair-platform,awsbatch-platform,azbatch-platform,eks-platform,gke-platform,googlebatch-platform,k8s-platform,lsf-platform,moab-platform,slurm-platform,uge-platform diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-cron.yml b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-cron.yml index 336e69d02..1f145189a 100644 --- a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-cron.yml +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-cron.yml @@ -43,9 +43,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,cron" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.1/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" ports: - containerPort: 8080 resources: diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-svc.yml b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-svc.yml index 120222614..1d7d6ac7f 100644 --- a/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-svc.yml +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/_templates/k8s/tower-svc.yml @@ -36,9 +36,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,ha" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.1/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" # TLS certificate for Studios #- name: TOWER_OIDC_PEM_PATH # value: '/data/certs/oidc.pem' diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_versioned_docs/version-25.1/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/configuration/overview.mdx b/platform-enterprise_versioned_docs/version-25.1/enterprise/configuration/overview.mdx index dfe4cabc5..e591684f7 100644 --- a/platform-enterprise_versioned_docs/version-25.1/enterprise/configuration/overview.mdx +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/configuration/overview.mdx @@ -416,50 +416,6 @@ services: These default memory allocation limits are included in your Kubernetes manifests ([tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml)) and Docker Compose ([docker-compose.yml](../_templates/docker/docker-compose.yml)) configuration templates. ::: -### JVM memory tuning - -For production deployments, configure JVM memory parameters via the `JAVA_OPTS` environment variable. The following baseline configuration is suitable for most deployments: - -```bash -JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -:::note -These default JVM memory settings are included in the configuration templates provided in these docs: -- Kubernetes: [tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml) -- Docker Compose: [tower.env](../_templates/docker/tower.env) -::: - -**Parameter descriptions:** -- **Heap memory** (`-Xms`/`-Xmx`): Memory pool for Java objects. Set initial (`Xms`) and maximum (`Xmx`) heap size. -- **Direct memory** (`MaxDirectMemorySize`): Off-heap memory used for NIO operations, network buffers, and file I/O. Critical for handling concurrent workflow API operations. -- **Netty memory accounting** (`io.netty.maxDirectMemory=0`): Disables Netty's internal tracking; relies on JVM direct memory limits instead. -- **Buffer caching** (`jdk.nio.maxCachedBufferSize`): Limits size of cached NIO buffers to prevent excessive memory retention. - -**When to adjust these values:** - -Increase `MaxDirectMemorySize` if you observe: -- `OutOfMemoryError: Direct buffer memory` in logs -- High concurrent workflow launch rates (>100 simultaneous workflows) -- Large configuration payloads or extensive API usage - -Increase heap memory (`-Xmx`) if you observe: -- `OutOfMemoryError: Java heap space` in logs -- Garbage collection pauses affecting performance -- Growing memory usage under sustained load - -**Example: High-concurrency deployment** -For deployments running 200+ concurrent workflows: -```bash -JAVA_OPTS: -Xms1000M -Xmx3000M -XX:MaxDirectMemorySize=1600m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -Ensure container/pod memory limits are set higher than JVM limits to accommodate non-heap memory usage. - -:::warning -These are starting recommendations. Monitor your deployment's actual memory usage and adjust based on your specific workload patterns. Undersized memory allocation can cause OOM failures and service instability. -::: - ## Compute environments Configuration values to enable computing platforms and customize Batch Forge resource naming. diff --git a/platform-enterprise_versioned_docs/version-25.1/enterprise/upgrade.md b/platform-enterprise_versioned_docs/version-25.1/enterprise/upgrade.md index 52cdf65a3..3f0fcff60 100644 --- a/platform-enterprise_versioned_docs/version-25.1/enterprise/upgrade.md +++ b/platform-enterprise_versioned_docs/version-25.1/enterprise/upgrade.md @@ -15,17 +15,6 @@ The database volume is persistent on the local machine by default if you use the 1. Download the latest versions of your deployment templates and update your Seqera container versions: - [docker-compose.yml](./_templates/docker/docker-compose.yml) for Docker Compose deployments - [tower-cron.yml](./_templates/k8s/tower-cron.yml) and [tower-svc.yml](./_templates/k8s/tower-svc.yml) for Kubernetes deployments -1. **JVM memory configuration defaults (recommended)**: The following `JAVA_OPTS` environment variable is included in the deployment templates downloaded in the preceding step, to optimize JVM memory settings: - - ```bash - JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 - ``` - - These baseline values are suitable for most deployments running moderate concurrent workflow loads. - - :::tip - These are starting recommendations that may require tuning based on your deployment's workload. See [Backend memory requirements](./configuration/overview.mdx#backend-memory-requirements) for detailed guidance on when and how to adjust these values for your environment. - ::: 1. Restart the application. 1. If you're using a containerized database as part of your implementation: 1. Stop the application. diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/docker/tower.env b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/docker/tower.env index 1e7900dee..aa01975f3 100644 --- a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/docker/tower.env +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/docker/tower.env @@ -4,9 +4,6 @@ TOWER_JWT_SECRET= TOWER_LICENSE= -# Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.2/enterprise/configuration/overview#backend-memory-requirements -JAVA_OPTS="-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" - # Compute environment settings TOWER_ENABLE_PLATFORMS=altair-platform,awsbatch-platform,awscloud-platform,azbatch-platform,eks-platform,gke-platform,googlebatch-platform,googlecloud-platform,k8s-platform,lsf-platform,moab-platform,slurm-platform,uge-platform diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-cron.yml b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-cron.yml index 12ef4d268..7176dcc55 100644 --- a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-cron.yml +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-cron.yml @@ -43,9 +43,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,cron" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" ports: - containerPort: 8080 resources: diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-svc.yml b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-svc.yml index e13cb31a0..8eaae680c 100644 --- a/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-svc.yml +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/_templates/k8s/tower-svc.yml @@ -36,9 +36,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,ha" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" # TLS certificate for Studios #- name: TOWER_OIDC_PEM_PATH # value: '/data/certs/oidc.pem' diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_versioned_docs/version-25.2/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/configuration/overview.mdx b/platform-enterprise_versioned_docs/version-25.2/enterprise/configuration/overview.mdx index 81fc2614d..35235a808 100644 --- a/platform-enterprise_versioned_docs/version-25.2/enterprise/configuration/overview.mdx +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/configuration/overview.mdx @@ -455,50 +455,6 @@ services: These default memory allocation limits are included in your Kubernetes manifests ([tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml)) and Docker Compose ([docker-compose.yml](../_templates/docker/docker-compose.yml)) configuration templates. ::: -### JVM memory tuning - -For production deployments, configure JVM memory parameters via the `JAVA_OPTS` environment variable. The following baseline configuration is suitable for most deployments: - -```bash -JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -:::note -These default JVM memory settings are included in the configuration templates provided in these docs: -- Kubernetes: [tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml) -- Docker Compose: [tower.env](../_templates/docker/tower.env) -::: - -**Parameter descriptions:** -- **Heap memory** (`-Xms`/`-Xmx`): Memory pool for Java objects. Set initial (`Xms`) and maximum (`Xmx`) heap size. -- **Direct memory** (`MaxDirectMemorySize`): Off-heap memory used for NIO operations, network buffers, and file I/O. Critical for handling concurrent workflow API operations. -- **Netty memory accounting** (`io.netty.maxDirectMemory=0`): Disables Netty's internal tracking; relies on JVM direct memory limits instead. -- **Buffer caching** (`jdk.nio.maxCachedBufferSize`): Limits size of cached NIO buffers to prevent excessive memory retention. - -**When to adjust these values:** - -Increase `MaxDirectMemorySize` if you observe: -- `OutOfMemoryError: Direct buffer memory` in logs -- High concurrent workflow launch rates (>100 simultaneous workflows) -- Large configuration payloads or extensive API usage - -Increase heap memory (`-Xmx`) if you observe: -- `OutOfMemoryError: Java heap space` in logs -- Garbage collection pauses affecting performance -- Growing memory usage under sustained load - -**Example: High-concurrency deployment** -For deployments running 200+ concurrent workflows: -```bash -JAVA_OPTS: -Xms1000M -Xmx3000M -XX:MaxDirectMemorySize=1600m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -Ensure container/pod memory limits are set higher than JVM limits to accommodate non-heap memory usage. - -:::warning -These are starting recommendations. Monitor your deployment's actual memory usage and adjust based on your specific workload patterns. Undersized memory allocation can cause OOM failures and service instability. -::: - ## Compute environments Configuration values to enable computing platforms and customize Batch Forge resource naming. diff --git a/platform-enterprise_versioned_docs/version-25.2/enterprise/upgrade.md b/platform-enterprise_versioned_docs/version-25.2/enterprise/upgrade.md index 5eff196f7..d3778ee30 100644 --- a/platform-enterprise_versioned_docs/version-25.2/enterprise/upgrade.md +++ b/platform-enterprise_versioned_docs/version-25.2/enterprise/upgrade.md @@ -70,18 +70,6 @@ The database volume is persistent on the local machine by default if you use the 1. Download the latest versions of your deployment templates and update your Seqera container versions: - [docker-compose.yml](./_templates/docker/docker-compose.yml) for Docker Compose deployments - [tower-cron.yml](./_templates/k8s/tower-cron.yml) and [tower-svc.yml](./_templates/k8s/tower-svc.yml) for Kubernetes deployments -1. **JVM memory configuration defaults (recommended)**: The following `JAVA_OPTS` environment variable is included in the deployment templates downloaded in the preceding step, to optimize JVM memory settings: - - ```bash - JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 - ``` - - These baseline values are suitable for most deployments running moderate concurrent workflow loads. - - :::tip - These are starting recommendations that may require tuning based on your deployment's workload. See [Backend memory requirements](./configuration/overview.mdx#backend-memory-requirements) for detailed guidance on when and how to adjust these values for your environment. - ::: - 1. If you're using Studios, download and apply the latest versions of the Kubernetes manifests: - [proxy.yml](./_templates/k8s/data_studios/proxy.yml) - [server.yml](./_templates/k8s/data_studios/server.yml) diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/docker/tower.env b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/docker/tower.env index 28491f14b..9874de2d6 100644 --- a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/docker/tower.env +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/docker/tower.env @@ -4,10 +4,6 @@ TOWER_JWT_SECRET= TOWER_LICENSE= - -# Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/enterprise/configuration/overview#backend-memory-requirements -JAVA_OPTS="-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" - # Compute environment settings TOWER_ENABLE_PLATFORMS=awsbatch-platform,azbatch-platform,gls-platform,googlebatch-platform,k8s-platform,uge-platform,slurm-platform diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-cron.yml b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-cron.yml index aa21d30e6..ed6f97da9 100644 --- a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-cron.yml +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-cron.yml @@ -43,9 +43,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,cron" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" ports: - containerPort: 8080 resources: diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-svc.yml b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-svc.yml index 256eb9693..9be690674 100644 --- a/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-svc.yml +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/_templates/k8s/tower-svc.yml @@ -36,9 +36,6 @@ spec: env: - name: MICRONAUT_ENVIRONMENTS value: "prod,redis,ha" - # Configuration to optimize JVM memory settings. See https://docs.seqera.io/platform-enterprise/25.2/enterprise/configuration/overview#backend-memory-requirements - - name: JAVA_OPTS - value: "-Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144" # TLS certificate for Studios #- name: TOWER_OIDC_PEM_PATH # value: '/data/certs/oidc.pem' diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/advanced-topics/jvm-memory-tuning.md b/platform-enterprise_versioned_docs/version-25.3/enterprise/advanced-topics/jvm-memory-tuning.md new file mode 100644 index 000000000..165880cf2 --- /dev/null +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/advanced-topics/jvm-memory-tuning.md @@ -0,0 +1,66 @@ +--- +title: "JVM memory tuning" +description: Configure JVM memory parameters for Seqera Platform Enterprise deployments +date created: "2025-12-17" +tags: [configuration, jvm, memory, tuning] +--- + +# JVM memory tuning + +:::warning +JVM memory tuning is an advanced topic that may cause instability and performance issues. +::: + +Seqera Platform scales memory allocation based on resources allocated to the application. To best inform available memory, set memory requests and limits on your deployments. We recommend increasing memory allocation before manually configuring JVM settings. + +If you wish to manually configure JVM memory, use the following baseline recommendations. + +## Memory parameters + +Set JVM memory parameters using the `JAVA_OPTS` environment variable. The following parameters control memory allocation: + +| Parameter | Description | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `-Xms` / `-Xmx` | Set the initial (`Xms`) and maximum (`Xmx`) heap size. The heap stores Java objects and should be 50-70% of total allocated memory. | +| `-XX:MaxDirectMemorySize` | Set the maximum direct (off-heap) memory. Used for NIO operations, network buffers, and file I/O. | +| `-XX:ActiveProcessorCount` | Set the number of CPUs available to the JVM. Should match the number of vCPUs allocated to the container. | + +## Resource allocation guidelines + +- **Heap (`-Xmx`)**: 50-70% of total allocated memory +- **Direct memory**: 10-20% of total allocated memory +- **Overhead** (metaspace, thread stacks, native memory): ~10% of total allocated memory + +Ensure total JVM memory (heap + direct memory + overhead) does not exceed container memory limits. + +## Example configurations + +The following table provides example configurations for common deployment sizes. These are starting points and may need to be tuned based on your specific usage patterns. + +| vCPU | RAM | Heap (`-Xmx`) | Direct Memory | `JAVA_OPTS` | +| :--: | :---: | :-----------: | :-----------: | ------------------------------------------------------------------------------- | +| 1 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=1 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 1 | 4 GB | 2.5 GB | 800 MB | `-XX:ActiveProcessorCount=1 -Xms1000M -Xmx2500M -XX:MaxDirectMemorySize=800m` | +| 2 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=2 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 2 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=2 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 2 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=2 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 2 GB | 1 GB | 512 MB | `-XX:ActiveProcessorCount=3 -Xms500M -Xmx1000M -XX:MaxDirectMemorySize=512m` | +| 3 | 4 GB | 2 GB | 800 MB | `-XX:ActiveProcessorCount=3 -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m` | +| 3 | 8 GB | 5 GB | 1.5 GB | `-XX:ActiveProcessorCount=3 -Xms2000M -Xmx5000M -XX:MaxDirectMemorySize=1500m` | +| 3 | 16 GB | 11 GB | 2.5 GB | `-XX:ActiveProcessorCount=3 -Xms4000M -Xmx11000M -XX:MaxDirectMemorySize=2500m` | + +## When to adjust memory settings + +Adjust your JVM memory settings if you observe the following issues in your deployment: + +**Increase heap memory (`-Xmx`)** if you see: + +- `OutOfMemoryError: Java heap space` errors in logs +- Garbage collection pauses affecting performance +- Steadily growing memory usage under sustained load + +**Increase direct memory (`MaxDirectMemorySize`)** if you see: + +- `OutOfMemoryError: Direct buffer memory` errors in logs +- High concurrent workflow launch rates (more than 100 simultaneous workflows) +- Large configuration payloads or extensive API usage diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/configuration/overview.mdx b/platform-enterprise_versioned_docs/version-25.3/enterprise/configuration/overview.mdx index 81fc2614d..35235a808 100644 --- a/platform-enterprise_versioned_docs/version-25.3/enterprise/configuration/overview.mdx +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/configuration/overview.mdx @@ -455,50 +455,6 @@ services: These default memory allocation limits are included in your Kubernetes manifests ([tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml)) and Docker Compose ([docker-compose.yml](../_templates/docker/docker-compose.yml)) configuration templates. ::: -### JVM memory tuning - -For production deployments, configure JVM memory parameters via the `JAVA_OPTS` environment variable. The following baseline configuration is suitable for most deployments: - -```bash -JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -:::note -These default JVM memory settings are included in the configuration templates provided in these docs: -- Kubernetes: [tower-svc.yml](../_templates/k8s/tower-svc.yml) and [tower-cron.yml](../_templates/k8s/tower-cron.yml) -- Docker Compose: [tower.env](../_templates/docker/tower.env) -::: - -**Parameter descriptions:** -- **Heap memory** (`-Xms`/`-Xmx`): Memory pool for Java objects. Set initial (`Xms`) and maximum (`Xmx`) heap size. -- **Direct memory** (`MaxDirectMemorySize`): Off-heap memory used for NIO operations, network buffers, and file I/O. Critical for handling concurrent workflow API operations. -- **Netty memory accounting** (`io.netty.maxDirectMemory=0`): Disables Netty's internal tracking; relies on JVM direct memory limits instead. -- **Buffer caching** (`jdk.nio.maxCachedBufferSize`): Limits size of cached NIO buffers to prevent excessive memory retention. - -**When to adjust these values:** - -Increase `MaxDirectMemorySize` if you observe: -- `OutOfMemoryError: Direct buffer memory` in logs -- High concurrent workflow launch rates (>100 simultaneous workflows) -- Large configuration payloads or extensive API usage - -Increase heap memory (`-Xmx`) if you observe: -- `OutOfMemoryError: Java heap space` in logs -- Garbage collection pauses affecting performance -- Growing memory usage under sustained load - -**Example: High-concurrency deployment** -For deployments running 200+ concurrent workflows: -```bash -JAVA_OPTS: -Xms1000M -Xmx3000M -XX:MaxDirectMemorySize=1600m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 -``` - -Ensure container/pod memory limits are set higher than JVM limits to accommodate non-heap memory usage. - -:::warning -These are starting recommendations. Monitor your deployment's actual memory usage and adjust based on your specific workload patterns. Undersized memory allocation can cause OOM failures and service instability. -::: - ## Compute environments Configuration values to enable computing platforms and customize Batch Forge resource naming. diff --git a/platform-enterprise_versioned_docs/version-25.3/enterprise/upgrade.md b/platform-enterprise_versioned_docs/version-25.3/enterprise/upgrade.md index 28e436bb0..5a06ec925 100644 --- a/platform-enterprise_versioned_docs/version-25.3/enterprise/upgrade.md +++ b/platform-enterprise_versioned_docs/version-25.3/enterprise/upgrade.md @@ -74,17 +74,6 @@ Starting from version 26.1, the frontend image running as root user will be depr 1. Download the latest versions of your deployment templates and update your Seqera container versions: - [docker-compose.yml](./_templates/docker/docker-compose.yml) for Docker Compose deployments - [tower-cron.yml](./_templates/k8s/tower-cron.yml) and [tower-svc.yml](./_templates/k8s/tower-svc.yml) for Kubernetes deployments -1. **JVM memory configuration defaults (recommended)**: The following `JAVA_OPTS` environment variable is included in the deployment templates downloaded in the preceding step, to optimize JVM memory settings: - - ```bash - JAVA_OPTS: -Xms1000M -Xmx2000M -XX:MaxDirectMemorySize=800m -Dio.netty.maxDirectMemory=0 -Djdk.nio.maxCachedBufferSize=262144 - ``` - - These baseline values are suitable for most deployments running moderate concurrent workflow loads. - - :::tip - These are starting recommendations that may require tuning based on your deployment's workload. See [Backend memory requirements](./configuration/overview.mdx#backend-memory-requirements) for detailed guidance on when and how to adjust these values for your environment. - ::: 1. If you're using Studios, download and apply the latest versions of the Kubernetes manifests: - [proxy.yml](./_templates/k8s/data_studios/proxy.yml) - [server.yml](./_templates/k8s/data_studios/server.yml) diff --git a/platform-enterprise_versioned_sidebars/version-24.1-sidebars.json b/platform-enterprise_versioned_sidebars/version-24.1-sidebars.json index 89a418100..c6be05750 100644 --- a/platform-enterprise_versioned_sidebars/version-24.1-sidebars.json +++ b/platform-enterprise_versioned_sidebars/version-24.1-sidebars.json @@ -57,7 +57,8 @@ "enterprise/advanced-topics/db-docker-to-RDS", "enterprise/advanced-topics/use-iam-role", "enterprise/advanced-topics/custom-launch-container", - "enterprise/advanced-topics/firewall-configuration" + "enterprise/advanced-topics/firewall-configuration", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting" diff --git a/platform-enterprise_versioned_sidebars/version-24.2-sidebars.json b/platform-enterprise_versioned_sidebars/version-24.2-sidebars.json index 7fde14c9e..09868c874 100644 --- a/platform-enterprise_versioned_sidebars/version-24.2-sidebars.json +++ b/platform-enterprise_versioned_sidebars/version-24.2-sidebars.json @@ -58,7 +58,8 @@ "enterprise/advanced-topics/use-iam-role", "enterprise/advanced-topics/custom-launch-container", "enterprise/advanced-topics/firewall-configuration", - "enterprise/advanced-topics/seqera-container-images" + "enterprise/advanced-topics/seqera-container-images", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting" diff --git a/platform-enterprise_versioned_sidebars/version-25.1-sidebars.json b/platform-enterprise_versioned_sidebars/version-25.1-sidebars.json index a8f00955d..cae86add7 100644 --- a/platform-enterprise_versioned_sidebars/version-25.1-sidebars.json +++ b/platform-enterprise_versioned_sidebars/version-25.1-sidebars.json @@ -58,7 +58,8 @@ "enterprise/advanced-topics/use-iam-role", "enterprise/advanced-topics/custom-launch-container", "enterprise/advanced-topics/firewall-configuration", - "enterprise/advanced-topics/seqera-container-images" + "enterprise/advanced-topics/seqera-container-images", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting" diff --git a/platform-enterprise_versioned_sidebars/version-25.2-sidebars.json b/platform-enterprise_versioned_sidebars/version-25.2-sidebars.json index e0dc82bd6..600226882 100644 --- a/platform-enterprise_versioned_sidebars/version-25.2-sidebars.json +++ b/platform-enterprise_versioned_sidebars/version-25.2-sidebars.json @@ -60,7 +60,8 @@ "enterprise/advanced-topics/custom-launch-container", "enterprise/advanced-topics/firewall-configuration", "enterprise/advanced-topics/seqera-container-images", - "enterprise/advanced-topics/content-security-policy" + "enterprise/advanced-topics/content-security-policy", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting" diff --git a/platform-enterprise_versioned_sidebars/version-25.3-sidebars.json b/platform-enterprise_versioned_sidebars/version-25.3-sidebars.json index b6baf2cbc..90ee6b5d1 100644 --- a/platform-enterprise_versioned_sidebars/version-25.3-sidebars.json +++ b/platform-enterprise_versioned_sidebars/version-25.3-sidebars.json @@ -59,7 +59,8 @@ "enterprise/advanced-topics/custom-launch-container", "enterprise/advanced-topics/firewall-configuration", "enterprise/advanced-topics/seqera-container-images", - "enterprise/advanced-topics/content-security-policy" + "enterprise/advanced-topics/content-security-policy", + "enterprise/advanced-topics/jvm-memory-tuning" ] }, "enterprise/general_troubleshooting"