From 77d7c5b9743dfe5d0ed5a12f849a19befadb9dbf Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:19:03 +0000 Subject: [PATCH 1/8] start-canton options to disable immediate flush [skip ci] Signed-off-by: Stephen Compall --- start-canton.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/start-canton.sh b/start-canton.sh index 5b8e21717c..925c2fec1c 100755 --- a/start-canton.sh +++ b/start-canton.sh @@ -18,6 +18,7 @@ function usage() { echo " -F same as -f, but does not start cometBFT, but rather assumes it is already running" echo " -e start canton using the canton provided BFT sequencer" echo " -m collect metrics and send them to our CI prometheus instance" + echo " -I disable immediate log flushing, potentially dropping logs on crash" echo " -c start a custom canton binary instead of the one on the PATH" } @@ -32,8 +33,9 @@ start_cometbft=0 use_cometbft=0 use_bft=0 collect_metrics=0 +immediate_log_flushing=1 -args=$(getopt -o "hdDap:c:wsbtfFegm" -l "help" -- "$@") +args=$(getopt -o "hdDap:c:wsbtfFegmI" -l "help" -- "$@") eval set -- "$args" @@ -91,6 +93,9 @@ do -m) collect_metrics=1 ;; + -I) + immediate_log_flushing=0 + ;; --) shift break @@ -206,12 +211,17 @@ tmux_cmd_canton() { windowName="$1" tokensFile="$2" participantsFile="$3" baseConfig="$4" confOverrides="$5" logFileHint="$6" mainLogFile="log/${logFileHint}.clog" consoleLogFile="log/${logFileHint}.out" + local log_flush_opts="" + if [[ $immediate_log_flushing -eq 0 ]]; then + log_flush_opts="--log-immediate-flush=false --kms-log-immediate-flush=false" + fi tmux_cmd "$windowName" \ "EXTRA_CLASSPATH=$COMETBFT_DRIVER/driver.jar \ COMETBFT_DOCKER_IP=${COMETBFT_DOCKER_IP-} \ CANTON_TOKEN_FILENAME=$tokensFile CANTON_PARTICIPANTS_FILENAME=$participantsFile JAVA_TOOL_OPTIONS=\"$JAVA_TOOL_OPTIONS\" $CANTON \ -c $baseConfig $confOverrides \ --log-level-canton=DEBUG \ + $log_flush_opts \ --log-encoder json \ --log-file-name $mainLogFile \ --bootstrap $bootstrapScriptPath \ From b3520b0f22da5070f027b2b9986c24813db5047f Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:19:57 +0000 Subject: [PATCH 2/8] add disable_immediate_flush to GHA, support in logback-test [skip ci] Signed-off-by: Stephen Compall --- .github/actions/tests/scala_test/action.yml | 7 +++++++ .github/workflows/build.scala_test.yml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/.github/actions/tests/scala_test/action.yml b/.github/actions/tests/scala_test/action.yml index aedf4b9c38..067bc11dc7 100644 --- a/.github/actions/tests/scala_test/action.yml +++ b/.github/actions/tests/scala_test/action.yml @@ -90,6 +90,10 @@ inputs: description: "Restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts)" required: false default: "false" + disable_immediate_flush: + description: "Whether to disable immediate flush of logs to disk (may improve test performance, but may lose logs if the job is killed)" + required: false + default: "false" runs: using: "composite" @@ -268,6 +272,9 @@ runs: if [[ "${{ inputs.is_frontend_test }}" == "true" ]]; then ./start-frontends.sh -d -s fi + if [[ "${{ inputs.disable_immediate_flush }}" == "true" ]]; then + export LOG_IMMEDIATE_FLUSH=false KMS_LOG_IMMEDIATE_FLUSH=false + fi ${{ inputs.pre_sbt_cmd }} post_sbt_cmd: | if [[ "${{ inputs.is_frontend_test }}" == "true" ]]; then diff --git a/.github/workflows/build.scala_test.yml b/.github/workflows/build.scala_test.yml index a7e5134898..0b4b0bcee9 100644 --- a/.github/workflows/build.scala_test.yml +++ b/.github/workflows/build.scala_test.yml @@ -58,6 +58,11 @@ on: required: false type: boolean default: false + disable_immediate_flush: + description: "Whether to disable immediate flush of logs to disk (may improve test performance, but may lose logs if the job is killed)" + required: false + type: boolean + default: "false" jobs: @@ -139,5 +144,6 @@ jobs: failure_notifications_slack_channel: ${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }} daml_base_version: ${{ inputs.daml_base_version }} oss_only: ${{ inputs.oss_only }} + disable_immediate_flush: ${{ inputs.disable_immediate_flush }} artifactory_user: ${{ inputs.oss_only && '' || vars.ARTIFACTORY_USER }} artifactory_password: ${{ inputs.oss_only && '' || secrets.ARTIFACTORY_PASSWORD }} From b1d6bea10e685f1b53e8e5bf106fb62dfd65d832 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:20:23 +0000 Subject: [PATCH 3/8] set immediate flush off for wallclock [skip ci] Signed-off-by: Stephen Compall --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6784000f6a..03ea38efc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -163,8 +163,8 @@ jobs: with: runs_on: self-hosted-k8s-large test_names_file: 'test-full-class-names.log' - start_canton_options: -w - # TODO(DACH-NY/canton-network-node#10912) Investigate why things got slower + start_canton_options: -w -I + disable_immediate_flush: true parallelism: 10 test_name: wall-clock-time with_gcp_creds: true From bd297952528b8feee017d120f7b4ad358d45093f Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:20:34 +0000 Subject: [PATCH 4/8] logback-test support [ci] Signed-off-by: Stephen Compall --- canton/community/testing/src/test/resources/logback-test.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/canton/community/testing/src/test/resources/logback-test.xml b/canton/community/testing/src/test/resources/logback-test.xml index 55a7af5b52..392d33d233 100644 --- a/canton/community/testing/src/test/resources/logback-test.xml +++ b/canton/community/testing/src/test/resources/logback-test.xml @@ -19,6 +19,8 @@ ${LOG_FILE_NAME:-log/canton_network_test.clog} ${LOG_APPEND:-true} + + ${LOG_IMMEDIATE_FLUSH:-true} [ignore] From 028490a4a23f757c1410a6e8cf0446dbf54e388f Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:26:04 +0000 Subject: [PATCH 5/8] [ci] Signed-off-by: Stephen Compall From c745c6779f39c7e5bed7f54bd29c0f5594590a33 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 18:37:38 +0000 Subject: [PATCH 6/8] quoting? [ci] Signed-off-by: Stephen Compall --- .github/workflows/build.scala_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.scala_test.yml b/.github/workflows/build.scala_test.yml index 0b4b0bcee9..d7f508e1e6 100644 --- a/.github/workflows/build.scala_test.yml +++ b/.github/workflows/build.scala_test.yml @@ -62,7 +62,7 @@ on: description: "Whether to disable immediate flush of logs to disk (may improve test performance, but may lose logs if the job is killed)" required: false type: boolean - default: "false" + default: false jobs: From 20a22a1d13c739eb745cab0b8d1b872c494f78b1 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 19:59:54 +0000 Subject: [PATCH 7/8] force false just in case option not passing through [ci] Signed-off-by: Stephen Compall --- canton/community/testing/src/test/resources/logback-test.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/canton/community/testing/src/test/resources/logback-test.xml b/canton/community/testing/src/test/resources/logback-test.xml index 392d33d233..994d24ca66 100644 --- a/canton/community/testing/src/test/resources/logback-test.xml +++ b/canton/community/testing/src/test/resources/logback-test.xml @@ -20,7 +20,7 @@ ${LOG_FILE_NAME:-log/canton_network_test.clog} ${LOG_APPEND:-true} - ${LOG_IMMEDIATE_FLUSH:-true} + false [ignore] From 31ec1c5dcae609c561df79df8b5de8c62e1faf60 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 26 Aug 2025 22:08:32 +0000 Subject: [PATCH 8/8] disable test parallel execution [ci] Signed-off-by: Stephen Compall --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index c9cb1d807d..aba8993b96 100644 --- a/build.sbt +++ b/build.sbt @@ -1781,6 +1781,7 @@ lazy val `apps-app`: Project = ), BuildCommon.sharedAppSettings, BuildCommon.cantonWarts, + Test / parallelExecution := false, bundleTask, assembly / test := {}, // don't run tests during assembly // when building the fat jar, we need to properly merge our artefacts