reduce CI time for wallclock runs

.github/actions/tests/scala_test/action.yml

@@ -90,6 +90,10 @@ inputs:
     description: "Restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts)"
     required: false
     default: "false"
+  disable_immediate_flush:
+    description: "Whether to disable immediate flush of logs to disk (may improve test performance, but may lose logs if the job is killed)"
+    required: false
+    default: "false"
 
 runs:
   using: "composite"
@@ -268,6 +272,9 @@ runs:
           if [[ "${{ inputs.is_frontend_test }}" == "true" ]]; then
             ./start-frontends.sh -d -s
           fi
+          if [[ "${{ inputs.disable_immediate_flush }}" == "true" ]]; then
+            export LOG_IMMEDIATE_FLUSH=false KMS_LOG_IMMEDIATE_FLUSH=false
+          fi
           ${{ inputs.pre_sbt_cmd }}
         post_sbt_cmd: |
           if [[ "${{ inputs.is_frontend_test }}" == "true" ]]; then

.github/workflows/build.scala_test.yml

@@ -58,6 +58,11 @@ on:
         required: false
         type: boolean
         default: false
+      disable_immediate_flush:
+        description: "Whether to disable immediate flush of logs to disk (may improve test performance, but may lose logs if the job is killed)"
+        required: false
+        type: boolean
+        default: false
 
 jobs:
 
@@ -139,5 +144,6 @@ jobs:
           failure_notifications_slack_channel: ${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }}
           daml_base_version: ${{ inputs.daml_base_version }}
           oss_only: ${{ inputs.oss_only }}
+          disable_immediate_flush: ${{ inputs.disable_immediate_flush }}
           artifactory_user: ${{ inputs.oss_only && '' || vars.ARTIFACTORY_USER }}
           artifactory_password: ${{ inputs.oss_only && '' || secrets.ARTIFACTORY_PASSWORD }}

.github/workflows/build.yml

@@ -163,8 +163,8 @@ jobs:
     with:
       runs_on: self-hosted-k8s-large
       test_names_file: 'test-full-class-names.log'
-      start_canton_options: -w
-      # TODO(DACH-NY/canton-network-node#10912) Investigate why things got slower
+      start_canton_options: -w -I
+      disable_immediate_flush: true
       parallelism: 10
       test_name: wall-clock-time
       with_gcp_creds: true

build.sbt

@@ -1781,6 +1781,7 @@ lazy val `apps-app`: Project =
       ),
       BuildCommon.sharedAppSettings,
       BuildCommon.cantonWarts,
+      Test / parallelExecution := false,
       bundleTask,
       assembly / test := {}, // don't run tests during assembly
       // when building the fat jar, we need to properly merge our artefacts

canton/community/testing/src/test/resources/logback-test.xml

@@ -19,6 +19,8 @@
   <appender name="FILE" class="ch.qos.logback.core.FileAppender">
     <file>${LOG_FILE_NAME:-log/canton_network_test.clog}</file>
     <append>${LOG_APPEND:-true}</append>
+    <!-- Allow for disabling flush on each log-line (faster, but may miss logs when crashing) -->
+    <immediateFlush>false</immediateFlush>
     <encoder class="com.digitalasset.canton.logging.CantonJsonEncoder">
       <fieldNames>
         <version>[ignore]</version>

start-canton.sh

@@ -18,6 +18,7 @@ function usage() {
   echo "  -F               same as -f, but does not start cometBFT, but rather assumes it is already running"
   echo "  -e               start canton using the canton provided BFT sequencer"
   echo "  -m               collect metrics and send them to our CI prometheus instance"
+  echo "  -I               disable immediate log flushing, potentially dropping logs on crash"
   echo "  -c <canton>      start a custom canton binary instead of the one on the PATH"
 }
 
@@ -32,8 +33,9 @@ start_cometbft=0
 use_cometbft=0
 use_bft=0
 collect_metrics=0
+immediate_log_flushing=1
 
-args=$(getopt -o "hdDap:c:wsbtfFegm" -l "help" -- "$@")
+args=$(getopt -o "hdDap:c:wsbtfFegmI" -l "help" -- "$@")
 
 eval set -- "$args"
 
@@ -91,6 +93,9 @@ do
         -m)
             collect_metrics=1
             ;;
+        -I)
+            immediate_log_flushing=0
+            ;;
         --)
             shift
             break
@@ -206,12 +211,17 @@ tmux_cmd_canton() {
   windowName="$1" tokensFile="$2" participantsFile="$3" baseConfig="$4" confOverrides="$5" logFileHint="$6"
   mainLogFile="log/${logFileHint}.clog"
   consoleLogFile="log/${logFileHint}.out"
+  local log_flush_opts=""
+  if [[ $immediate_log_flushing -eq 0 ]]; then
+    log_flush_opts="--log-immediate-flush=false --kms-log-immediate-flush=false"
+  fi
   tmux_cmd "$windowName" \
     "EXTRA_CLASSPATH=$COMETBFT_DRIVER/driver.jar \
      COMETBFT_DOCKER_IP=${COMETBFT_DOCKER_IP-} \
      CANTON_TOKEN_FILENAME=$tokensFile CANTON_PARTICIPANTS_FILENAME=$participantsFile JAVA_TOOL_OPTIONS=\"$JAVA_TOOL_OPTIONS\" $CANTON \
       -c $baseConfig $confOverrides \
       --log-level-canton=DEBUG \
+      $log_flush_opts \
       --log-encoder json \
       --log-file-name $mainLogFile \
       --bootstrap $bootstrapScriptPath \