Migrate controller e2e tests to use Controller CLI

cloudbuild.yaml

@@ -79,9 +79,8 @@ steps:
   entrypoint: /bin/bash
   args:
   - -c
-  - cd e2e-tests;
-    docker build -t ${_REPOSITORY}/e2e-tests:${_TAG} .;
-    docker build -t ${_REPOSITORY}/e2e-tests/controller-spark:${_TAG} ./controller-spark/.;
+  - docker build -t ${_REPOSITORY}/e2e-tests:${_TAG} -f e2e-tests/Dockerfile ./e2e-tests/;
+    docker build -t ${_REPOSITORY}/e2e-tests/controller-spark:${_TAG} -f e2e-tests/controller-spark/Dockerfile .;
   waitFor: ['-']
 
 - name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
@@ -203,17 +202,18 @@ steps:
   id: 'Run E2E Test for Dockerized Controller and Spark Thriftserver'
   waitFor: [ 'Bring up controller and Spark containers',
              'Run E2E Test for BULK_EXPORT mode with HAPI source' ]
+#  allowFailure: true
 
 # The controller logs don't appear in Cloud Build output because we run it in
 # the detached mode. For debugging controller failures we can use something like
 # the following (and forcing the previous step to have 0 exit code or by
 # adding allowFailure: true).
-# - name: 'gcr.io/cloud-builders/docker'
-#   id: 'PRINT CONTROLLER LOGS'
-#   entrypoint: /bin/bash
-#   args:
-#   - -c
-#   - docker logs pipeline-controller
+#- name: 'gcr.io/cloud-builders/docker'
+#  id: 'PRINT CONTROLLER LOGS'
+#  entrypoint: /bin/bash
+#  args:
+#  - -c
+#  - docker logs pipeline-controller
 
 - name: 'docker/compose'
   id: 'Bring down controller and Spark containers'

doc/docs/additional.md

@@ -213,17 +213,11 @@ controller <url> dwh delete --snapshot-id <snapshot id>
 first. A valid snapshot-id is the full id as shown in the list e.g.
 `dwh/controller_DEV_DWH_TIMESTAMP_2025_08_14T17_47_15_357080Z`
 
-**Note on CLI Access in Docker container:** If you are running the pipeline
-controller in a docker container as defined in ./Dockerfile e.g. such as when
-using the Single Machine docker compose configuration, you can access the CLI
-tool by running the following command from the host machine.
+**Note on CLI Access with Docker ** If you are running the pipeline controller
+in a docker container as defined in ./Dockerfile e.g. such as when using the
+Single Machine docker compose configuration, you can access the CLI tool by
+running the following command from the host machine.
 
 ```sh
-docker exec -it <container_id or container_name> /bin/bash
-```
-
-And then access the CLI tool as described above.
-
-```sh
-controller <url> <command> [options]
+docker exec <container_id or container_name> controller <url> <command> [options]
 ```

e2e-tests/controller-spark/Dockerfile

@@ -33,5 +33,10 @@ ENV PARQUET_SUBDIR="dwh"
 ENV DOCKER_NETWORK="--use_docker_network"
 ENV HOME_DIR="/workspace/e2e-tests/controller-spark"
 ENV DWH_TYPE="PARQUET"
+ENV CONTROLLER_CLI_DIR="./controller-cli"
+
+COPY ./pipelines/controller-cli/ ${CONTROLLER_CLI_DIR}/
+WORKDIR ${CONTROLLER_CLI_DIR}
+RUN pip3 install .
 
 ENTRYPOINT cd ${HOME_DIR}; ./controller_spark_sql_validation.sh ${HOME_DIR} ${PARQUET_SUBDIR} ${DOCKER_NETWORK} ${DWH_TYPE}

e2e-tests/controller-spark/controller_spark_sql_validation.sh

@@ -188,10 +188,7 @@ function fhir_source_query() {
 #######################################################################
 function run_pipeline() {
   local runMode=$1
-  curl --location --request POST "${PIPELINE_CONTROLLER_URL}/run?runMode=${runMode}" \
-  --connect-timeout 5 \
-  --header 'Content-Type: application/json' \
-  --header 'Accept: */*' -v
+  controller ${PIPELINE_CONTROLLER_URL} run --mode ${runMode}
 }
 
 function wait_for_completion() {
@@ -200,11 +197,8 @@ function wait_for_completion() {
 
   while [[ $(date -u +%s) -le ${end_time} ]]
   do
-    local pipeline_status=$(curl --location --request GET "${PIPELINE_CONTROLLER_URL}/status?" \
-    --connect-timeout 5 \
-    --header 'Content-Type: application/json' \
-    --header 'Accept: */*' -v \
-    | jq -r '.pipelineStatus')
+    local pipeline_status=$(controller "${PIPELINE_CONTROLLER_URL}" status \
+    | sed -n '/{/,/}/p' | jq -r '.pipelineStatus')
 
     if [[ "${pipeline_status}" == "RUNNING" ]]
     then