Add ability to run tests in parallel [PULP-753]

Author: gerrod3

.github/workflows/test.yml

@@ -115,5 +115,5 @@ jobs:
           PULP_ENABLED_PLUGINS: "${{ matrix.pulp_enabled_plugins }}"
           OAS_VERSION: "${{ matrix.oas_version }}"
         run: |
-          .ci/run_container.sh make livetest
+          .ci/run_container.sh make paralleltest
 ...

Makefile

@@ -41,6 +41,9 @@ test: | tests/cli.toml
 livetest: | tests/cli.toml
 	python3 -m pytest -v tests pulp-glue/tests -m live
 
+paralleltest: | tests/cli.toml
+	python3 -m pytest -v tests pulp-glue/tests -m live -n 8
+
 unittest:
 	python3 -m pytest -v tests pulp-glue/tests cookiecutter/pulp_filter_extension.py -m "not live"
 

cookiecutter/ci/{{ cookiecutter.__project_name }}/.github/workflows/test.yml

@@ -87,5 +87,5 @@ jobs:
           OAS_VERSION: "${{ matrix.oas_version }}"
           {%- endraw %}
         run: |
-          .ci/run_container.sh make {% if cookiecutter.unittests %}live{% endif %}test
+          .ci/run_container.sh make {% if cookiecutter.unittests %}parallel{% endif %}test
 ...

cookiecutter/ci/{{ cookiecutter.__project_name }}/Makefile

@@ -60,6 +60,9 @@ test: | tests/cli.toml
 livetest: | tests/cli.toml
 	python3 -m pytest -v tests {%- if cookiecutter.glue %} pulp-glue{{ cookiecutter.__app_label_suffix }}/tests {%- endif %} -m live
 
+paralleltest: | tests/cli.toml
+	python3 -m pytest -v tests {%- if cookiecutter.glue %} pulp-glue{{ cookiecutter.__app_label_suffix }}/tests {%- endif %} -m live -n 8
+
 unittest:
 	python3 -m pytest -v tests {%- if cookiecutter.glue %} pulp-glue{{ cookiecutter.__app_label_suffix }}/tests {%- endif %}
 	{%- if cookiecutter.__app_label_suffix == "" %} cookiecutter/pulp_filter_extension.py {%- endif %} -m "not live"

pulpcore/cli/core/task.py

@@ -153,6 +153,10 @@ def cancel(
         states.append("running")
     if states:
         tasks = task_ctx.list(limit=1 << 64, offset=0, parameters={"state__in": states})
+        if pulp_ctx.api._dry_run:
+            raise click.ClickException(
+                _("Trying to cancel {} tasks in safe-mode, aborting").format(len(tasks))
+            )
         for task in tasks:
             task_ctx.cancel(task["pulp_href"], background=True)
         for task in tasks:

test_requirements.txt

@@ -1,5 +1,6 @@
 # Test requirements
 pytest>=7.0.0,<8.5
+pytest-xdist
 pytest-subtests>=0.12.0,<0.15
 python-gnupg>=0.5.0,<0.6
 trustme>=1.1.0,<1.3

tests/conftest.py

@@ -14,6 +14,7 @@ def pulp_cli_vars(pulp_cli_vars: t.Dict[str, str]) -> t.Dict[str, str]:
     result.update(
         {
             "FILE_REMOTE_URL": urljoin(PULP_FIXTURES_URL, "/file/PULP_MANIFEST"),
+            "FILE_REMOTE2_URL": urljoin(PULP_FIXTURES_URL, "/file2/PULP_MANIFEST"),
             "FILE_LARGE_REMOTE_URL": urljoin(PULP_FIXTURES_URL, "/file-perf/PULP_MANIFEST"),
             "CONTAINER_REMOTE_URL": "https://quay.io",
             "CONTAINER_IMAGE": "libpod/alpine",

tests/scripts/pulp_ansible/test_content.sh

@@ -7,35 +7,34 @@ set -eu
 pulp debug has-plugin --name "ansible" || exit 23
 
 cleanup() {
-  pulp ansible repository destroy --name "cli_test_ansible_repository" || true
-  pulp ansible repository destroy --name "cli_test_ansible_repository_verify" || true
-  pulp ansible repository destroy --name "cli_test_ansible_repository_upload" || true
+  pulp ansible repository destroy --name "cli_test_ansible_content_repository" || true
+  pulp ansible repository destroy --name "cli_test_ansible_content_repository_verify" || true
+  pulp ansible repository destroy --name "cli_test_ansible_content_repository_upload" || true
 }
 trap cleanup EXIT
 
-pulp orphan cleanup --protection-time 0
 
 if pulp debug has-plugin --name "ansible" --specifier ">=0.15.0"
 then
   gpg --output pulp_pubkey.key --armor --export "pulp-fixture-signing-key"
-  expect_succ pulp ansible repository create --name "cli_test_ansible_repository_verify" --gpgkey @pulp_pubkey.key
+  expect_succ pulp ansible repository create --name "cli_test_ansible_content_repository_verify" --gpgkey @pulp_pubkey.key
 else
-  expect_succ pulp ansible repository create --name "cli_test_ansible_repository_verify"
+  expect_succ pulp ansible repository create --name "cli_test_ansible_content_repository_verify"
 fi
 
 # Test ansible collection-version upload
 wget "https://galaxy.ansible.com/download/ansible-posix-1.3.0.tar.gz"
 sha256=$(sha256sum ansible-posix-1.3.0.tar.gz | cut -d' ' -f1)
 
-expect_succ pulp ansible repository create --name "cli_test_ansible_repository_upload"
-expect_succ pulp ansible content upload --file "ansible-posix-1.3.0.tar.gz" --repository "cli_test_ansible_repository_upload"
+expect_succ pulp ansible repository create --name "cli_test_ansible_content_repository_upload"
+expect_succ pulp ansible content upload --file "ansible-posix-1.3.0.tar.gz" --repository "cli_test_ansible_content_repository_upload"
 expect_succ pulp artifact list --sha256 "$sha256"
 test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
 expect_succ pulp ansible content list --name "posix" --namespace "ansible" --version "1.3.0"
 test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
 content_href="$(echo "$OUTPUT" | jq -r .[0].pulp_href)"
 expect_succ pulp ansible content show --href "$content_href"
-expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository_upload" --version 1
+expect_succ pulp ansible repository content list --repository "cli_test_ansible_content_repository_upload" --version 1
 test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
 
 # Test ansible role upload
@@ -57,29 +56,29 @@ then
   tar --extract --file="ansible-posix-1.3.0.tar.gz" "MANIFEST.json"
   collection_path="$(realpath 'MANIFEST.json')"
   signature_path="$("$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"/assets/sign_detached.sh "$collection_path" | jq -r .signature)"
-  expect_succ pulp ansible content --type "signature" upload --file "$signature_path" --collection "$content_href" --repository "cli_test_ansible_repository_verify"
+  expect_succ pulp ansible content --type "signature" upload --file "$signature_path" --collection "$content_href" --repository "cli_test_ansible_content_repository_verify"
   expect_succ pulp ansible content --type "signature" list --collection "$content_href" --pubkey-fingerprint "0C1A894EBB86AFAE218424CADDEF3019C2D4A8CF"
   test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
   content3_href="$(echo "$OUTPUT" | jq -r .[0].pulp_href)"
   expect_succ pulp ansible content --type "signature" show --href "$content3_href"
 fi
 
 # New content commands
-expect_succ pulp ansible repository create --name "cli_test_ansible_repository"
-expect_succ pulp ansible repository content add --repository "cli_test_ansible_repository" --name "posix" --namespace "ansible" --version "1.3.0"
-expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 1
+expect_succ pulp ansible repository create --name "cli_test_ansible_content_repository"
+expect_succ pulp ansible repository content add --repository "cli_test_ansible_content_repository" --name "posix" --namespace "ansible" --version "1.3.0"
+expect_succ pulp ansible repository content list --repository "cli_test_ansible_content_repository" --version 1
 test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
-expect_succ pulp ansible repository content --type "role" add --repository "cli_test_ansible_repository" --name "kubernetes-modules" --namespace "ansible" --version "0.0.1"
-expect_succ pulp ansible repository content --type "role" list --repository "cli_test_ansible_repository" --version 2
+expect_succ pulp ansible repository content --type "role" add --repository "cli_test_ansible_content_repository" --name "kubernetes-modules" --namespace "ansible" --version "0.0.1"
+expect_succ pulp ansible repository content --type "role" list --repository "cli_test_ansible_content_repository" --version 2
 test "$(echo "$OUTPUT" | jq -r length)" -eq "1"
 
 if pulp debug has-plugin --name "core" --specifier ">=3.11.0"
 then
-  expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 2 --all-types
+  expect_succ pulp ansible repository content list --repository "cli_test_ansible_content_repository" --version 2 --all-types
   test "$(echo "$OUTPUT" | jq -r length)" -eq "2"
 fi
 
-expect_succ pulp ansible repository content remove --repository "cli_test_ansible_repository" --href "$content_href"
-expect_succ pulp ansible repository content remove --repository "cli_test_ansible_repository" --href "$content2_href"
-expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository"
+expect_succ pulp ansible repository content remove --repository "cli_test_ansible_content_repository" --href "$content_href"
+expect_succ pulp ansible repository content remove --repository "cli_test_ansible_content_repository" --href "$content2_href"
+expect_succ pulp ansible repository content list --repository "cli_test_ansible_content_repository"
 test "$(echo "$OUTPUT" | jq -r length)" -eq "0"

tests/scripts/pulp_ansible/test_distribution.sh

@@ -7,21 +7,21 @@ set -eu
 pulp debug has-plugin --name "ansible" || exit 23
 
 cleanup() {
-  pulp ansible repository destroy --name "cli_test_ansible_repository" || true
+  pulp ansible repository destroy --name "cli_test_ansible_distribution_repository" || true
   pulp ansible distribution destroy --name "cli_test_ansible_distro" || true
   pulp ansible distribution destroy --name "cli_test_ansible_ver_distro" || true
 }
 trap cleanup EXIT
 
-expect_succ pulp ansible repository create --name "cli_test_ansible_repository"
+expect_succ pulp ansible repository create --name "cli_test_ansible_distribution_repository"
 
 expect_succ pulp ansible distribution create --name "cli_test_ansible_distro" \
   --base-path "cli_test_ansible_distro" \
-  --repository "cli_test_ansible_repository"
+  --repository "cli_test_ansible_distribution_repository"
 
 expect_succ pulp ansible distribution create --name "cli_test_ansible_ver_distro" \
   --base-path "cli_test_ansible_ver_distro" \
-  --repository "cli_test_ansible_repository" \
+  --repository "cli_test_ansible_distribution_repository" \
   --version "0"
 
 expect_succ pulp ansible distribution list

tests/scripts/pulp_ansible/test_sign.sh

@@ -8,25 +8,24 @@ pulp debug has-plugin --name "ansible" --specifier ">=0.12.0" || exit 23
 [ "$(pulp signing-service list --name "sign_ansible" | jq 'length')" = "1" ] || exit 23
 
 cleanup() {
-  pulp ansible remote -t "collection" destroy --name "cli_test_ansible_collection_remote" || true
-  pulp ansible repository destroy --name "cli_test_ansible_repository" || true
-  pulp orphan cleanup || true
+  pulp ansible remote -t "collection" destroy --name "cli_test_ansible_sign_remote" || true
+  pulp ansible repository destroy --name "cli_test_ansible_sign_repository" || true
 }
 trap cleanup EXIT
 
 # Prepare
-expect_succ pulp ansible remote -t "collection" create --name "cli_test_ansible_collection_remote" \
+expect_succ pulp ansible remote -t "collection" create --name "cli_test_ansible_sign_remote" \
 --url "$ANSIBLE_COLLECTION_REMOTE_URL" --requirements "collections:
   - robertdebock.ansible_development_environment"
-expect_succ pulp ansible repository create --name "cli_test_ansible_repository"
+expect_succ pulp ansible repository create --name "cli_test_ansible_sign_repository"
 HREF="$(echo "$OUTPUT" | jq -r '.pulp_href')"
-expect_succ pulp ansible repository sync --repository "cli_test_ansible_repository" --remote "cli_test_ansible_collection_remote"
+expect_succ pulp ansible repository sync --repository "cli_test_ansible_sign_repository" --remote "cli_test_ansible_sign_remote"
 
 # Test sign
-expect_succ pulp ansible repository sign --repository "cli_test_ansible_repository" --signing-service "sign_ansible"
+expect_succ pulp ansible repository sign --repository "cli_test_ansible_sign_repository" --signing-service "sign_ansible"
 
 # Verify sign
 expect_succ pulp ansible repository version list --repository "$HREF"
 test "$(echo "$OUTPUT" | jq -r length)" -eq 3
-expect_succ pulp ansible repository version show --repository "cli_test_ansible_repository" --version 2
+expect_succ pulp ansible repository version show --repository "cli_test_ansible_sign_repository" --version 2
 test "$(echo "$OUTPUT" | jq -r '.content_summary.present."ansible.collection_signature".count')" -gt 0