diff --git a/.github/actions/kfp-k8s/action.yml b/.github/actions/kfp-k8s/action.yml index 5b830b10ce1..12ea39aaf29 100644 --- a/.github/actions/kfp-k8s/action.yml +++ b/.github/actions/kfp-k8s/action.yml @@ -45,3 +45,12 @@ runs: if: ${{ steps.generate-kfp-kubernetes-proto-files.outcome == 'success' }} run: | pip install -e ./kubernetes_platform/python[dev] --find-links=sdk/python/dist + + # testing reinstalling kfp package from source with no deps + - name: Reinstall kfp from source with no deps + id: reinstall-kfp + shell: bash + if: ${{ steps.install-kfp-kubernetes.outcome == 'success' }} + working-directory: ./sdk/python + run: | + pip install . -I --no-deps \ No newline at end of file diff --git a/backend/metadata_writer/Dockerfile b/backend/metadata_writer/Dockerfile index 749807d3666..046d8ae0b92 100644 --- a/backend/metadata_writer/Dockerfile +++ b/backend/metadata_writer/Dockerfile @@ -1,5 +1,5 @@ # ml-metadata package depends on tensorflow package -FROM python:3.9 +FROM python:3.11 COPY backend/metadata_writer/requirements.txt /kfp/metadata_writer/ RUN python3 -m pip install -r /kfp/metadata_writer/requirements.txt diff --git a/backend/metadata_writer/update_requirements.sh b/backend/metadata_writer/update_requirements.sh index a907c2a9fc5..285f7f332da 100755 --- a/backend/metadata_writer/update_requirements.sh +++ b/backend/metadata_writer/update_requirements.sh @@ -1,5 +1,5 @@ #!/bin/bash # This image should be in sync with Dockerfile. -IMAGE="python:3.9" +IMAGE="python:3.11" ../../hack/update-requirements.sh $IMAGE requirements.txt diff --git a/backend/src/apiserver/config/testdata/sample_pipeline.yaml b/backend/src/apiserver/config/testdata/sample_pipeline.yaml index 4a0edac76e4..5a339453641 100644 --- a/backend/src/apiserver/config/testdata/sample_pipeline.yaml +++ b/backend/src/apiserver/config/testdata/sample_pipeline.yaml @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef hello_world():\n print('hello')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-hello-world root: diff --git a/backend/src/apiserver/resource/resource_manager_test.go b/backend/src/apiserver/resource/resource_manager_test.go index bbb563aa2ba..6ebd7100ff4 100644 --- a/backend/src/apiserver/resource/resource_manager_test.go +++ b/backend/src/apiserver/resource/resource_manager_test.go @@ -3479,7 +3479,7 @@ spec: - name: ENABLE_CACHING valueFrom: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} + - {name: KFP_V2_IMAGE, value: 'python:3.11'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type": "INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": {"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}}, @@ -3487,7 +3487,7 @@ spec: "instanceSchema": "", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'} envFrom: - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 + image: python:3.11 volumeMounts: - {mountPath: /kfp-launcher, name: kfp-launcher} inputs: @@ -3581,7 +3581,7 @@ spec: - name: ENABLE_CACHING valueFrom: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} + - {name: KFP_V2_IMAGE, value: 'python:3.11'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type": "INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", "schemaTitle": "system.Dataset", "instanceSchema": ""}}, "outputParameters": @@ -3589,7 +3589,7 @@ spec: "", "metadataPath": "/tmp/outputs/model/data"}}}'} envFrom: - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 + image: python:3.11 volumeMounts: - {mountPath: /kfp-launcher, name: kfp-launcher} inputs: @@ -4113,7 +4113,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: @@ -4146,7 +4146,7 @@ deploymentSpec: executors: exec-hello-world: container: - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipelines/p1/versions/v1 root: diff --git a/backend/src/apiserver/server/pipeline_upload_server_test.go b/backend/src/apiserver/server/pipeline_upload_server_test.go index 76e9ad66b7e..227d0afdf90 100644 --- a/backend/src/apiserver/server/pipeline_upload_server_test.go +++ b/backend/src/apiserver/server/pipeline_upload_server_test.go @@ -47,24 +47,24 @@ const ( func TestUploadPipeline(t *testing.T) { // TODO(v2): when we add a field to distinguish between v1 and v2 template, verify it's in the response tt := []struct { - name string - spec []byte + name string + spec []byte apiVersion string }{{ - name: "upload argo workflow YAML", - spec: []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"), + name: "upload argo workflow YAML", + spec: []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"), apiVersion: "v1beta1", }, { - name: "upload argo workflow YAML", - spec: []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"), + name: "upload argo workflow YAML", + spec: []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"), apiVersion: "v2beta1", }, { - name: "upload pipeline v2 job in proto yaml", - spec: []byte(v2SpecHelloWorld), + name: "upload pipeline v2 job in proto yaml", + spec: []byte(v2SpecHelloWorld), apiVersion: "v1beta1", }, { - name: "upload pipeline v2 job in proto yaml", - spec: []byte(v2SpecHelloWorld), + name: "upload pipeline v2 job in proto yaml", + spec: []byte(v2SpecHelloWorld), apiVersion: "v2beta1", }} for _, test := range tt { @@ -732,7 +732,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: @@ -789,7 +789,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: @@ -830,7 +830,7 @@ deploymentSpec: executors: exec-hello-world: container: - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world- root: @@ -855,7 +855,7 @@ deploymentSpec: executors: exec-hello-world: container: - image: python:3.9 + image: python:3.11 pipelineInfo: name: hEllo-world root: @@ -880,7 +880,7 @@ deploymentSpec: executors: exec-hello-world: container: - image: python:3.9 + image: python:3.11 pipelineInfo: name: more than 128 characters more than 128 characters more than 128 characters more than 128 characters more than 128 characters root: @@ -905,7 +905,7 @@ deploymentSpec: executors: exec-hello-world: container: - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-worl.d root: diff --git a/backend/src/apiserver/server/test/pipeline_with_volume.yaml b/backend/src/apiserver/server/test/pipeline_with_volume.yaml index 99c0009236b..3cc8963f003 100644 --- a/backend/src/apiserver/server/test/pipeline_with_volume.yaml +++ b/backend/src/apiserver/server/test/pipeline_with_volume.yaml @@ -66,7 +66,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-comp-2: container: args: @@ -92,7 +92,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-comp-3: container: args: @@ -118,7 +118,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc diff --git a/backend/src/apiserver/server/test/v2-hello-world.json b/backend/src/apiserver/server/test/v2-hello-world.json index bff7b6b7b2a..cb99850f3bf 100644 --- a/backend/src/apiserver/server/test/v2-hello-world.json +++ b/backend/src/apiserver/server/test/v2-hello-world.json @@ -22,7 +22,7 @@ "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Hello world', description='')\n_parser.add_argument(\"--text\", dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml b/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml index 67354ed309c..bdf24297add 100644 --- a/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml +++ b/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml @@ -293,7 +293,7 @@ deploymentSpec: \ try:\n os.makedirs(os.path.dirname(output_file))\n except\ \ OSError:\n pass\n with open(output_file, 'w') as f:\n \ \ f.write(_output_serializers[idx](_outputs[idx]))\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict: container: args: @@ -344,7 +344,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-2: container: args: @@ -398,7 +398,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-3: container: args: @@ -452,7 +452,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-4: container: args: @@ -503,7 +503,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train: container: args: @@ -620,7 +620,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train-2: container: args: @@ -737,7 +737,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: xgboost-sample-pipeline root: diff --git a/backend/src/apiserver/storage/pipeline_store_test.go b/backend/src/apiserver/storage/pipeline_store_test.go index 9683ba50ef6..0eda1b500b1 100644 --- a/backend/src/apiserver/storage/pipeline_store_test.go +++ b/backend/src/apiserver/storage/pipeline_store_test.go @@ -1936,7 +1936,7 @@ executors: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: diff --git a/backend/src/apiserver/template/testdata/hello_world.yaml b/backend/src/apiserver/template/testdata/hello_world.yaml index 8f37ce80098..fe727b4c3e5 100644 --- a/backend/src/apiserver/template/testdata/hello_world.yaml +++ b/backend/src/apiserver/template/testdata/hello_world.yaml @@ -31,7 +31,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: namespace/n1/pipeline/hello-world root: diff --git a/backend/src/apiserver/template/testdata/hello_world_schema_2_0_0.yaml b/backend/src/apiserver/template/testdata/hello_world_schema_2_0_0.yaml index ac46b6c1aa3..eb7293d27aa 100644 --- a/backend/src/apiserver/template/testdata/hello_world_schema_2_0_0.yaml +++ b/backend/src/apiserver/template/testdata/hello_world_schema_2_0_0.yaml @@ -30,7 +30,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: diff --git a/backend/src/apiserver/template/testdata/pipeline_with_volume.yaml b/backend/src/apiserver/template/testdata/pipeline_with_volume.yaml index 64247b4bbff..24b6c02c65a 100644 --- a/backend/src/apiserver/template/testdata/pipeline_with_volume.yaml +++ b/backend/src/apiserver/template/testdata/pipeline_with_volume.yaml @@ -68,7 +68,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-comp-2: container: args: @@ -94,7 +94,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-comp-3: container: args: @@ -120,7 +120,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc diff --git a/backend/src/cache/server/mutation_test.go b/backend/src/cache/server/mutation_test.go index b0b082f3580..3c0abe8107a 100644 --- a/backend/src/cache/server/mutation_test.go +++ b/backend/src/cache/server/mutation_test.go @@ -52,7 +52,7 @@ var ( Command: []string{"python"}, Env: []corev1.EnvVar{{ Name: ArgoWorkflowTemplateEnvKey, - Value: `{"name": "Does not matter","container":{"command":["echo", "Hello"],"image":"python:3.9"}}`, + Value: `{"name": "Does not matter","container":{"command":["echo", "Hello"],"image":"python:3.11"}}`, }}, }, }, @@ -169,9 +169,9 @@ func TestMutatePodIfCached(t *testing.T) { func TestMutatePodIfCachedWithCacheEntryExist(t *testing.T) { executionCache := &model.ExecutionCache{ - ExecutionCacheKey: "1933d178a14bc415466cfd1b3ca2100af975e8c59e1ff9d502fcf18eb5cbd7f7", + ExecutionCacheKey: "07f2c42567af4f141a52887e0a113c9aefdfdfd5e6b06b9908f7fdb0b43739af", ExecutionOutput: "testOutput", - ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.9"}}`, + ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.11"}}`, MaxCacheStaleness: -1, } fakeClientManager.CacheStore().CreateExecutionCache(executionCache) @@ -188,9 +188,9 @@ func TestMutatePodIfCachedWithCacheEntryExist(t *testing.T) { func TestDefaultImage(t *testing.T) { executionCache := &model.ExecutionCache{ - ExecutionCacheKey: "1933d178a14bc415466cfd1b3ca2100af975e8c59e1ff9d502fcf18eb5cbd7f7", + ExecutionCacheKey: "07f2c42567af4f141a52887e0a113c9aefdfdfd5e6b06b9908f7fdb0b43739af", ExecutionOutput: "testOutput", - ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.9"}}`, + ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.11"}}`, MaxCacheStaleness: -1, } fakeClientManager.CacheStore().CreateExecutionCache(executionCache) @@ -209,7 +209,7 @@ func TestSetImage(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", ExecutionOutput: "testOutput", - ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.9"}}`, + ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.11"}}`, MaxCacheStaleness: -1, } fakeClientManager.CacheStore().CreateExecutionCache(executionCache) @@ -226,7 +226,7 @@ func TestCacheNodeRestriction(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", ExecutionOutput: "testOutput", - ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.9"},"nodeSelector":{"disktype":"ssd"}}`, + ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.11"},"nodeSelector":{"disktype":"ssd"}}`, MaxCacheStaleness: -1, } fakeClientManager.CacheStore().CreateExecutionCache(executionCache) @@ -239,9 +239,9 @@ func TestCacheNodeRestriction(t *testing.T) { func TestMutatePodIfCachedWithTeamplateCleanup(t *testing.T) { executionCache := &model.ExecutionCache{ - ExecutionCacheKey: "c81988503d55a5817d79bd972017d95c37f72b024e522b4d79787d9f599c0725", + ExecutionCacheKey: "4b868c5d0b64e4d93e529eadaa04f0451eb5ae5c652dd79c08bdc47a6a1fe67a", ExecutionOutput: "testOutput", - ExecutionTemplate: `Cache key was calculated from this: {"container":{"command":["echo", "Hello"],"image":"python:3.9"},"outputs":"anything"}`, + ExecutionTemplate: `Cache key was calculated from this: {"container":{"command":["echo", "Hello"],"image":"python:3.11"},"outputs":"anything"}`, MaxCacheStaleness: -1, } fakeClientManager.CacheStore().CreateExecutionCache(executionCache) @@ -253,7 +253,7 @@ func TestMutatePodIfCachedWithTeamplateCleanup(t *testing.T) { "name": "Does not matter", "metadata": "anything", "container": { - "image": "python:3.9", + "image": "python:3.11", "command": ["echo", "Hello"] }, "outputs": "anything", diff --git a/backend/src/crd/kubernetes/v2beta1/pipelineversion_types_test.go b/backend/src/crd/kubernetes/v2beta1/pipelineversion_types_test.go index 41abdc5672e..d548959ddfb 100644 --- a/backend/src/crd/kubernetes/v2beta1/pipelineversion_types_test.go +++ b/backend/src/crd/kubernetes/v2beta1/pipelineversion_types_test.go @@ -675,7 +675,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef hello_world(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: diff --git a/backend/src/v2/cacheutils/cache_test.go b/backend/src/v2/cacheutils/cache_test.go index 8336f17733e..9786a369a42 100644 --- a/backend/src/v2/cacheutils/cache_test.go +++ b/backend/src/v2/cacheutils/cache_test.go @@ -96,7 +96,7 @@ func TestGenerateCacheKey(t *testing.T) { "output_parameter_two": "INT", }, cmdArgs: []string{"sh", "ec", "test"}, - image: "python:3.9", + image: "python:3.11", want: &cachekey.CacheKey{ InputArtifactNames: map[string]*cachekey.ArtifactNameList{ "dataset_one": {ArtifactNames: []string{"1"}}, @@ -130,7 +130,7 @@ func TestGenerateCacheKey(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", }, }, @@ -184,7 +184,7 @@ func TestGenerateCacheKey(t *testing.T) { "output_parameter_one": "STRING", }, cmdArgs: []string{"sh", "ec", "test"}, - image: "python:3.9", + image: "python:3.11", pvcNames: []string{"workspace-pvc", "data-pvc"}, want: &cachekey.CacheKey{ InputArtifactNames: map[string]*cachekey.ArtifactNameList{ @@ -210,7 +210,7 @@ func TestGenerateCacheKey(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", PvcNames: []string{"workspace-pvc", "data-pvc"}, }, }, @@ -272,7 +272,7 @@ func TestGenerateFingerPrint(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", }, } tests := []struct { @@ -316,11 +316,11 @@ func TestGenerateFingerPrint(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", }, }, wantEqual: true, - fingerPrint: "4e8a5d7d70997b0a35429fcd481af8fcd5b9f58ef4391bdb6ad900fd1c63622b", + fingerPrint: "b498530c9016917298999ec0b01f364a180f8a3862bea0fa672a383ca3f22e4e", }, { name: "Generated Different FingerPrint", cacheKey: &cachekey.CacheKey{ @@ -332,11 +332,11 @@ func TestGenerateFingerPrint(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "run"}, - Image: "python:3.9", + Image: "python:3.11", }, }, wantEqual: false, - fingerPrint: "0a4cc1f15cdfad5170e1358518f7128c5278500a670db1b9a3f3d83b93db396e", + fingerPrint: "3d9a2a778fa3174c6cfc6e639c507c265b5f21ef6e5b1dd70b236462cc6da464", }, } cacheClient, err := NewClient(false, &tls.Config{}) @@ -363,7 +363,7 @@ func TestGenerateFingerPrint_ConsidersPVCNames(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", }, } @@ -376,7 +376,7 @@ func TestGenerateFingerPrint_ConsidersPVCNames(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", PvcNames: []string{"workspace-pvc", "data-pvc"}, }, } @@ -390,7 +390,7 @@ func TestGenerateFingerPrint_ConsidersPVCNames(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", PvcNames: []string{"workspace-pvc", "data-pvc"}, }, } @@ -404,7 +404,7 @@ func TestGenerateFingerPrint_ConsidersPVCNames(t *testing.T) { }, ContainerSpec: &cachekey.ContainerSpec{ CmdArgs: []string{"sh", "ec", "test"}, - Image: "python:3.9", + Image: "python:3.11", PvcNames: []string{"data-pvc", "workspace-pvc", "extra"}, }, } diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index 50df3fa5c09..cdb0fb7f30c 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -59,7 +59,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { "Valid - nvidia.com/gpu", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -100,7 +100,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { "Valid - amd.com/gpu", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -141,7 +141,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { "Valid - cloud-tpus.google.com/v3", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -182,7 +182,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { "Valid - cloud-tpus.google.com/v2", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -223,7 +223,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { "Valid - custom string", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -300,7 +300,7 @@ func Test_initPodSpecPatch_acceleratorConfig(t *testing.T) { func Test_initPodSpecPatch_resource_placeholders(t *testing.T) { containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -416,7 +416,7 @@ func Test_initPodSpecPatch_resource_placeholders(t *testing.T) { func Test_initPodSpecPatch_legacy_resources(t *testing.T) { containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -466,7 +466,7 @@ func Test_initPodSpecPatch_legacy_resources(t *testing.T) { func Test_initPodSpecPatch_modelcar_input_artifact(t *testing.T) { containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, } @@ -583,7 +583,7 @@ func Test_initPodSpecPatch_resourceRequests(t *testing.T) { "Valid - with requests", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -621,7 +621,7 @@ func Test_initPodSpecPatch_resourceRequests(t *testing.T) { "Valid - zero requests", args{ &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -697,7 +697,7 @@ func Test_initPodSpecPatch_TaskConfig_ForwardsResourcesOnly(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -752,7 +752,7 @@ func Test_initPodSpecPatch_TaskConfig_ForwardsResourcesOnly(t *testing.T) { func Test_initPodSpecPatch_inputTaskFinalStatus(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Args: []string{"--executor-input", "{{$}}", "--function_to_execute", "exit-op"}, } @@ -972,7 +972,7 @@ func TestGetWorkspaceMount(t *testing.T) { // Ensure that when workspace is used, missing RunName leads to an error during pod spec init. func Test_initPodSpecPatch_WorkspaceRequiresRunName(t *testing.T) { - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.9"} + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.11"} componentSpec := &pipelinespec.ComponentSpec{} executorInput := &pipelinespec.ExecutorInput{ Inputs: &pipelinespec.ExecutorInput_Inputs{ @@ -1091,7 +1091,7 @@ func TestValidateVolumeMounts(t *testing.T) { } func TestWorkspaceMount_PassthroughVolumes_CaptureOnly(t *testing.T) { - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.9"} + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.11"} componentSpec := &pipelinespec.ComponentSpec{ TaskConfigPassthroughs: []*pipelinespec.TaskConfigPassthrough{ { @@ -1134,7 +1134,7 @@ func TestWorkspaceMount_PassthroughVolumes_CaptureOnly(t *testing.T) { } func TestWorkspaceMount_PassthroughVolumes_ApplyAndCapture(t *testing.T) { - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.9"} + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.11"} componentSpec := &pipelinespec.ComponentSpec{ TaskConfigPassthroughs: []*pipelinespec.TaskConfigPassthrough{ { @@ -1192,7 +1192,7 @@ func TestWorkspaceMount_PassthroughVolumes_ApplyAndCapture(t *testing.T) { func Test_initPodSpecPatch_TaskConfig_Env_Passthrough_CaptureOnly(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Env: []*pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_EnvVar{{ Name: "FOO", Value: "bar", @@ -1235,7 +1235,7 @@ func Test_initPodSpecPatch_TaskConfig_Env_Passthrough_CaptureOnly(t *testing.T) func Test_initPodSpecPatch_TaskConfig_Resources_Passthrough_ApplyAndCapture(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.9", + Image: "python:3.11", Args: []string{"--function_to_execute", "add"}, Command: []string{"sh", "-ec", "python3 -m kfp.components.executor_main"}, Resources: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec_ResourceSpec{ @@ -1291,7 +1291,7 @@ func Test_initPodSpecPatch_TaskConfig_Resources_Passthrough_ApplyAndCapture(t *t func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_Passthrough(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.9"} + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.11"} componentSpec := &pipelinespec.ComponentSpec{ TaskConfigPassthroughs: []*pipelinespec.TaskConfigPassthrough{ {Field: pipelinespec.TaskConfigPassthroughType_KUBERNETES_AFFINITY, ApplyToTask: false}, @@ -1389,7 +1389,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_Passthro func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_ApplyAndCapture(t *testing.T) { proxy.InitializeConfigWithEmptyForTests() - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.9"} + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{Image: "python:3.11"} componentSpec := &pipelinespec.ComponentSpec{ TaskConfigPassthroughs: []*pipelinespec.TaskConfigPassthrough{ {Field: pipelinespec.TaskConfigPassthroughType_KUBERNETES_AFFINITY, ApplyToTask: true}, diff --git a/backend/src/v2/test/Dockerfile b/backend/src/v2/test/Dockerfile index caf7acc9c6a..63c5368acdb 100644 --- a/backend/src/v2/test/Dockerfile +++ b/backend/src/v2/test/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim +FROM python:3.11-slim WORKDIR /workdir COPY backend/src/v2/test/requirements.txt backend/src/v2/test/ diff --git a/backend/src/v2/test/components/run_sample.yaml b/backend/src/v2/test/components/run_sample.yaml index ceb5f340208..26d28519af6 100644 --- a/backend/src/v2/test/components/run_sample.yaml +++ b/backend/src/v2/test/components/run_sample.yaml @@ -24,7 +24,7 @@ inputs: - {name: backend_compiler, type: Binary} implementation: container: - image: python:3.9-alpine + image: python:3.11-alpine command: - sh - -exc diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb b/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb index 9e0d394e65b..39a55ae76aa 100644 Binary files a/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb and b/backend/test/proto_tests/testdata/generated-1791485/pipeline_spec.pb differ diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.json b/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.json index cd2c395b4c8..cd7ef772864 100644 --- a/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.json +++ b/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.json @@ -34,7 +34,7 @@ "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef hello_world():\n print(\"hello world\")\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb b/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb index 6b6c23e1401..24af7ab2346 100644 Binary files a/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb and b/backend/test/proto_tests/testdata/generated-1791485/pipeline_version.pb differ diff --git a/backend/test/proto_tests/testdata/generated-1791485/run_completed.pb b/backend/test/proto_tests/testdata/generated-1791485/run_completed.pb index 08da9027c27..8fdf9af3af0 100644 Binary files a/backend/test/proto_tests/testdata/generated-1791485/run_completed.pb and b/backend/test/proto_tests/testdata/generated-1791485/run_completed.pb differ diff --git a/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.json b/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.json index 0685b66eadd..57ab5c01d60 100644 --- a/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.json +++ b/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.json @@ -29,7 +29,7 @@ "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef hello_world():\n print(\"hello world\")\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.pb b/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.pb index cb9a24f070c..e026a77fff6 100644 Binary files a/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.pb and b/backend/test/proto_tests/testdata/generated-1791485/run_completed_with_spec.pb differ diff --git a/backend/test/proto_tests/testdata/pipelinespec.yaml b/backend/test/proto_tests/testdata/pipelinespec.yaml index 6d16209981b..9715246b257 100644 --- a/backend/test/proto_tests/testdata/pipelinespec.yaml +++ b/backend/test/proto_tests/testdata/pipelinespec.yaml @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef hello_world():\n print(\"hello world\")\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-hello-world root: diff --git a/backend/test/resources/v2-hello-world.yaml b/backend/test/resources/v2-hello-world.yaml index 203c205f26b..fa43d444663 100644 --- a/backend/test/resources/v2-hello-world.yaml +++ b/backend/test/resources/v2-hello-world.yaml @@ -30,7 +30,7 @@ deploymentSpec: _parsed_args = vars(_parser.parse_args()) _outputs = hello_world(**_parsed_args) - image: python:3.9 + image: python:3.11 pipelineInfo: name: hello-world root: diff --git a/backend/test/v2/resources/pvc-mount.yaml b/backend/test/v2/resources/pvc-mount.yaml index 73137fc1a57..194adbea0e4 100644 --- a/backend/test/v2/resources/pvc-mount.yaml +++ b/backend/test/v2/resources/pvc-mount.yaml @@ -41,7 +41,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef consumer() -> None:\n with open('/data/file.txt', 'r') as\ \ f:\n print(f.read())\n\n" - image: python:3.9 + image: python:3.11 exec-producer: container: args: @@ -71,7 +71,7 @@ deploymentSpec: \ *\n\ndef producer() -> str:\n with open('/data/file.txt', 'w') as f:\n\ \ f.write('hello')\n with open('/data/file.txt', 'r') as f:\n\ \ return f.read()\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pvc-mount-pipeline root: diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py index e059cf7ec28..93276c4d96b 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py @@ -19,7 +19,7 @@ @dsl.component( - base_image='python:3.9', packages_to_install=['tensorflow==2.16.1'] + base_image='python:3.11', packages_to_install=['tensorflow==2.16.1'] ) def get_training_artifacts( docker_region: str, diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py index 5f504592185..1d231954c21 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py @@ -18,7 +18,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9') +@dsl.component(base_image='python:3.11') def maybe_set_tfrecord_args( dataprep_previous_run_dir: str, static_covariates: List[str], diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py index 9ce66e525e1..190f37259d1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py @@ -18,7 +18,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9') +@dsl.component(base_image='python:3.11') def set_dataprep_args( model_blocks: List[str], ts_identifier_columns: List[str], diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py index ee8d666f95c..7e6a7222f02 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py @@ -18,7 +18,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9') +@dsl.component(base_image='python:3.11') def set_eval_args( big_query_source: dsl.Input[dsl.Artifact], quantiles: List[float] ) -> NamedTuple( diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py index 9a4bad35041..c7616564769 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py @@ -19,7 +19,7 @@ @dsl.component( - base_image='python:3.9', packages_to_install=['tensorflow==2.16.1'] + base_image='python:3.11', packages_to_install=['tensorflow==2.16.1'] ) def set_test_set( dataprep_dir: dsl.InputPath(), diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py index 47d50ee622a..433e4dec337 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py @@ -18,7 +18,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9') +@dsl.component(base_image='python:3.11') def set_tfrecord_args( dataprep_dir: dsl.InputPath(), static_covariates: List[str], diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py index 66bf1b684e8..a42fa7c8dca 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py @@ -18,7 +18,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9') +@dsl.component(base_image='python:3.11') def set_train_args( quantiles: List[float], model_blocks: List[str], diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py index dc66280ef9b..8af846c646e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py @@ -17,7 +17,7 @@ @dsl.component( - base_image='python:3.9', + base_image='python:3.11', packages_to_install=[ 'google-cloud-aiplatform[tensorboard]==1.87.0', 'protobuf==3.20.*', diff --git a/components/kserve/Dockerfile b/components/kserve/Dockerfile index 507003ccf0d..b74d0ad0c28 100644 --- a/components/kserve/Dockerfile +++ b/components/kserve/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-bullseye +FROM python:3.11-slim-bullseye RUN apt-get update && apt-get install -y gcc python3-dev COPY requirements.txt . diff --git a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json index 2e5c6991282..3ac69ac6d8c 100644 --- a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json +++ b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json @@ -107,7 +107,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nfrom kfp.dsl import *\nfrom typing import *\n\ndef preprocess(\n # An input parameter of type string.\n message: str,\n # An input parameter of type dict.\n input_dict_parameter: Dict[str, int],\n # An input parameter of type list.\n input_list_parameter: List[str],\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n output_dataset_one: Output[Dataset],\n # A locally accessible filepath for another output artifact of type\n # `Dataset`.\n output_dataset_two_path: OutputPath('Dataset'),\n # A locally accessible filepath for an output parameter of type string.\n output_parameter_path: OutputPath(str),\n # A locally accessible filepath for an output parameter of type bool.\n output_bool_parameter_path: OutputPath(bool),\n # A locally accessible filepath for an output parameter of type dict.\n output_dict_parameter_path: OutputPath(Dict[str, int]),\n # A locally accessible filepath for an output parameter of type list.\n output_list_parameter_path: OutputPath(List[str]),\n):\n \"\"\"Dummy preprocessing step.\"\"\"\n\n # Use Dataset.path to access a local file path for writing.\n # One can also use Dataset.uri to access the actual URI file path.\n with open(output_dataset_one.path, 'w') as f:\n f.write(message)\n\n # OutputPath is used to just pass the local file path of the output artifact\n # to the function.\n with open(output_dataset_two_path, 'w') as f:\n f.write(message)\n\n with open(output_parameter_path, 'w') as f:\n f.write(message)\n\n with open(output_bool_parameter_path, 'w') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-train": { @@ -122,7 +122,7 @@ "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "\nimport json\nimport inspect\nfrom typing import *\n\n# Copyright 2021 The Kubeflow Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Classes for input/output types in KFP SDK.\n\nThese are only compatible with v2 Pipelines.\n\"\"\"\n\nimport os\nfrom typing import Dict, Generic, List, Optional, Type, TypeVar, Union\n\n\n_GCS_LOCAL_MOUNT_PREFIX = '/gcs/'\n_MINIO_LOCAL_MOUNT_PREFIX = '/minio/'\n_S3_LOCAL_MOUNT_PREFIX = '/s3/'\n\n\nclass Artifact(object):\n \"\"\"Generic Artifact class.\n\n This class is meant to represent the metadata around an input or output\n machine-learning Artifact. Artifacts have URIs, which can either be a location\n on disk (or Cloud storage) or some other resource identifier such as\n an API resource name.\n\n Artifacts carry a `metadata` field, which is a dictionary for storing\n metadata related to this artifact.\n \"\"\"\n TYPE_NAME = 'system.Artifact'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n \"\"\"Initializes the Artifact with the given name, URI and metadata.\"\"\"\n self.uri = uri or ''\n self.name = name or ''\n self.metadata = metadata or {}\n\n @property\n def path(self):\n return self._get_path()\n\n @path.setter\n def path(self, path):\n self._set_path(path)\n\n def _get_path(self) -> Optional[str]:\n if self.uri.startswith('gs://'):\n return _GCS_LOCAL_MOUNT_PREFIX + self.uri[len('gs://'):]\n elif self.uri.startswith('minio://'):\n return _MINIO_LOCAL_MOUNT_PREFIX + self.uri[len('minio://'):]\n elif self.uri.startswith('s3://'):\n return _S3_LOCAL_MOUNT_PREFIX + self.uri[len('s3://'):]\n return None\n\n def _set_path(self, path):\n if path.startswith(_GCS_LOCAL_MOUNT_PREFIX):\n path = 'gs://' + path[len(_GCS_LOCAL_MOUNT_PREFIX):]\n elif path.startswith(_MINIO_LOCAL_MOUNT_PREFIX):\n path = 'minio://' + path[len(_MINIO_LOCAL_MOUNT_PREFIX):]\n elif path.startswith(_S3_LOCAL_MOUNT_PREFIX):\n path = 's3://' + path[len(_S3_LOCAL_MOUNT_PREFIX):]\n self.uri = path\n\n\nclass Model(Artifact):\n \"\"\"An artifact representing an ML Model.\"\"\"\n TYPE_NAME = 'system.Model'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n @property\n def framework(self) -> str:\n return self._get_framework()\n\n def _get_framework(self) -> str:\n return self.metadata.get('framework', '')\n\n @framework.setter\n def framework(self, framework: str):\n self._set_framework(framework)\n\n def _set_framework(self, framework: str):\n self.metadata['framework'] = framework\n\n\nclass Dataset(Artifact):\n \"\"\"An artifact representing an ML Dataset.\"\"\"\n TYPE_NAME = 'system.Dataset'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n\nclass Metrics(Artifact):\n \"\"\"Represent a simple base Artifact type to store key-value scalar metrics.\"\"\"\n TYPE_NAME = 'system.Metrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def log_metric(self, metric: str, value: float):\n \"\"\"Sets a custom scalar metric.\n\n Args:\n metric: Metric key\n value: Value of the metric.\n \"\"\"\n self.metadata[metric] = value\n\n\nclass ClassificationMetrics(Artifact):\n \"\"\"Represents Artifact class to store Classification Metrics.\"\"\"\n TYPE_NAME = 'system.ClassificationMetrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def log_roc_data_point(self, fpr: float, tpr: float, threshold: float):\n \"\"\"Logs a single data point in the ROC Curve.\n\n Args:\n fpr: False positive rate value of the data point.\n tpr: True positive rate value of the data point.\n threshold: Threshold value for the data point.\n \"\"\"\n\n roc_reading = {\n 'confidenceThreshold': threshold,\n 'recall': tpr,\n 'falsePositiveRate': fpr\n }\n if 'confidenceMetrics' not in self.metadata.keys():\n self.metadata['confidenceMetrics'] = []\n\n self.metadata['confidenceMetrics'].append(roc_reading)\n\n def log_roc_curve(self, fpr: List[float], tpr: List[float],\n threshold: List[float]):\n \"\"\"Logs an ROC curve.\n\n The list length of fpr, tpr and threshold must be the same.\n\n Args:\n fpr: List of false positive rate values.\n tpr: List of true positive rate values.\n threshold: List of threshold values.\n \"\"\"\n if len(fpr) != len(tpr) or len(fpr) != len(threshold) or len(tpr) != len(\n threshold):\n raise ValueError('Length of fpr, tpr and threshold must be the same. '\n 'Got lengths {}, {} and {} respectively.'.format(\n len(fpr), len(tpr), len(threshold)))\n\n for i in range(len(fpr)):\n self.log_roc_data_point(fpr=fpr[i], tpr=tpr[i], threshold=threshold[i])\n\n def set_confusion_matrix_categories(self, categories: List[str]):\n \"\"\"Stores confusion matrix categories.\n\n Args:\n categories: List of strings specifying the categories.\n \"\"\"\n\n self._categories = []\n annotation_specs = []\n for category in categories:\n annotation_spec = {'displayName': category}\n self._categories.append(category)\n annotation_specs.append(annotation_spec)\n\n self._matrix = []\n for row in range(len(self._categories)):\n self._matrix.append({'row': [0] * len(self._categories)})\n\n self._confusion_matrix = {}\n self._confusion_matrix['annotationSpecs'] = annotation_specs\n self._confusion_matrix['rows'] = self._matrix\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix_row(self, row_category: str, row: List[float]):\n \"\"\"Logs a confusion matrix row.\n\n Args:\n row_category: Category to which the row belongs.\n row: List of integers specifying the values for the row.\n\n Raises:\n ValueError: If row_category is not in the list of categories\n set in set_categories call.\n \"\"\"\n if row_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n if len(row) != len(self._categories):\n raise ValueError('Invalid row. Expected size: {} got: {}'.\\\n format(len(self._categories), len(row)))\n\n self._matrix[self._categories.index(row_category)] = {'row': row}\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix_cell(self, row_category: str, col_category: str,\n value: int):\n \"\"\"Logs a cell in the confusion matrix.\n\n Args:\n row_category: String representing the name of the row category.\n col_category: String representing the name of the column category.\n value: Int value of the cell.\n\n Raises:\n ValueError: If row_category or col_category is not in the list of\n categories set in set_categories.\n \"\"\"\n if row_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n if col_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n self._matrix[self._categories.index(row_category)]['row'][\n self._categories.index(col_category)] = value\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix(self, categories: List[str],\n matrix: List[List[int]]):\n \"\"\"Logs a confusion matrix.\n\n Args:\n categories: List of the category names.\n matrix: Complete confusion matrix.\n\n Raises:\n ValueError: Length of categories does not match number of rows or columns.\n \"\"\"\n self.set_confusion_matrix_categories(categories)\n\n if len(matrix) != len(categories):\n raise ValueError('Invalid matrix: {} passed for categories: {}'.\\\n format(matrix, categories))\n\n for index in range(len(categories)):\n if len(matrix[index]) != len(categories):\n raise ValueError('Invalid matrix: {} passed for categories: {}'.\\\n format(matrix, categories))\n\n self.log_confusion_matrix_row(categories[index], matrix[index])\n\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n\nclass SlicedClassificationMetrics(Artifact):\n \"\"\"Metrics class representing Sliced Classification Metrics.\n\n Similar to ClassificationMetrics clients using this class are expected to use\n log methods of the class to log metrics with the difference being each log\n method takes a slice to associate the ClassificationMetrics.\n\n \"\"\"\n\n TYPE_NAME = 'system.SlicedClassificationMetrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def _upsert_classification_metrics_for_slice(self, slice: str):\n \"\"\"Upserts the classification metrics instance for a slice.\"\"\"\n if slice not in self._sliced_metrics:\n self._sliced_metrics[slice] = ClassificationMetrics()\n\n def _update_metadata(self, slice: str):\n \"\"\"Updates metadata to adhere to the metrics schema.\"\"\"\n self.metadata = {}\n self.metadata['evaluationSlices'] = []\n for slice in self._sliced_metrics.keys():\n slice_metrics = {\n 'slice': slice,\n 'sliceClassificationMetrics': self._sliced_metrics[slice].metadata\n }\n self.metadata['evaluationSlices'].append(slice_metrics)\n\n def log_roc_reading(self, slice: str, threshold: float, tpr: float,\n fpr: float):\n \"\"\"Logs a single data point in the ROC Curve of a slice.\n\n Args:\n slice: String representing slice label.\n threshold: Thresold value for the data point.\n tpr: True positive rate value of the data point.\n fpr: False positive rate value of the data point.\n \"\"\"\n\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_roc_reading(threshold, tpr, fpr)\n self._update_metadata(slice)\n\n def load_roc_readings(self, slice: str, readings: List[List[float]]):\n \"\"\"Supports bulk loading ROC Curve readings for a slice.\n\n Args:\n slice: String representing slice label.\n readings: A 2-D list providing ROC Curve data points.\n The expected order of the data points is: threshold,\n true_positive_rate, false_positive_rate.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].load_roc_readings(readings)\n self._update_metadata(slice)\n\n def set_confusion_matrix_categories(self, slice: str, categories: List[str]):\n \"\"\"Stores confusion matrix categories for a slice..\n\n Categories are stored in the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n categories: List of strings specifying the categories.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].set_confusion_matrix_categories(categories)\n self._update_metadata(slice)\n\n def log_confusion_matrix_row(self, slice: str, row_category: str,\n row: List[int]):\n \"\"\"Logs a confusion matrix row for a slice.\n\n Row is updated on the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n row_category: Category to which the row belongs.\n row: List of integers specifying the values for the row.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_row(row_category, row)\n self._update_metadata(slice)\n\n def log_confusion_matrix_cell(self, slice: str, row_category: str,\n col_category: str, value: int):\n \"\"\"Logs a confusion matrix cell for a slice..\n\n Cell is updated on the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n row_category: String representing the name of the row category.\n col_category: String representing the name of the column category.\n value: Int value of the cell.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_cell(\n row_category, col_category, value)\n self._update_metadata(slice)\n\n def load_confusion_matrix(self, slice: str, categories: List[str],\n matrix: List[List[int]]):\n \"\"\"Supports bulk loading the whole confusion matrix for a slice.\n\n Args:\n slice: String representing slice label.\n categories: List of the category names.\n matrix: Complete confusion matrix.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_cell(categories, matrix)\n self._update_metadata(slice)\n\n\nT = TypeVar('T')\n\n\nclass InputAnnotation():\n \"\"\"Marker type for input artifacts.\"\"\"\n pass\n\n\n\nclass OutputAnnotation():\n \"\"\"Marker type for output artifacts.\"\"\"\n pass\n\n\n# TODO: Use typing.Annotated instead of this hack.\n# With typing.Annotated (Python 3.9+ or typing_extensions package), the\n# following would look like:\n# Input = typing.Annotated[T, InputAnnotation]\n# Output = typing.Annotated[T, OutputAnnotation]\n\n\n# Input represents an Input artifact of type T.\nInput = Union[T, InputAnnotation]\n\n# Output represents an Output artifact of type T.\nOutput = Union[T, OutputAnnotation]\n\n\ndef is_artifact_annotation(typ) -> bool:\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[0] == Union and subs_tree[2] in [InputAnnotation, OutputAnnotation]\n\n if not hasattr(typ, '__origin__'):\n return False\n\n\n if typ.__origin__ != Union and type(typ.__origin__) != type(Union):\n return False\n\n\n if not hasattr(typ, '__args__') or len(typ.__args__) != 2:\n return False\n\n if typ.__args__[1] not in [InputAnnotation, OutputAnnotation]:\n return False\n\n return True\n\ndef is_input_artifact(typ) -> bool:\n \"\"\"Returns True if typ is of type Input[T].\"\"\"\n if not is_artifact_annotation(typ):\n return False\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[2] == InputAnnotation\n\n return typ.__args__[1] == InputAnnotation\n\ndef is_output_artifact(typ) -> bool:\n \"\"\"Returns True if typ is of type Output[T].\"\"\"\n if not is_artifact_annotation(typ):\n return False\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[2] == OutputAnnotation\n\n return typ.__args__[1] == OutputAnnotation\n\ndef get_io_artifact_class(typ):\n if not is_artifact_annotation(typ):\n return None\n if typ == Input or typ == Output:\n return None\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n if len(subs_tree) != 3:\n return None\n return subs_tree[1]\n\n return typ.__args__[0]\n\ndef get_io_artifact_annotation(typ):\n if not is_artifact_annotation(typ):\n return None\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n if len(subs_tree) != 3:\n return None\n return subs_tree[2]\n\n return typ.__args__[1]\n\n\n\n_SCHEMA_TITLE_TO_TYPE: Dict[str, Artifact] = {\n x.TYPE_NAME: x\n for x in [Artifact, Model, Dataset, Metrics, ClassificationMetrics]\n}\n\n\ndef create_runtime_artifact(runtime_artifact: Dict) -> Artifact:\n \"\"\"Creates an Artifact instance from the specified RuntimeArtifact.\n\n Args:\n runtime_artifact: Dictionary representing JSON-encoded RuntimeArtifact.\n \"\"\"\n schema_title = runtime_artifact.get('type', {}).get('schemaTitle', '')\n\n artifact_type = _SCHEMA_TITLE_TO_TYPE.get(schema_title)\n if not artifact_type:\n artifact_type = Artifact\n return artifact_type(\n uri=runtime_artifact.get('uri', ''),\n name=runtime_artifact.get('name', ''),\n metadata=runtime_artifact.get('metadata', {}),\n )\n\nclass InputPath:\n '''When creating component from function, :class:`.InputPath` should be used as function parameter annotation to tell the system to pass the *data file path* to the function instead of passing the actual data.'''\n def __init__(self, type=None):\n self.type = type\n\nclass OutputPath:\n '''When creating component from function, :class:`.OutputPath` should be used as function parameter annotation to tell the system that the function wants to output data by writing it into a file with the given path instead of returning the data from the function.'''\n def __init__(self, type=None):\n self.type = type\n\nclass Executor():\n \"\"\"Executor executes v2-based Python function components.\"\"\"\n\n def __init__(self, executor_input: Dict, function_to_execute: Callable):\n self._func = function_to_execute\n self._input = executor_input\n self._input_artifacts: Dict[str, Artifact] = {}\n self._output_artifacts: Dict[str, Artifact] = {}\n\n for name, artifacts in self._input.get('inputs', {}).get('artifacts',\n {}).items():\n artifacts_list = artifacts.get('artifacts')\n if artifacts_list:\n self._input_artifacts[name] = self._make_input_artifact(\n artifacts_list[0])\n\n for name, artifacts in self._input.get('outputs', {}).get('artifacts',\n {}).items():\n artifacts_list = artifacts.get('artifacts')\n if artifacts_list:\n self._output_artifacts[name] = self._make_output_artifact(\n artifacts_list[0])\n\n self._return_annotation = inspect.signature(self._func).return_annotation\n self._executor_output = {}\n\n @classmethod\n def _make_input_artifact(cls, runtime_artifact: Dict):\n return create_runtime_artifact(runtime_artifact)\n\n @classmethod\n def _make_output_artifact(cls, runtime_artifact: Dict):\n import os\n artifact = create_runtime_artifact(runtime_artifact)\n os.makedirs(os.path.dirname(artifact.path), exist_ok=True)\n return artifact\n\n def _get_input_artifact(self, name: str):\n return self._input_artifacts.get(name)\n\n def _get_output_artifact(self, name: str):\n return self._output_artifacts.get(name)\n\n def _get_input_parameter_value(self, parameter_name: str, parameter_type: Any):\n parameter = self._input.get('inputs', {}).get('parameters',\n {}).get(parameter_name, None)\n if parameter is None:\n return None\n\n if parameter.get('stringValue'):\n if parameter_type == str:\n return parameter['stringValue']\n elif parameter_type == bool:\n # Use `.lower()` so it can also handle 'True' and 'False' (resulted from\n # `str(True)` and `str(False)`, respectively.\n return json.loads(parameter['stringValue'].lower())\n else:\n return json.loads(parameter['stringValue'])\n elif parameter.get('intValue'):\n return int(parameter['intValue'])\n elif parameter.get('doubleValue'):\n return float(parameter['doubleValue'])\n\n def _get_output_parameter_path(self, parameter_name: str):\n parameter_name = self._maybe_strip_path_suffix(parameter_name)\n parameter = self._input.get('outputs',\n {}).get('parameters',\n {}).get(parameter_name, None)\n if parameter is None:\n return None\n\n import os\n path = parameter.get('outputFile', None)\n if path:\n os.makedirs(os.path.dirname(path), exist_ok=True)\n return path\n\n def _get_output_artifact_path(self, artifact_name: str):\n artifact_name = self._maybe_strip_path_suffix(artifact_name)\n output_artifact = self._output_artifacts.get(artifact_name)\n if not output_artifact:\n raise ValueError(\n 'Failed to get output artifact path for artifact name {}'.format(\n artifact_name))\n return output_artifact.path\n\n def _get_input_artifact_path(self, artifact_name: str):\n artifact_name = self._maybe_strip_path_suffix(artifact_name)\n input_artifact = self._input_artifacts.get(artifact_name)\n if not input_artifact:\n raise ValueError(\n 'Failed to get input artifact path for artifact name {}'.format(\n artifact_name))\n return input_artifact.path\n\n def _write_output_parameter_value(self, name: str,\n value: Union[str, int, float, bool, dict,\n list, Dict, List]):\n if type(value) == str:\n output = {'stringValue': value}\n elif type(value) == int:\n output = {'intValue': value}\n elif type(value) == float:\n output = {'doubleValue': value}\n else:\n # For bool, list, dict, List, Dict, json serialize the value.\n output = {'stringValue': json.dumps(value)}\n\n if not self._executor_output.get('parameters'):\n self._executor_output['parameters'] = {}\n\n self._executor_output['parameters'][name] = output\n\n def _write_output_artifact_payload(self, name: str, value: Any):\n path = self._get_output_artifact_path(name)\n with open(path, 'w') as f:\n f.write(str(value))\n\n # TODO: extract to a util\n @classmethod\n def _get_short_type_name(cls, type_name: str) -> str:\n \"\"\"Extracts the short form type name.\n\n This method is used for looking up serializer for a given type.\n\n For example:\n typing.List -> List\n typing.List[int] -> List\n typing.Dict[str, str] -> Dict\n List -> List\n str -> str\n\n Args:\n type_name: The original type name.\n\n Returns:\n The short form type name or the original name if pattern doesn't match.\n \"\"\"\n import re\n match = re.match('(typing\\.)?(?P\\w+)(?:\\[.+\\])?', type_name)\n if match:\n return match.group('type')\n else:\n return type_name\n\n # TODO: merge with type_utils.is_parameter_type\n @classmethod\n def _is_parameter(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return annotation in [str, int, float, bool, dict, list]\n\n # Annotation could be, for instance `typing.Dict[str, str]`, etc.\n return cls._get_short_type_name(str(annotation)) in ['Dict', 'List']\n\n @classmethod\n def _is_artifact(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return issubclass(annotation, Artifact)\n return False\n\n @classmethod\n def _is_named_tuple(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return issubclass(annotation, tuple) and hasattr(\n annotation, '_fields') and hasattr(annotation, '__annotations__')\n return False\n\n def _handle_single_return_value(self, output_name: str, annotation_type: Any,\n return_value: Any):\n if self._is_parameter(annotation_type):\n if type(return_value) != annotation_type:\n raise ValueError(\n 'Function `{}` returned value of type {}; want type {}'.format(\n self._func.__name__, type(return_value), annotation_type))\n self._write_output_parameter_value(output_name, return_value)\n elif self._is_artifact(annotation_type):\n self._write_output_artifact_payload(output_name, return_value)\n else:\n raise RuntimeError(\n 'Unknown return type: {}. Must be one of `str`, `int`, `float`, or a'\n ' subclass of `Artifact`'.format(annotation_type))\n\n def _write_executor_output(self, func_output: Optional[Any] = None):\n if self._output_artifacts:\n self._executor_output['artifacts'] = {}\n\n for name, artifact in self._output_artifacts.items():\n runtime_artifact = {\n 'name': artifact.name,\n 'uri': artifact.uri,\n 'metadata': artifact.metadata,\n }\n artifacts_list = {'artifacts': [runtime_artifact]}\n\n self._executor_output['artifacts'][name] = artifacts_list\n\n if func_output is not None:\n if self._is_parameter(self._return_annotation) or self._is_artifact(\n self._return_annotation):\n # Note: single output is named `Output` in component.yaml.\n self._handle_single_return_value('Output', self._return_annotation,\n func_output)\n elif self._is_named_tuple(self._return_annotation):\n if len(self._return_annotation._fields) != len(func_output):\n raise RuntimeError(\n 'Expected {} return values from function `{}`, got {}'.format(\n len(self._return_annotation._fields), self._func.__name__,\n len(func_output)))\n for i in range(len(self._return_annotation._fields)):\n field = self._return_annotation._fields[i]\n field_type = self._return_annotation.__annotations__[field]\n if type(func_output) == tuple:\n field_value = func_output[i]\n else:\n field_value = getattr(func_output, field)\n self._handle_single_return_value(field, field_type, field_value)\n else:\n raise RuntimeError(\n 'Unknown return type: {}. Must be one of `str`, `int`, `float`, a'\n ' subclass of `Artifact`, or a NamedTuple collection of these types.'\n .format(self._return_annotation))\n\n import os\n os.makedirs(\n os.path.dirname(self._input['outputs']['outputFile']), exist_ok=True)\n with open(self._input['outputs']['outputFile'], 'w') as f:\n f.write(json.dumps(self._executor_output))\n\n def _maybe_strip_path_suffix(self, name) -> str:\n if name.endswith('_path'):\n name = name[0:-len('_path')]\n if name.endswith('_file'):\n name = name[0:-len('_file')]\n return name\n\n def execute(self):\n annotations = inspect.getfullargspec(self._func).annotations\n\n # Function arguments.\n func_kwargs = {}\n\n for k, v in annotations.items():\n if k == 'return':\n continue\n\n if self._is_parameter(v):\n func_kwargs[k] = self._get_input_parameter_value(k, v)\n\n if is_artifact_annotation(v):\n if is_input_artifact(v):\n func_kwargs[k] = self._get_input_artifact(k)\n if is_output_artifact(v):\n func_kwargs[k] = self._get_output_artifact(k)\n\n elif isinstance(v, OutputPath):\n if self._is_parameter(v.type):\n func_kwargs[k] = self._get_output_parameter_path(k)\n else:\n func_kwargs[k] = self._get_output_artifact_path(k)\n elif isinstance(v, InputPath):\n func_kwargs[k] = self._get_input_artifact_path(k)\n\n result = self._func(**func_kwargs)\n self._write_executor_output(result)\n\n\ndef train(\n # Use InputPath to get a locally accessible path for the input artifact\n # of type `Dataset`.\n dataset_one_path: InputPath('Dataset'),\n # Use Input[T] to get a metadata-rich handle to the input artifact\n # of type `Dataset`.\n dataset_two: Input[Dataset],\n # An input parameter of type string.\n message: str,\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n model: Output[Model],\n # An input parameter of type bool.\n input_bool: bool,\n # An input parameter of type dict.\n input_dict: Dict[str, int],\n # An input parameter of type List[str].\n input_list: List[str],\n # An input parameter of type int with a default value.\n num_steps: int = 100,\n):\n \"\"\"Dummy Training step\"\"\"\n with open(dataset_one_path, 'r') as input_file:\n dataset_one_contents = input_file.read()\n\n with open(dataset_two.path, 'r') as input_file:\n dataset_two_contents = input_file.read()\n\n line = (f'dataset_one_contents: {dataset_one_contents} || '\n f'dataset_two_contents: {dataset_two_contents} || '\n f'message: {message} || '\n f'input_bool: {input_bool}, type {type(input_bool)} || '\n f'input_dict: {input_dict}, type {type(input_dict)} || '\n f'input_list: {input_list}, type {type(input_list)} \\n')\n\n with open(model.path, 'w') as output_file:\n for i in range(num_steps):\n output_file.write('Step {}\\n{}\\n=====\\n'.format(i, line))\n\n # model is an instance of Model artifact, which has a .metadata dictionary\n # to store arbitrary metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\n\ndef executor_main():\n import argparse\n import json\n\n parser = argparse.ArgumentParser(description='Process some integers.')\n parser.add_argument('--executor_input', type=str)\n parser.add_argument('--function_to_execute', type=str)\n\n args, _ = parser.parse_known_args()\n executor_input = json.loads(args.executor_input)\n function_to_execute = globals()[args.function_to_execute]\n\n executor = Executor(executor_input=executor_input,\n function_to_execute=function_to_execute)\n\n executor.execute()\n\n\nif __name__ == '__main__':\n executor_main()\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml index 58b6df51e79..3d884289b7b 100644 --- a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml +++ b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml @@ -6,7 +6,7 @@ deploymentSpec: executors: exec-preprocess: container: - image: python:3.9 + image: python:3.11 args: - --executor_input - '{{$}}' @@ -105,7 +105,7 @@ deploymentSpec: \ Model artifact, which has a .metadata dictionary\n # to store arbitrary\ \ metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\ \n" - image: python:3.9 + image: python:3.11 components: comp-preprocess: inputDefinitions: diff --git a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json index 6f4f6f7a7ed..7f963eea75e 100644 --- a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json +++ b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json @@ -929,7 +929,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -> list:\n return [\n {\n 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n },\n ]\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-args-generator-op-2": { @@ -949,7 +949,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -> list:\n return [\n {\n 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n },\n ]\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-flip-coin-op": { @@ -969,7 +969,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n return result\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-struct": { @@ -989,7 +989,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text": { @@ -1009,7 +1009,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-2": { @@ -1029,7 +1029,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-3": { @@ -1049,7 +1049,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-4": { @@ -1069,7 +1069,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-5": { @@ -1089,7 +1089,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-6": { @@ -1109,7 +1109,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-7": { @@ -1129,7 +1129,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-8": { @@ -1149,7 +1149,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-print-text-9": { @@ -1169,7 +1169,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml index f12bc94a805..99b53f660d9 100644 --- a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml +++ b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml @@ -588,7 +588,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-args-generator-op-2: container: args: @@ -617,7 +617,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op: container: args: @@ -645,7 +645,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-struct: container: args: @@ -671,7 +671,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text: container: args: @@ -698,7 +698,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-2: container: args: @@ -725,7 +725,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-3: container: args: @@ -752,7 +752,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-4: container: args: @@ -779,7 +779,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-5: container: args: @@ -806,7 +806,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-6: container: args: @@ -833,7 +833,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-7: container: args: @@ -860,7 +860,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-8: container: args: @@ -887,7 +887,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-9: container: args: @@ -914,7 +914,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-loops-and-conditions-multi-layers root: diff --git a/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json b/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json index 4bbf2bcb0df..770764dca26 100644 --- a/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json +++ b/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json @@ -48,7 +48,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_params(\n int_param: int = 1234, \n double_param: float = 56.78, \n string_param: str = 'lorem ipsum', \n bool_param: bool = True, \n list_string_param: List[str] = ['lorem', 'ipsum'], \n list_int_param: List[int] = [123, 456, 789], \n struct_param: Dict[str, int] = { 'key_1': 12345, 'key_2': 6789 }):\n print(\"int_param: \", int_param)\n print(\"double_param: \", double_param)\n print(\"string_param: \", string_param)\n print(\"bool_param: \", bool_param)\n print(\"list_string_param: \", list_string_param)\n print(\"list_int_param: \", list_int_param)\n print(\"struct_param: \", struct_param)\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json index 00e235989eb..d58cccaa4cd 100644 --- a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json +++ b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json @@ -327,7 +327,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef convert_csv_to_apache_parquet(\n data_path,\n output_data_path,\n):\n '''Converts CSV table to Apache Parquet.\n\n [Apache Parquet](https://parquet.apache.org/)\n\n Annotations:\n author: Alexey Volkov \n '''\n from pyarrow import csv, parquet\n\n table = csv.read_csv(data_path)\n parquet.write_table(table, output_data_path)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Convert csv to apache parquet', description='Converts CSV table to Apache Parquet.\\n\\n [Apache Parquet](https://parquet.apache.org/)\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--output-data\", dest=\"output_data_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = convert_csv_to_apache_parquet(**_parsed_args)\n\n_output_serializers = [\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-predict": { @@ -351,7 +351,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path, # Also supports LibSVM\n model_path,\n predictions_path,\n label_column = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in CSV format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column: Column containing the label data.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n data_path,\n )\n\n if label_column is not None:\n df = df.drop(columns=[df.columns[label_column]])\n\n testing_data = xgboost.DMatrix(\n data=df,\n )\n\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(testing_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in CSV format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column: Column containing the label data.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-predict-2": { @@ -375,7 +375,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path,\n model_path,\n predictions_path,\n label_column_name = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in Apache Parquet format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(data_path)\n if label_column_name:\n df = df.drop(columns=[label_column_name])\n\n evaluation_data = xgboost.DMatrix(\n data=df,\n )\n\n # Training\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(evaluation_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in Apache Parquet format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-predict-3": { @@ -399,7 +399,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path,\n model_path,\n predictions_path,\n label_column_name = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in Apache Parquet format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(data_path)\n if label_column_name:\n df = df.drop(columns=[label_column_name])\n\n evaluation_data = xgboost.DMatrix(\n data=df,\n )\n\n # Training\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(evaluation_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in Apache Parquet format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-predict-4": { @@ -423,7 +423,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path, # Also supports LibSVM\n model_path,\n predictions_path,\n label_column = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in CSV format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column: Column containing the label data.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n data_path,\n )\n\n if label_column is not None:\n df = df.drop(columns=[df.columns[label_column]])\n\n testing_data = xgboost.DMatrix(\n data=df,\n )\n\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(testing_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in CSV format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column: Column containing the label data.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-train": { @@ -459,7 +459,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_train(\n training_data_path, # Also supports LibSVM\n model_path,\n model_config_path,\n starting_model_path = None,\n\n label_column = 0,\n num_iterations = 10,\n booster_params = None,\n\n # Booster parameters\n objective = 'reg:squarederror',\n booster = 'gbtree',\n learning_rate = 0.3,\n min_split_loss = 0,\n max_depth = 6,\n):\n '''Train an XGBoost model.\n\n Args:\n training_data_path: Path for the training data in CSV format.\n model_path: Output path for the trained model in binary XGBoost format.\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\n starting_model_path: Path for the existing trained model to start from.\n label_column: Column containing the label data.\n num_boost_rounds: Number of boosting iterations.\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\n objective: The learning task and the corresponding learning objective.\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n The most common values are:\n \"reg:squarederror\" - Regression with squared loss (default).\n \"reg:logistic\" - Logistic regression.\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\n\n Annotations:\n author: Alexey Volkov \n '''\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n training_data_path,\n )\n\n training_data = xgboost.DMatrix(\n data=df.drop(columns=[df.columns[label_column]]),\n label=df[df.columns[label_column]],\n )\n\n booster_params = booster_params or {}\n booster_params.setdefault('objective', objective)\n booster_params.setdefault('booster', booster)\n booster_params.setdefault('learning_rate', learning_rate)\n booster_params.setdefault('min_split_loss', min_split_loss)\n booster_params.setdefault('max_depth', max_depth)\n\n starting_model = None\n if starting_model_path:\n starting_model = xgboost.Booster(model_file=starting_model_path)\n\n model = xgboost.train(\n params=booster_params,\n dtrain=training_data,\n num_boost_round=num_iterations,\n xgb_model=starting_model\n )\n\n # Saving the model in binary format\n model.save_model(model_path)\n\n model_config_str = model.save_config()\n with open(model_config_path, 'w') as model_config_file:\n model_config_file.write(model_config_str)\n\nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost train', description='Train an XGBoost model.\\n\\n Args:\\n training_data_path: Path for the training data in CSV format.\\n model_path: Output path for the trained model in binary XGBoost format.\\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\\n starting_model_path: Path for the existing trained model to start from.\\n label_column: Column containing the label data.\\n num_boost_rounds: Number of boosting iterations.\\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\n objective: The learning task and the corresponding learning objective.\\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\n The most common values are:\\n \"reg:squarederror\" - Regression with squared loss (default).\\n \"reg:logistic\" - Logistic regression.\\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--training-data\", dest=\"training_data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--starting-model\", dest=\"starting_model_path\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--num-iterations\", dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\", dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\", dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\", dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\", dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_train(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-xgboost-train-2": { @@ -495,7 +495,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_train(\n training_data_path,\n model_path,\n model_config_path,\n label_column_name,\n\n starting_model_path = None,\n\n num_iterations = 10,\n booster_params = None,\n\n # Booster parameters\n objective = 'reg:squarederror',\n booster = 'gbtree',\n learning_rate = 0.3,\n min_split_loss = 0,\n max_depth = 6,\n):\n '''Train an XGBoost model.\n\n Args:\n training_data_path: Path for the training data in Apache Parquet format.\n model_path: Output path for the trained model in binary XGBoost format.\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\n starting_model_path: Path for the existing trained model to start from.\n label_column_name: Name of the column containing the label data.\n num_boost_rounds: Number of boosting iterations.\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\n objective: The learning task and the corresponding learning objective.\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n The most common values are:\n \"reg:squarederror\" - Regression with squared loss (default).\n \"reg:logistic\" - Logistic regression.\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\n\n Annotations:\n author: Alexey Volkov \n '''\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(training_data_path)\n training_data = xgboost.DMatrix(\n data=df.drop(columns=[label_column_name]),\n label=df[[label_column_name]],\n )\n # Training\n booster_params = booster_params or {}\n booster_params.setdefault('objective', objective)\n booster_params.setdefault('booster', booster)\n booster_params.setdefault('learning_rate', learning_rate)\n booster_params.setdefault('min_split_loss', min_split_loss)\n booster_params.setdefault('max_depth', max_depth)\n\n starting_model = None\n if starting_model_path:\n starting_model = xgboost.Booster(model_file=starting_model_path)\n\n model = xgboost.train(\n params=booster_params,\n dtrain=training_data,\n num_boost_round=num_iterations,\n xgb_model=starting_model\n )\n\n # Saving the model in binary format\n model.save_model(model_path)\n\n model_config_str = model.save_config()\n with open(model_config_path, 'w') as model_config_file:\n model_config_file.write(model_config_str)\n\nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost train', description='Train an XGBoost model.\\n\\n Args:\\n training_data_path: Path for the training data in Apache Parquet format.\\n model_path: Output path for the trained model in binary XGBoost format.\\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\\n starting_model_path: Path for the existing trained model to start from.\\n label_column_name: Name of the column containing the label data.\\n num_boost_rounds: Number of boosting iterations.\\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\n objective: The learning task and the corresponding learning objective.\\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\n The most common values are:\\n \"reg:squarederror\" - Regression with squared loss (default).\\n \"reg:logistic\" - Logistic regression.\\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--training-data\", dest=\"training_data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--starting-model\", dest=\"starting_model_path\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--num-iterations\", dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\", dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\", dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\", dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\", dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_train(**_parsed_args)\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml index c5612c190d0..c0bec7ff5ac 100644 --- a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml +++ b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml @@ -253,7 +253,7 @@ deploymentSpec: \ try:\n os.makedirs(os.path.dirname(output_file))\n except\ \ OSError:\n pass\n with open(output_file, 'w') as f:\n \ \ f.write(_output_serializers[idx](_outputs[idx]))\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict: container: args: @@ -304,7 +304,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-2: container: args: @@ -358,7 +358,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-3: container: args: @@ -412,7 +412,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-4: container: args: @@ -463,7 +463,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train: container: args: @@ -571,7 +571,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train-2: container: args: @@ -679,7 +679,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: xgboost-sample-pipeline root: diff --git a/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx b/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx index d0948760c7a..be456910cc9 100644 --- a/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx +++ b/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx @@ -76,7 +76,7 @@ describe('StaticNodeDetailsV2', () => { expect(screen.getAllByText('STRING').length).toEqual(2); screen.getByText('Image'); - screen.getByText('python:3.9'); + screen.getByText('python:3.11'); screen.getByText('Command'); expect(screen.getAllByText('sh').length).toEqual(2); @@ -131,7 +131,7 @@ describe('StaticNodeDetailsV2', () => { expect(screen.getAllByText('STRING').length).toEqual(1); screen.getByText('Image'); - screen.getByText('python:3.9'); + screen.getByText('python:3.11'); screen.getByText('Command'); expect(screen.getAllByText('sh').length).toEqual(2); diff --git a/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml b/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml index fc07ad77ade..303a52d775d 100644 --- a/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml +++ b/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml @@ -76,7 +76,7 @@ deploymentSpec: \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ \ content = file.read()\n print(content)\n return content\n\ \n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -111,7 +111,7 @@ deploymentSpec: \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ \ as file:\n content = file.read()\n print(content)\n return\ \ content\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: diff --git a/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml b/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml index c32ada3c000..1f3f41515b5 100644 --- a/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml +++ b/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml @@ -6,7 +6,7 @@ deploymentSpec: executors: exec-preprocess: container: - image: python:3.9 + image: python:3.11 args: - --executor_input - '{{$}}' @@ -105,7 +105,7 @@ deploymentSpec: \ Model artifact, which has a .metadata dictionary\n # to store arbitrary\ \ metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\ \n" - image: python:3.9 + image: python:3.11 components: comp-preprocess: inputDefinitions: diff --git a/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml b/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml index f12bc94a805..99b53f660d9 100644 --- a/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml +++ b/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml @@ -588,7 +588,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-args-generator-op-2: container: args: @@ -617,7 +617,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op: container: args: @@ -645,7 +645,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-struct: container: args: @@ -671,7 +671,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text: container: args: @@ -698,7 +698,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-2: container: args: @@ -725,7 +725,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-3: container: args: @@ -752,7 +752,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-4: container: args: @@ -779,7 +779,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-5: container: args: @@ -806,7 +806,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-6: container: args: @@ -833,7 +833,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-7: container: args: @@ -860,7 +860,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-8: container: args: @@ -887,7 +887,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-9: container: args: @@ -914,7 +914,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-loops-and-conditions-multi-layers root: diff --git a/frontend/src/data/test/xgboost_sample_pipeline.yaml b/frontend/src/data/test/xgboost_sample_pipeline.yaml index e7a54104173..1a3a4eac6b6 100644 --- a/frontend/src/data/test/xgboost_sample_pipeline.yaml +++ b/frontend/src/data/test/xgboost_sample_pipeline.yaml @@ -253,7 +253,7 @@ deploymentSpec: \ try:\n os.makedirs(os.path.dirname(output_file))\n except\ \ OSError:\n pass\n with open(output_file, 'w') as f:\n \ \ f.write(_output_serializers[idx](_outputs[idx]))\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict: container: args: @@ -304,7 +304,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-2: container: args: @@ -358,7 +358,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-3: container: args: @@ -412,7 +412,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-predict-4: container: args: @@ -463,7 +463,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train: container: args: @@ -571,7 +571,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 exec-xgboost-train-2: container: args: @@ -679,7 +679,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: xgboost-sample-pipeline root: diff --git a/frontend/src/lib/v2/WorkflowUtils.test.ts b/frontend/src/lib/v2/WorkflowUtils.test.ts index d3d5f382634..b9894da130c 100644 --- a/frontend/src/lib/v2/WorkflowUtils.test.ts +++ b/frontend/src/lib/v2/WorkflowUtils.test.ts @@ -143,7 +143,7 @@ PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-scr "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef preprocess(\n # An input parameter of type string.\n message: str,\n # An input parameter of type dict.\n input_dict_parameter: Dict[str, int],\n # An input parameter of type list.\n input_list_parameter: List[str],\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n output_dataset_one: Output[Dataset],\n # A locally accessible filepath for another output artifact of type\n # `Dataset`.\n output_dataset_two_path: OutputPath('Dataset'),\n # A locally accessible filepath for an output parameter of type string.\n output_parameter_path: OutputPath(str),\n # A locally accessible filepath for an output parameter of type bool.\n output_bool_parameter_path: OutputPath(bool),\n # A locally accessible filepath for an output parameter of type dict.\n output_dict_parameter_path: OutputPath(Dict[str, int]),\n # A locally accessible filepath for an output parameter of type list.\n output_list_parameter_path: OutputPath(List[str]),\n):\n \"\"\"Dummy preprocessing step.\"\"\"\n\n # Use Dataset.path to access a local file path for writing.\n # One can also use Dataset.uri to access the actual URI file path.\n with open(output_dataset_one.path, 'w') as f:\n f.write(message)\n\n # OutputPath is used to just pass the local file path of the output artifact\n # to the function.\n with open(output_dataset_two_path, 'w') as f:\n f.write(message)\n\n with open(output_parameter_path, 'w') as f:\n f.write(message)\n\n with open(output_bool_parameter_path, 'w') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\n", ], env: [], - image: 'python:3.9', + image: 'python:3.11', lifecycle: undefined, resources: undefined, }); @@ -174,7 +174,7 @@ PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-scr "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef producer() -> str:\n with open('/data/file.txt', 'w') as file:\n file.write('Hello world')\n with open('/data/file.txt', 'r') as file:\n content = file.read()\n print(content)\n return content\n\n", ], env: [], - image: 'python:3.9', + image: 'python:3.11', lifecycle: undefined, resources: undefined, }); diff --git a/hack/update-requirements.sh b/hack/update-requirements.sh index 382e95a82de..f192b74a091 100755 --- a/hack/update-requirements.sh +++ b/hack/update-requirements.sh @@ -16,7 +16,7 @@ # Usage: ./update_requirements.sh requirements.txt set -euo pipefail -IMAGE=${1:-"python:3.9"} +IMAGE=${1:-"python:3.11"} docker run -i --rm --entrypoint "" "$IMAGE" sh -c ' python3 -m pip install pip setuptools --quiet --upgrade python3 -m pip install pip-tools==6.14.0 --quiet diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml index 03250adee61..4cfa55cadcb 100644 --- a/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: @@ -55,9 +55,10 @@ platforms: exec-comp: configMapAsEnv: - configMapName: my-cm - configNameParameter: + configMapNameParameter: runtimeValue: constant: my-cm keyToEnv: - configMapKey: foo envVar: CONFIG_MAP_VAR + optional: false diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml index a9507e8e406..586f0899f66 100644 --- a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: @@ -55,7 +55,7 @@ platforms: exec-comp: configMapAsVolume: - configMapName: my-cm - configNameParameter: + configMapNameParameter: runtimeValue: constant: my-cm mountPath: /mnt/my_vol diff --git a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_dynamic_pvc.yaml b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_dynamic_pvc.yaml index 93f568e1800..a31c8abca7a 100644 --- a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_dynamic_pvc.yaml +++ b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_dynamic_pvc.yaml @@ -96,7 +96,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -113,7 +113,7 @@ deploymentSpec: \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ \ content = file.read()\n print(content)\n return content\n\ \n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -132,7 +132,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -150,7 +150,7 @@ deploymentSpec: \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ \ as file:\n content = file.read()\n print(content)\n return\ \ content\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -214,7 +214,7 @@ root: taskInfo: name: producer schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc.yaml b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc.yaml index 18060a50157..ffa4587f4f4 100644 --- a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc.yaml +++ b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc.yaml @@ -86,7 +86,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -101,7 +101,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -161,7 +161,7 @@ root: taskInfo: name: deletepvc schemaVersion: 2.1.0 -sdkVersion: kfp-2.13.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc_from_task_output.yaml b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc_from_task_output.yaml index f14df06e469..01787d7067b 100644 --- a/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc_from_task_output.yaml +++ b/kubernetes_platform/python/test/snapshot/data/create_mount_delete_existing_pvc_from_task_output.yaml @@ -92,7 +92,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -107,7 +107,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -126,7 +126,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -142,7 +142,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef get_pvc_name() -> str:\n return 'static-pvc-name-cmdepfto'\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -206,7 +206,7 @@ root: taskInfo: name: get-pvc-name schemaVersion: 2.1.0 -sdkVersion: kfp-2.13.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml index fe36d5ace76..c1268279c0d 100644 --- a/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml +++ b/kubernetes_platform/python/test/snapshot/data/empty_dir_mounts.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/field_path_as_env.yaml b/kubernetes_platform/python/test/snapshot/data/field_path_as_env.yaml index bf7623b9899..da3034a1824 100644 --- a/kubernetes_platform/python/test/snapshot/data/field_path_as_env.yaml +++ b/kubernetes_platform/python/test/snapshot/data/field_path_as_env.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/general_ephemeral_volume.yaml b/kubernetes_platform/python/test/snapshot/data/general_ephemeral_volume.yaml index 4fe7937402e..c6dc96588bd 100644 --- a/kubernetes_platform/python/test/snapshot/data/general_ephemeral_volume.yaml +++ b/kubernetes_platform/python/test/snapshot/data/general_ephemeral_volume.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/image_pull_secrets.yaml b/kubernetes_platform/python/test/snapshot/data/image_pull_secrets.yaml index 9d5a99a6d8b..9e4f6131b5b 100644 --- a/kubernetes_platform/python/test/snapshot/data/image_pull_secrets.yaml +++ b/kubernetes_platform/python/test/snapshot/data/image_pull_secrets.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: @@ -55,3 +55,6 @@ platforms: exec-comp: imagePullSecret: - secretName: my-secret + secretNameParameter: + runtimeValue: + constant: my-secret diff --git a/kubernetes_platform/python/test/snapshot/data/node_selector.yaml b/kubernetes_platform/python/test/snapshot/data/node_selector.yaml index 566ceae62f3..833978c5c53 100644 --- a/kubernetes_platform/python/test/snapshot/data/node_selector.yaml +++ b/kubernetes_platform/python/test/snapshot/data/node_selector.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/nodeaffinity.yaml b/kubernetes_platform/python/test/snapshot/data/nodeaffinity.yaml index db8e0218a4b..9bf6336caba 100644 --- a/kubernetes_platform/python/test/snapshot/data/nodeaffinity.yaml +++ b/kubernetes_platform/python/test/snapshot/data/nodeaffinity.yaml @@ -38,7 +38,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_hello_with_empty_json():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-print-hello-with-json-affinity: container: args: @@ -66,7 +66,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_hello_with_json_affinity():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-print-hello-with-json-affinity-2: container: args: @@ -94,7 +94,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_hello_with_json_affinity():\n pass\n\n" - image: python:3.9 + image: python:3.11 exec-print-hello-with-preferred-affinity: container: args: @@ -122,7 +122,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_hello_with_preferred_affinity():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: test-node-affinity root: diff --git a/kubernetes_platform/python/test/snapshot/data/secret_as_env.yaml b/kubernetes_platform/python/test/snapshot/data/secret_as_env.yaml index f6e702d2686..c5a7cb4ba33 100644 --- a/kubernetes_platform/python/test/snapshot/data/secret_as_env.yaml +++ b/kubernetes_platform/python/test/snapshot/data/secret_as_env.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: @@ -57,6 +57,7 @@ platforms: - keyToEnv: - envVar: SECRET_VAR secretKey: password + optional: false secretName: my-secret secretNameParameter: runtimeValue: diff --git a/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml index 75cf8deb14f..1685e76c5ae 100644 --- a/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml +++ b/kubernetes_platform/python/test/snapshot/data/secret_as_vol.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/timeout.yaml b/kubernetes_platform/python/test/snapshot/data/timeout.yaml index 141fea715fa..4a4eaf1f617 100644 --- a/kubernetes_platform/python/test/snapshot/data/timeout.yaml +++ b/kubernetes_platform/python/test/snapshot/data/timeout.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/kubernetes_platform/python/test/snapshot/data/toleration.yaml b/kubernetes_platform/python/test/snapshot/data/toleration.yaml index 6f797aa39d2..023da7a52cb 100644 --- a/kubernetes_platform/python/test/snapshot/data/toleration.yaml +++ b/kubernetes_platform/python/test/snapshot/data/toleration.yaml @@ -17,7 +17,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: @@ -46,7 +46,7 @@ root: taskInfo: name: comp schemaVersion: 2.1.0 -sdkVersion: kfp-2.11.0 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py index 72311fc78f9..f9569fed687 100644 --- a/sdk/python/kfp/__init__.py +++ b/sdk/python/kfp/__init__.py @@ -26,7 +26,7 @@ if sys.version_info < (3, 9): warnings.warn( - ('KFP will drop support for Python 3.9 on Oct 1, 2025. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.10. See https://devguide.python.org/versions/ for more details.' + ('KFP will drop support for Python 3.9 on October 1, 2026. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.10. See https://devguide.python.org/versions/ for more details.' ), FutureWarning, stacklevel=2, diff --git a/sdk/python/kfp/cli/compile_test.py b/sdk/python/kfp/cli/compile_test.py index c11d43e2a4e..150f87af42c 100644 --- a/sdk/python/kfp/cli/compile_test.py +++ b/sdk/python/kfp/cli/compile_test.py @@ -29,7 +29,7 @@ def my_comp(): @dsl.container_component def my_container_comp(): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo', 'hello world'], args=[], ) @@ -55,7 +55,7 @@ def my_comp(): @dsl.container_component def my_container_comp(): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo', 'hello world'], args=[], ) diff --git a/sdk/python/kfp/cli/component_test.py b/sdk/python/kfp/cli/component_test.py index 67c257ecb52..fb0af92c79d 100644 --- a/sdk/python/kfp/cli/component_test.py +++ b/sdk/python/kfp/cli/component_test.py @@ -494,7 +494,7 @@ def test_docker_file_is_created_correctly(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -523,7 +523,7 @@ def test_docker_file_is_created_correctly_with_one_url(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -554,7 +554,7 @@ def test_docker_file_is_created_correctly_with_two_urls(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -597,7 +597,7 @@ def test_existing_dockerfile_can_be_overwritten(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -636,7 +636,7 @@ def test_dockerfile_can_contain_custom_kfp_package(self): file_start = textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -665,7 +665,7 @@ def test_docker_file_is_created_one_trusted_host(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt @@ -695,7 +695,7 @@ def test_docker_file_is_created_two_trusted_host(self): textwrap.dedent('''\ # Generated by KFP. - FROM python:3.9 + FROM python:3.11 WORKDIR /usr/local/src/kfp/components COPY runtime-requirements.txt runtime-requirements.txt diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index 012f9eb2b2c..84a9b099546 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -700,7 +700,7 @@ def test_use_task_final_status_in_non_exit_op_yaml(self): - {name: message, type: PipelineTaskFinalStatus} implementation: container: - image: python:3.9 + image: python:3.11 command: - echo - {inputValue: message} @@ -975,14 +975,14 @@ def orchestrator_pipeline(): def test_pipeline_with_parameterized_container_image(self): with tempfile.TemporaryDirectory() as tmpdir: - @dsl.component(base_image='docker.io/python:3.9.17') + @dsl.component(base_image='docker.io/python:3.11.17') def empty_component(): pass @dsl.pipeline() def simple_pipeline(img: str): task = empty_component() - # overwrite base_image="docker.io/python:3.9.17" + # overwrite base_image="docker.io/python:3.11.17" task.set_container_image(img) output_yaml = os.path.join(tmpdir, 'result.yaml') @@ -1010,14 +1010,14 @@ def simple_pipeline(img: str): def test_pipeline_with_constant_container_image(self): with tempfile.TemporaryDirectory() as tmpdir: - @dsl.component(base_image='docker.io/python:3.9.17') + @dsl.component(base_image='docker.io/python:3.11.17') def empty_component(): pass @dsl.pipeline() def simple_pipeline(): task = empty_component() - # overwrite base_image="docker.io/python:3.9.17" + # overwrite base_image="docker.io/python:3.11.17" task.set_container_image('constant-value') output_yaml = os.path.join(tmpdir, 'result.yaml') @@ -1504,7 +1504,7 @@ def test_compile_container_component_simple(self): def hello_world_container() -> dsl.ContainerSpec: """Hello world component.""" return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo', 'hello world'], args=[], ) @@ -1527,7 +1527,7 @@ def test_compile_container_with_simple_io(self): @dsl.container_component def container_simple_io(text: str, output_path: dsl.OutputPath(str)): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['my_program', text], args=['--output_path', output_path]) @@ -2574,7 +2574,7 @@ def my_component(string: str, model: bool) -> str: def my_container_component(text: str, output_path: OutputPath(str)): """component description.""" return ContainerSpec( - image='python:3.9', + image='python:3.11', command=['my_program', text], args=['--output_path', output_path]) diff --git a/sdk/python/kfp/components/load_yaml_utilities_test.py b/sdk/python/kfp/components/load_yaml_utilities_test.py index 8130bb2e851..a3cfb82b602 100644 --- a/sdk/python/kfp/components/load_yaml_utilities_test.py +++ b/sdk/python/kfp/components/load_yaml_utilities_test.py @@ -119,9 +119,10 @@ def test_load_component_from_url(self): {'Output': structures.OutputSpec(type='String')}) self.assertEqual(component._component_inputs, {'value'}) self.assertEqual(component.name, 'identity') - self.assertEqual( - component.component_spec.implementation.container.image, - 'python:3.9') + # TODO: uncomment once PR #12383 is merged since this is checking against a version on master + # self.assertEqual( + # component.component_spec.implementation.container.image, + # 'python:3.11') if __name__ == '__main__': diff --git a/sdk/python/kfp/dsl/component_decorator.py b/sdk/python/kfp/dsl/component_decorator.py index 507f11fc9eb..e2f71aaa678 100644 --- a/sdk/python/kfp/dsl/component_decorator.py +++ b/sdk/python/kfp/dsl/component_decorator.py @@ -116,7 +116,7 @@ def my_function_one(input: str, output: Output[Model]): ... @dsl.component( - base_image='python:3.9', + base_image='python:3.11', output_component_file='my_function.yaml' ) def my_function_two(input: Input[Mode])): diff --git a/sdk/python/kfp/dsl/component_decorator_test.py b/sdk/python/kfp/dsl/component_decorator_test.py index 4b51de638f4..6e622b91844 100644 --- a/sdk/python/kfp/dsl/component_decorator_test.py +++ b/sdk/python/kfp/dsl/component_decorator_test.py @@ -35,7 +35,7 @@ def hello_world(text: str) -> str: def test_as_decorator_syntactic_sugar_some_args(self): - @component(base_image='python:3.9') + @component(base_image='python:3.11') def hello_world(text: str) -> str: """Hello world component.""" return text @@ -52,7 +52,7 @@ def comp(text: str) -> str: def test_some_args(self): - @component(base_image='python:3.9') + @component(base_image='python:3.11') def comp(text: str) -> str: return text diff --git a/sdk/python/kfp/dsl/component_factory.py b/sdk/python/kfp/dsl/component_factory.py index 7822e74309b..640dee24147 100644 --- a/sdk/python/kfp/dsl/component_factory.py +++ b/sdk/python/kfp/dsl/component_factory.py @@ -43,7 +43,7 @@ from kfp.dsl.types import type_annotations from kfp.dsl.types import type_utils -_DEFAULT_BASE_IMAGE = 'python:3.9' +_DEFAULT_BASE_IMAGE = 'python:3.11' SINGLE_OUTPUT_NAME = 'Output' @@ -766,7 +766,7 @@ def create_component_from_func( if base_image is None: base_image = _DEFAULT_BASE_IMAGE warnings.warn( - ("The default base_image used by the @dsl.component decorator will switch from 'python:3.9' to 'python:3.10' on Oct 1, 2025. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3.10." + ("The default base_image used by the @dsl.component decorator will switch from 'python:3.11' to 'python:3.12' on Oct 1, 2027. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3.12." ), FutureWarning, stacklevel=3, diff --git a/sdk/python/kfp/dsl/component_factory_test.py b/sdk/python/kfp/dsl/component_factory_test.py index 9ffaa24b3aa..a9db60f9f42 100644 --- a/sdk/python/kfp/dsl/component_factory_test.py +++ b/sdk/python/kfp/dsl/component_factory_test.py @@ -367,7 +367,7 @@ def test_default_base_image(self): with self.assertWarnsRegex( FutureWarning, - r"The default base_image used by the @dsl\.component decorator will switch from 'python:3\.9' to 'python:3\.10' on Oct 1, 2025\. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3\.10\." + r"The default base_image used by the @dsl\.component decorator will switch from 'python:3\.11' to 'python:3\.12' on Oct 1, 2027\. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3\.12\." ): @dsl.component diff --git a/sdk/python/kfp/dsl/container_component_decorator_test.py b/sdk/python/kfp/dsl/container_component_decorator_test.py index b690fd5e375..247776ec444 100644 --- a/sdk/python/kfp/dsl/container_component_decorator_test.py +++ b/sdk/python/kfp/dsl/container_component_decorator_test.py @@ -30,7 +30,7 @@ def test_func_with_no_arg(self): def hello_world() -> dsl.ContainerSpec: """Hello world component.""" return dsl.ContainerSpec( - image='python3.9', + image='python3.11', command=['echo', 'hello world'], args=[], ) @@ -47,7 +47,7 @@ def hello_world_io( text_output_path: dsl.OutputPath(str)) -> dsl.ContainerSpec: """Hello world component with input and output.""" return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo'], args=['--text', text, '--output_path', text_output_path]) diff --git a/sdk/python/kfp/dsl/pipeline_task.py b/sdk/python/kfp/dsl/pipeline_task.py index eda1407ed6d..1975dba22fd 100644 --- a/sdk/python/kfp/dsl/pipeline_task.py +++ b/sdk/python/kfp/dsl/pipeline_task.py @@ -651,7 +651,7 @@ def set_container_image( Pipeline Parameters or outputs from previous tasks, which are resolved at runtime. Args: - name: The container image name as a static string (e.g., "python:3.9-alpine") + name: The container image name as a static string (e.g., "python:3.11-alpine") or a dynamic reference (e.g., a PipelineParameter instance or a task output like `task.outputs['image_name']`). diff --git a/sdk/python/kfp/dsl/placeholders.py b/sdk/python/kfp/dsl/placeholders.py index d8b0dfb9446..194dcd23a5f 100644 --- a/sdk/python/kfp/dsl/placeholders.py +++ b/sdk/python/kfp/dsl/placeholders.py @@ -179,7 +179,7 @@ class ConcatPlaceholder(Placeholder): def container_with_concat_placeholder(text1: str, text2: Output[Dataset], output_path: OutputPath(str)): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', ConcatPlaceholder(['prefix-', text1, text2.uri]) @@ -227,7 +227,7 @@ def container_with_if_placeholder(output_path: OutputPath(str), dataset: Output[Dataset], optional_input: str = 'default'): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', IfPresentPlaceholder( diff --git a/sdk/python/kfp/dsl/placeholders_test.py b/sdk/python/kfp/dsl/placeholders_test.py index 6c2fade8b6d..7d6526958f9 100644 --- a/sdk/python/kfp/dsl/placeholders_test.py +++ b/sdk/python/kfp/dsl/placeholders_test.py @@ -514,7 +514,7 @@ def container_with_placeholder_in_fstring( text1: str, ): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', f'prefix-{text1}', @@ -540,7 +540,7 @@ def container_with_placeholder_in_fstring( text2: str, ): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', f'another-prefix-{dsl.ConcatPlaceholder([text1, text2])}', @@ -557,7 +557,7 @@ def container_with_placeholder_in_fstring( text2: str, ): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'echo', f"another-prefix-{dsl.IfPresentPlaceholder(input_name='text1', then=['val'])}", diff --git a/sdk/python/kfp/dsl/structures_test.py b/sdk/python/kfp/dsl/structures_test.py index 29ebc7f16a5..0f80f0b4318 100644 --- a/sdk/python/kfp/dsl/structures_test.py +++ b/sdk/python/kfp/dsl/structures_test.py @@ -454,7 +454,7 @@ def test_env(self): def test_from_container_dict_no_placeholders(self): expected_container_spec = structures.ContainerSpecImplementation( - image='python:3.9', + image='python:3.11', command=['sh', '-c', 'dummy'], args=['--executor_input', '{{$}}', '--function_to_execute', 'func'], env={'ENV1': 'val1'}, @@ -465,7 +465,7 @@ def test_from_container_dict_no_placeholders(self): '--executor_input', '{{$}}', '--function_to_execute', 'func' ], 'command': ['sh', '-c', 'dummy'], - 'image': 'python:3.9', + 'image': 'python:3.11', 'env': { 'ENV1': 'val1' }, diff --git a/sdk/python/kfp/init_test.py b/sdk/python/kfp/init_test.py index ca2815c21fa..b061cb82fab 100644 --- a/sdk/python/kfp/init_test.py +++ b/sdk/python/kfp/init_test.py @@ -26,7 +26,7 @@ def test(self): with self.assertWarnsRegex( FutureWarning, - r'KFP will drop support for Python 3.9 on Oct 1, 2025. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.10. See https://devguide.python.org/versions/ for more details.' + r'KFP will drop support for Python 3.9 on October 1, 2026. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.10. See https://devguide.python.org/versions/ for more details.' ): # simulate first import from kfp importlib.reload(mod) diff --git a/sdk/python/kfp/local/docker_task_handler_test.py b/sdk/python/kfp/local/docker_task_handler_test.py index bfd265485fe..7a08f062ce7 100755 --- a/sdk/python/kfp/local/docker_task_handler_test.py +++ b/sdk/python/kfp/local/docker_task_handler_test.py @@ -299,7 +299,7 @@ def artifact_maker(x: str, a: Output[Artifact]): kwargs = run_mock.call_args[1] self.assertEqual( kwargs['image'], - 'python:3.9', + 'python:3.11', ) self.assertTrue( any('def artifact_maker' in c for c in kwargs['command'])) diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index 22785af552a..72219a4cb9e 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496 + - name: components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e value: '{"executorLabel":"exec-add-numbers","inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_INTEGER"},"b":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496 + - name: implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.9"}' + add_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"add-numbers"}}}},"tasks":{"add-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"componentInputParameter":"b"}}},"taskInfo":{"name":"add-numbers"}}}},"inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_INTEGER"},"b":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"componentInputParameter":"b"}}},"taskInfo":{"name":"add-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task-name value: add-numbers - name: parent-dag-id diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index bc30681e4fc..ae3297850df 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431 + - name: components-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"artifacts":{"input":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431 + - name: implementations-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef crust_comp(input: dsl.Dataset):\n with open(input.path, ''r'') as f:\n print(''input: - '', f.read())\n\n"],"image":"python:3.9"}' - - name: components-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d + '', f.read())\n\n"],"image":"python:3.11"}' + - name: components-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518 value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d + - name: implementations-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518 value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,7 +33,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_comp(dataset: dsl.Output[dsl.Dataset]):\n with open(dataset.path, - ''w'') as f:\n f.write(''foo'')\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(''foo'')\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"dataset","producerSubtask":"core-comp"}]}}},"tasks":{"core-comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: components-comp-mantle @@ -220,11 +220,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d}}' + value: '{{workflow.parameters.components-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d}}' + value: '{{workflow.parameters.implementations-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518}}' - name: task-name value: core-comp - name: parent-dag-id @@ -358,11 +358,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431}}' + value: '{{workflow.parameters.components-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"artifacts":{"input":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431}}' + value: '{{workflow.parameters.implementations-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 16a8f9d86b2..35c2bf4351c 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431 + - name: components-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"artifacts":{"input":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431 + - name: implementations-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef crust_comp(input: dsl.Dataset):\n with open(input.path, ''r'') as f:\n print(''input: - '', f.read())\n\n"],"image":"python:3.9"}' - - name: components-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d + '', f.read())\n\n"],"image":"python:3.11"}' + - name: components-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518 value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d + - name: implementations-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518 value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,7 +33,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_comp(dataset: dsl.Output[dsl.Dataset]):\n with open(dataset.path, - ''w'') as f:\n f.write(''foo'')\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(''foo'')\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"dataset","producerSubtask":"core-comp"}]}}},"tasks":{"core-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: components-comp-mantle @@ -220,11 +220,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d}}' + value: '{{workflow.parameters.components-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-6dc68c2089ddf2062dbae792f6214699e1d4a930573b3b4c3cc0626bea7bee2d}}' + value: '{{workflow.parameters.implementations-8e952934307908424012b1f26f02dd613c9508616be5a191f53bbb92ea5ff518}}' - name: task-name value: core-comp - name: parent-dag-id @@ -358,11 +358,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431}}' + value: '{{workflow.parameters.components-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"artifacts":{"input":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-a0055e53510db02175ab79ccc2c9c06c8b7bd7592f3d2ec6903b3de5d6b65431}}' + value: '{{workflow.parameters.implementations-f7a5dac7766188990a50c9769a1a548cdf1cf99da8f89bbad299b3175f7e8141}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index 1fb3f5e1af9..824b4699e7c 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08 + - name: components-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2 value: '{"executorLabel":"exec-add","inputDefinitions":{"artifacts":{"in_datasets":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08 + - name: implementations-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,10 +21,10 @@ spec: add(\n in_datasets: Input[List[Dataset]],\n out_dataset: Output[Dataset],\n):\n nums = []\n for dataset in in_datasets:\n with open(dataset.path) as f:\n nums.append(int(f.read()))\n with open(out_dataset.path, - ''w'') as f:\n f.write(str(sum(nums)))\n\n"],"image":"python:3.9"}' - - name: components-be8ee4858aa0dabdc7c419be456f8e7751d0de0a26fe140352aaadb1050c4ba3 + ''w'') as f:\n f.write(str(sum(nums)))\n\n"],"image":"python:3.11"}' + - name: components-0f454994223bfba6e739848f3c0233b8b8af4c822d95fddc776ea487b22500f0 value: '{"executorLabel":"exec-add-two-ints","inputDefinitions":{"artifacts":{"in_dataset1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"in_dataset2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-be8ee4858aa0dabdc7c419be456f8e7751d0de0a26fe140352aaadb1050c4ba3 + - name: implementations-0f454994223bfba6e739848f3c0233b8b8af4c822d95fddc776ea487b22500f0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_two_ints"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -38,12 +38,12 @@ spec: Output[Dataset],\n):\n with open(in_dataset1.path) as f:\n in_dataset1 = int(f.read())\n\n with open(in_dataset2.path) as f:\n in_dataset2 = int(f.read())\n\n with open(out_dataset.path, ''w'') as f:\n f.write(str(in_dataset1 - + in_dataset2))\n\n"],"image":"python:3.9"}' + + in_dataset2))\n\n"],"image":"python:3.11"}' - name: components-comp-add-two-lists-of-datasets value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"out_dataset","producerSubtask":"add-two-ints"}]}}},"tasks":{"add":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"artifacts":{"in_datasets":{"componentInputArtifact":"in_datasets1"}}},"taskInfo":{"name":"add"}},"add-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-2"},"inputs":{"artifacts":{"in_datasets":{"componentInputArtifact":"in_datasets2"}}},"taskInfo":{"name":"add-2"}},"add-two-ints":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-two-ints"},"dependentTasks":["add","add-2"],"inputs":{"artifacts":{"in_dataset1":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"add"}},"in_dataset2":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"add-2"}}}},"taskInfo":{"name":"add-two-ints"}}}},"inputDefinitions":{"artifacts":{"in_datasets1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true},"in_datasets2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: components-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49 + - name: components-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49 + - name: implementations-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -54,7 +54,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef double(\n num: int,\n out_dataset: Output[Dataset],\n):\n with open(out_dataset.path, - ''w'') as f:\n f.write(str(2 * num))\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(str(2 * num))\n\n"],"image":"python:3.11"}' - name: components-comp-condition-5 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--double-2-out_dataset":{"artifactSelectors":[{"outputArtifactKey":"out_dataset","producerSubtask":"double-2"}]}}},"tasks":{"double-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--threshold":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--double-2-out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - name: components-comp-for-loop-4 @@ -246,11 +246,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08}}' + value: '{{workflow.parameters.components-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"artifacts":{"in_datasets":{"componentInputArtifact":"in_datasets1"}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08}}' + value: '{{workflow.parameters.implementations-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2}}' - name: task-name value: add - name: parent-dag-id @@ -270,11 +270,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08}}' + value: '{{workflow.parameters.components-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-2"},"inputs":{"artifacts":{"in_datasets":{"componentInputArtifact":"in_datasets2"}}},"taskInfo":{"name":"add-2"}}' - name: container - value: '{{workflow.parameters.implementations-c57c4368bb3853e287d208a6153cc7cfdd0a6392a5da536df9a7944cb4018d08}}' + value: '{{workflow.parameters.implementations-16f08981216475ab27b039fb96f85c515876ffd8fd48dc99c163ad67ab8791c2}}' - name: task-name value: add-2 - name: parent-dag-id @@ -294,11 +294,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-be8ee4858aa0dabdc7c419be456f8e7751d0de0a26fe140352aaadb1050c4ba3}}' + value: '{{workflow.parameters.components-0f454994223bfba6e739848f3c0233b8b8af4c822d95fddc776ea487b22500f0}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-two-ints"},"dependentTasks":["add","add-2"],"inputs":{"artifacts":{"in_dataset1":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"add"}},"in_dataset2":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"add-2"}}}},"taskInfo":{"name":"add-two-ints"}}' - name: container - value: '{{workflow.parameters.implementations-be8ee4858aa0dabdc7c419be456f8e7751d0de0a26fe140352aaadb1050c4ba3}}' + value: '{{workflow.parameters.implementations-0f454994223bfba6e739848f3c0233b8b8af4c822d95fddc776ea487b22500f0}}' - name: task-name value: add-two-ints - name: parent-dag-id @@ -327,11 +327,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49}}' + value: '{{workflow.parameters.components-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}' - name: container - value: '{{workflow.parameters.implementations-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49}}' + value: '{{workflow.parameters.implementations-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47}}' - name: task-name value: double-2 - name: parent-dag-id @@ -528,11 +528,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49}}' + value: '{{workflow.parameters.components-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ddb6006c75f716bd2d06e80c76e9743f6cc6dd588e1c1dda65066ba1d0ec3b49}}' + value: '{{workflow.parameters.implementations-7d8f01a93fab448a938303bbb205e2cf7bfadf50bc37698ff2784aebf5363d47}}' - name: task-name value: double - name: parent-dag-id diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 3d70bf6de8b..c63c37aa716 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-80d990c0f4b48d97644425ebeb33e4614a065cb6fef18de932c1a962063f9ed1 + - name: components-fe3df0f6fa90face22ffbe395cfab1dc3052f996d7af9c457c592a156a228e9c value: '{"executorLabel":"exec-add","inputDefinitions":{"artifacts":{"in_datasets":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-80d990c0f4b48d97644425ebeb33e4614a065cb6fef18de932c1a962063f9ed1 + - name: implementations-fe3df0f6fa90face22ffbe395cfab1dc3052f996d7af9c457c592a156a228e9c value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,20 +21,20 @@ spec: add(in_datasets: Input[List[Dataset]], out_dataset: Output[Dataset]):\n nums = []\n for dataset in in_datasets:\n with open(dataset.path) as f:\n nums.append(int(f.read()))\n with open(out_dataset.path, - ''w'') as f:\n f.write(str(sum(nums)))\n\n"],"image":"python:3.9"}' - - name: components-699e6289d2b910228c41f94be6dd87fc0851d8dd1e68d93412ad045a9db72279 + ''w'') as f:\n f.write(str(sum(nums)))\n\n"],"image":"python:3.11"}' + - name: components-10160d6ed5205864573d7fc5ab62e00d8b050cf985d80b0bbaae43ba7d32c6c4 value: '{"executorLabel":"exec-add-container","inputDefinitions":{"artifacts":{"in_datasets":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-699e6289d2b910228c41f94be6dd87fc0851d8dd1e68d93412ad045a9db72279 + - name: implementations-10160d6ed5205864573d7fc5ab62e00d8b050cf985d80b0bbaae43ba7d32c6c4 value: '{"args":["\nimport argparse\nimport json\nimport os\n\ndef main(in_datasets, out_dataset_uri):\n in_dicts = json.loads(in_datasets)\n uris = [d[''uri''] for d in in_dicts]\n total = 0\n for uri in uris:\n with open(uri.replace(''gs://'', ''/gcs/'')) as f:\n total += int(f.read())\n\n outpath = out_dataset_uri.replace(''gs://'', ''/gcs/'')\n os.makedirs(os.path.dirname(outpath), exist_ok=True)\n with open(outpath, ''w'') as f:\n f.write(str(total))\n\nparser = argparse.ArgumentParser()\nparser.add_argument(''in_datasets'')\nparser.add_argument(''out_dataset_uri'')\nargs - = parser.parse_args()\n\nmain(args.in_datasets, args.out_dataset_uri)\n","{{$.inputs.artifacts[''in_datasets'']}}","{{$.outputs.artifacts[''out_dataset''].uri}}"],"command":["python","-c"],"image":"python:3.9"}' - - name: components-bd592e219a10055cde863d9b28fdaf17dfe7658370d390ac470bb7be0cf58b9b + = parser.parse_args()\n\nmain(args.in_datasets, args.out_dataset_uri)\n","{{$.inputs.artifacts[''in_datasets'']}}","{{$.outputs.artifacts[''out_dataset''].uri}}"],"command":["python","-c"],"image":"python:3.11"}' + - name: components-0242e2764b7c7b68c6cabf3e21fd30e2eebe9df2d233c19bf4b4c103b3ff7560 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-bd592e219a10055cde863d9b28fdaf17dfe7658370d390ac470bb7be0cf58b9b + - name: implementations-0242e2764b7c7b68c6cabf3e21fd30e2eebe9df2d233c19bf4b4c103b3ff7560 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -45,7 +45,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef double(num: int, out_dataset: Output[Dataset]):\n with open(out_dataset.path, - ''w'') as f:\n f.write(str(2 * num))\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(str(2 * num))\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--double-out_dataset":{"artifactSelectors":[{"outputArtifactKey":"out_dataset","producerSubtask":"double"}]}}},"tasks":{"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--double-out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - name: components-root @@ -231,11 +231,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-bd592e219a10055cde863d9b28fdaf17dfe7658370d390ac470bb7be0cf58b9b}}' + value: '{{workflow.parameters.components-0242e2764b7c7b68c6cabf3e21fd30e2eebe9df2d233c19bf4b4c103b3ff7560}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-bd592e219a10055cde863d9b28fdaf17dfe7658370d390ac470bb7be0cf58b9b}}' + value: '{{workflow.parameters.implementations-0242e2764b7c7b68c6cabf3e21fd30e2eebe9df2d233c19bf4b4c103b3ff7560}}' - name: task-name value: double - name: parent-dag-id @@ -401,11 +401,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-80d990c0f4b48d97644425ebeb33e4614a065cb6fef18de932c1a962063f9ed1}}' + value: '{{workflow.parameters.components-fe3df0f6fa90face22ffbe395cfab1dc3052f996d7af9c457c592a156a228e9c}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"in_datasets":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--double-out_dataset","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-80d990c0f4b48d97644425ebeb33e4614a065cb6fef18de932c1a962063f9ed1}}' + value: '{{workflow.parameters.implementations-fe3df0f6fa90face22ffbe395cfab1dc3052f996d7af9c457c592a156a228e9c}}' - name: task-name value: add - name: parent-dag-id @@ -426,11 +426,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-699e6289d2b910228c41f94be6dd87fc0851d8dd1e68d93412ad045a9db72279}}' + value: '{{workflow.parameters.components-10160d6ed5205864573d7fc5ab62e00d8b050cf985d80b0bbaae43ba7d32c6c4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-container"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"in_datasets":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--double-out_dataset","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"add-container"}}' - name: container - value: '{{workflow.parameters.implementations-699e6289d2b910228c41f94be6dd87fc0851d8dd1e68d93412ad045a9db72279}}' + value: '{{workflow.parameters.implementations-10160d6ed5205864573d7fc5ab62e00d8b050cf985d80b0bbaae43ba7d32c6c4}}' - name: task-name value: add-container - name: parent-dag-id diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index a3d0510166b..dd8a3f82615 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-454f675bb7800adf80b4227cf4afab17ccd8f2158391d6c83b382a19f6c06d20 + - name: components-ec1f774a9791d5bf8f1f6f2d765e693d12aa053c0068a46d18df151f5a9a8545 value: '{"executorLabel":"exec-create-file","inputDefinitions":{"parameters":{"content":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"file":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-454f675bb7800adf80b4227cf4afab17ccd8f2158391d6c83b382a19f6c06d20 + - name: implementations-ec1f774a9791d5bf8f1f6f2d765e693d12aa053c0068a46d18df151f5a9a8545 value: '{"args":["--executor_input","{{$}}","--function_to_execute","create_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef create_file(file: Output[Artifact], content: str):\n print(f''Creating - file with content: {content}'')\n with open(file.path, ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.9"}' - - name: components-8de1f97f1c6fbc2c49f794738b59fedba4bfc0367ad809393ff85e905f70886d + file with content: {content}'')\n with open(file.path, ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.11"}' + - name: components-f1c1eb97565294e29b1a6409078e07ca9d3302c9ccdacd7751eaa032a7d99f43 value: '{"executorLabel":"exec-read-single-dataset-generate-model","inputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"id":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"results":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-8de1f97f1c6fbc2c49f794738b59fedba4bfc0367ad809393ff85e905f70886d + - name: implementations-f1c1eb97565294e29b1a6409078e07ca9d3302c9ccdacd7751eaa032a7d99f43 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_single_dataset_generate_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -35,10 +35,10 @@ spec: read_single_dataset_generate_model(data: Dataset, id: str, results:Output[Model]):\n print(f''Reading file: {data.path}'')\n with open(data.path, ''r'') as f:\n info = f.read()\n with open(results.path, ''w'') as f2:\n f2.write(f\"{info}-{id}\")\n results.metadata[''model''] - = info\n results.metadata[''model_name''] = f\"model-artifact-inner-iteration-{info}-{id}\"\n\n"],"image":"python:3.9"}' - - name: components-110fc7a806182548336d0e41ebf637176c62d600d89da66d667e5241685993a3 + = info\n results.metadata[''model_name''] = f\"model-artifact-inner-iteration-{info}-{id}\"\n\n"],"image":"python:3.11"}' + - name: components-acfe6b8e77458ff7c9eb4a10a23639b58364c808c1c39d8c4c1235861d099dbf value: '{"executorLabel":"exec-create-dataset","inputDefinitions":{"parameters":{"content":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-110fc7a806182548336d0e41ebf637176c62d600d89da66d667e5241685993a3 + - name: implementations-acfe6b8e77458ff7c9eb4a10a23639b58364c808c1c39d8c4c1235861d099dbf value: '{"args":["--executor_input","{{$}}","--function_to_execute","create_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -49,14 +49,14 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef create_dataset(data: Output[Dataset], content: str):\n print(f''Creating - file with content: {content}'')\n with open(data.path, ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.9"}' + file with content: {content}'')\n with open(data.path, ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.11"}' - name: components-comp-single-node-dag value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"data","producerSubtask":"create-dataset"}]}}},"tasks":{"create-dataset":{"cachingOptions":{},"componentRef":{"name":"comp-create-dataset"},"inputs":{"parameters":{"content":{"componentInputParameter":"char"}}},"taskInfo":{"name":"create-dataset"}}}},"inputDefinitions":{"parameters":{"char":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: components-comp-for-loop-2 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--read-single-dataset-generate-model-results":{"artifactSelectors":[{"outputArtifactKey":"results","producerSubtask":"read-single-dataset-generate-model"}]},"pipelinechannel--single-node-dag-Output":{"artifactSelectors":[{"outputArtifactKey":"Output","producerSubtask":"single-node-dag"}]}}},"tasks":{"read-single-dataset-generate-model":{"cachingOptions":{},"componentRef":{"name":"comp-read-single-dataset-generate-model"},"dependentTasks":["single-node-dag"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"single-node-dag"}}},"parameters":{"id":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"read-single-dataset-generate-model"}},"single-node-dag":{"cachingOptions":{},"componentRef":{"name":"comp-single-node-dag"},"inputs":{"parameters":{"char":{"componentInputParameter":"pipelinechannel--split-chars-Output-loop-item"}}},"taskInfo":{"name":"single-node-dag"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--split-chars-Output":{"parameterType":"LIST"},"pipelinechannel--split-chars-Output-loop-item":{"parameterType":"STRING"},"pipelinechannel--split-ids-Output-loop-item":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--read-single-dataset-generate-model-results":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"},"isArtifactList":true},"pipelinechannel--single-node-dag-Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - - name: components-f13f4d96d8af0136b7ec2fd8bb04bf61e6936dec4fb6215eb91ac03b7800c8f9 + - name: components-d4b6b9e1cff17ea3ca93d63109e65319f3c20876c7867400587cb270fceed458 value: '{"executorLabel":"exec-read-datasets","inputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-f13f4d96d8af0136b7ec2fd8bb04bf61e6936dec4fb6215eb91ac03b7800c8f9 + - name: implementations-d4b6b9e1cff17ea3ca93d63109e65319f3c20876c7867400587cb270fceed458 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_datasets"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -68,10 +68,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef read_datasets(data: List[Dataset]) -\u003e str:\n for d in data:\n print(f''Reading dataset {d.name} file: {d.path}'')\n with open(d.path, ''r'') as f:\n print(f.read())\n\n return - ''files read''\n\n"],"image":"python:3.9"}' - - name: components-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702 + ''files read''\n\n"],"image":"python:3.11"}' + - name: components-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3 value: '{"executorLabel":"exec-read-models","inputDefinitions":{"artifacts":{"models":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702 + - name: implementations-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_models"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -84,10 +84,10 @@ spec: read_models(models: List[Model],) -\u003e str:\n for m in models:\n print(f''Reading model {m.name} file: {m.path}'')\n with open(m.path, ''r'') as f:\n info = f.read()\n print(f\"Model raw data: {info}\")\n print(f\"Model - metadata: {m.metadata}\")\n return ''models read''\n\n"],"image":"python:3.9"}' - - name: components-99e3dc1afb3392e03ef7e4ef8b3f63c142e5fbc03c571b8879a6e9429b814760 + metadata: {m.metadata}\")\n return ''models read''\n\n"],"image":"python:3.11"}' + - name: components-cb38e5430ba66f0db667b439264d51cf1757fcf432fc9ea4fe8e38fbd468cd1a value: '{"executorLabel":"exec-read-single-file","inputDefinitions":{"artifacts":{"file":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-99e3dc1afb3392e03ef7e4ef8b3f63c142e5fbc03c571b8879a6e9429b814760 + - name: implementations-cb38e5430ba66f0db667b439264d51cf1757fcf432fc9ea4fe8e38fbd468cd1a value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_single_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -99,12 +99,12 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef read_single_file(file: Artifact) -\u003e str:\n print(f''Reading file: {file.path}'')\n with open(file.path, ''r'') as f:\n print(f.read())\n\n return - file.uri\n\n"],"image":"python:3.9"}' + file.uri\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--read-single-dataset-generate-model-results":{"artifactSelectors":[{"outputArtifactKey":"pipelinechannel--read-single-dataset-generate-model-results","producerSubtask":"for-loop-2"}]}}},"tasks":{"create-file":{"cachingOptions":{},"componentRef":{"name":"comp-create-file"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"create-file"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"componentInputParameter":"pipelinechannel--split-chars-Output"},"pipelinechannel--split-ids-Output-loop-item":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"parameterIterator":{"itemInput":"pipelinechannel--split-chars-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-chars-Output"}},"taskInfo":{"name":"for-loop-2"}},"read-datasets":{"cachingOptions":{},"componentRef":{"name":"comp-read-datasets"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--single-node-dag-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"read-datasets"}},"read-models":{"cachingOptions":{},"componentRef":{"name":"comp-read-models"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"models":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--read-single-dataset-generate-model-results","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"read-models"}},"read-single-file":{"cachingOptions":{},"componentRef":{"name":"comp-read-single-file"},"dependentTasks":["create-file"],"inputs":{"artifacts":{"file":{"taskOutputArtifact":{"outputArtifactKey":"file","producerTask":"create-file"}}}},"taskInfo":{"name":"read-single-file"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--split-chars-Output":{"parameterType":"LIST"},"pipelinechannel--split-ids-Output":{"parameterType":"LIST"},"pipelinechannel--split-ids-Output-loop-item":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--read-single-dataset-generate-model-results":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - - name: components-194a4bfb844b1828b520d4f1d24304bbdba6a0c71f95c1e315dfbfa4e171f4ea + - name: components-be1feef4ec8e00ce2e55f9604db9c6b0cca05dcfeaf633230ca6bf0723babac8 value: '{"executorLabel":"exec-split-chars","inputDefinitions":{"parameters":{"model_ids":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-194a4bfb844b1828b520d4f1d24304bbdba6a0c71f95c1e315dfbfa4e171f4ea + - name: implementations-be1feef4ec8e00ce2e55f9604db9c6b0cca05dcfeaf633230ca6bf0723babac8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","split_chars"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -114,10 +114,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - split_chars(model_ids: str) -\u003e list:\n return model_ids.split('','')\n\n"],"image":"python:3.9"}' - - name: components-b7700b8f98f5bc8b5097b759b614a5652ffb5baa093a27ca516cd10ce62e33de + split_chars(model_ids: str) -\u003e list:\n return model_ids.split('','')\n\n"],"image":"python:3.11"}' + - name: components-e8b2565c05c3de8bb0b7d77062539f34be122f1b8169f7198d40d58499d26d4e value: '{"executorLabel":"exec-split-ids","inputDefinitions":{"parameters":{"model_ids":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-b7700b8f98f5bc8b5097b759b614a5652ffb5baa093a27ca516cd10ce62e33de + - name: implementations-e8b2565c05c3de8bb0b7d77062539f34be122f1b8169f7198d40d58499d26d4e value: '{"args":["--executor_input","{{$}}","--function_to_execute","split_ids"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -127,7 +127,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - split_ids(model_ids: str) -\u003e list:\n return model_ids.split('','')\n\n"],"image":"python:3.9"}' + split_ids(model_ids: str) -\u003e list:\n return model_ids.split('','')\n\n"],"image":"python:3.11"}' - name: components-comp-collecting-artifacts value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"pipelinechannel--read-single-dataset-generate-model-results","producerSubtask":"for-loop-1"}]}}},"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-chars","split-ids"],"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-chars"}},"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}},"split-chars":{"cachingOptions":{},"componentRef":{"name":"comp-split-chars"},"inputs":{"parameters":{"model_ids":{"componentInputParameter":"model_chars"}}},"taskInfo":{"name":"split-chars"}},"split-ids":{"cachingOptions":{},"componentRef":{"name":"comp-split-ids"},"inputs":{"parameters":{"model_ids":{"componentInputParameter":"model_ids"}}},"taskInfo":{"name":"split-ids"}}}},"inputDefinitions":{"parameters":{"model_chars":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"model_ids":{"defaultValue":"","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - name: components-root @@ -312,11 +312,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-110fc7a806182548336d0e41ebf637176c62d600d89da66d667e5241685993a3}}' + value: '{{workflow.parameters.components-acfe6b8e77458ff7c9eb4a10a23639b58364c808c1c39d8c4c1235861d099dbf}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-create-dataset"},"inputs":{"parameters":{"content":{"componentInputParameter":"char"}}},"taskInfo":{"name":"create-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-110fc7a806182548336d0e41ebf637176c62d600d89da66d667e5241685993a3}}' + value: '{{workflow.parameters.implementations-acfe6b8e77458ff7c9eb4a10a23639b58364c808c1c39d8c4c1235861d099dbf}}' - name: task-name value: create-dataset - name: parent-dag-id @@ -421,11 +421,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8de1f97f1c6fbc2c49f794738b59fedba4bfc0367ad809393ff85e905f70886d}}' + value: '{{workflow.parameters.components-f1c1eb97565294e29b1a6409078e07ca9d3302c9ccdacd7751eaa032a7d99f43}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-single-dataset-generate-model"},"dependentTasks":["single-node-dag"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"single-node-dag"}}},"parameters":{"id":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"read-single-dataset-generate-model"}}' - name: container - value: '{{workflow.parameters.implementations-8de1f97f1c6fbc2c49f794738b59fedba4bfc0367ad809393ff85e905f70886d}}' + value: '{{workflow.parameters.implementations-f1c1eb97565294e29b1a6409078e07ca9d3302c9ccdacd7751eaa032a7d99f43}}' - name: task-name value: read-single-dataset-generate-model - name: parent-dag-id @@ -534,11 +534,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-454f675bb7800adf80b4227cf4afab17ccd8f2158391d6c83b382a19f6c06d20}}' + value: '{{workflow.parameters.components-ec1f774a9791d5bf8f1f6f2d765e693d12aa053c0068a46d18df151f5a9a8545}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-create-file"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"create-file"}}' - name: container - value: '{{workflow.parameters.implementations-454f675bb7800adf80b4227cf4afab17ccd8f2158391d6c83b382a19f6c06d20}}' + value: '{{workflow.parameters.implementations-ec1f774a9791d5bf8f1f6f2d765e693d12aa053c0068a46d18df151f5a9a8545}}' - name: task-name value: create-file - name: parent-dag-id @@ -564,11 +564,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f13f4d96d8af0136b7ec2fd8bb04bf61e6936dec4fb6215eb91ac03b7800c8f9}}' + value: '{{workflow.parameters.components-d4b6b9e1cff17ea3ca93d63109e65319f3c20876c7867400587cb270fceed458}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-datasets"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--single-node-dag-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"read-datasets"}}' - name: container - value: '{{workflow.parameters.implementations-f13f4d96d8af0136b7ec2fd8bb04bf61e6936dec4fb6215eb91ac03b7800c8f9}}' + value: '{{workflow.parameters.implementations-d4b6b9e1cff17ea3ca93d63109e65319f3c20876c7867400587cb270fceed458}}' - name: task-name value: read-datasets - name: parent-dag-id @@ -589,11 +589,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702}}' + value: '{{workflow.parameters.components-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-models"},"dependentTasks":["for-loop-2"],"inputs":{"artifacts":{"models":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--read-single-dataset-generate-model-results","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"read-models"}}' - name: container - value: '{{workflow.parameters.implementations-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702}}' + value: '{{workflow.parameters.implementations-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3}}' - name: task-name value: read-models - name: parent-dag-id @@ -614,11 +614,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-99e3dc1afb3392e03ef7e4ef8b3f63c142e5fbc03c571b8879a6e9429b814760}}' + value: '{{workflow.parameters.components-cb38e5430ba66f0db667b439264d51cf1757fcf432fc9ea4fe8e38fbd468cd1a}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-single-file"},"dependentTasks":["create-file"],"inputs":{"artifacts":{"file":{"taskOutputArtifact":{"outputArtifactKey":"file","producerTask":"create-file"}}}},"taskInfo":{"name":"read-single-file"}}' - name: container - value: '{{workflow.parameters.implementations-99e3dc1afb3392e03ef7e4ef8b3f63c142e5fbc03c571b8879a6e9429b814760}}' + value: '{{workflow.parameters.implementations-cb38e5430ba66f0db667b439264d51cf1757fcf432fc9ea4fe8e38fbd468cd1a}}' - name: task-name value: read-single-file - name: parent-dag-id @@ -713,11 +713,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-194a4bfb844b1828b520d4f1d24304bbdba6a0c71f95c1e315dfbfa4e171f4ea}}' + value: '{{workflow.parameters.components-be1feef4ec8e00ce2e55f9604db9c6b0cca05dcfeaf633230ca6bf0723babac8}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-split-chars"},"inputs":{"parameters":{"model_ids":{"componentInputParameter":"model_chars"}}},"taskInfo":{"name":"split-chars"}}' - name: container - value: '{{workflow.parameters.implementations-194a4bfb844b1828b520d4f1d24304bbdba6a0c71f95c1e315dfbfa4e171f4ea}}' + value: '{{workflow.parameters.implementations-be1feef4ec8e00ce2e55f9604db9c6b0cca05dcfeaf633230ca6bf0723babac8}}' - name: task-name value: split-chars - name: parent-dag-id @@ -737,11 +737,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b7700b8f98f5bc8b5097b759b614a5652ffb5baa093a27ca516cd10ce62e33de}}' + value: '{{workflow.parameters.components-e8b2565c05c3de8bb0b7d77062539f34be122f1b8169f7198d40d58499d26d4e}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-split-ids"},"inputs":{"parameters":{"model_ids":{"componentInputParameter":"model_ids"}}},"taskInfo":{"name":"split-ids"}}' - name: container - value: '{{workflow.parameters.implementations-b7700b8f98f5bc8b5097b759b614a5652ffb5baa093a27ca516cd10ce62e33de}}' + value: '{{workflow.parameters.implementations-e8b2565c05c3de8bb0b7d77062539f34be122f1b8169f7198d40d58499d26d4e}}' - name: task-name value: split-ids - name: parent-dag-id @@ -790,11 +790,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702}}' + value: '{{workflow.parameters.components-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-models-2"},"dependentTasks":["collecting-artifacts"],"inputs":{"artifacts":{"models":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"collecting-artifacts"}}}},"taskInfo":{"name":"read-models"}}' - name: container - value: '{{workflow.parameters.implementations-a20411795c50176906ce97cafe2d672030b0584c63cc3f061d6b5fa4d8d18702}}' + value: '{{workflow.parameters.implementations-36399431fc5940f221467d3cd76e824ebecf7385e9634fda1cf2b1ded43ffbe3}}' - name: task-name value: read-models - name: parent-dag-id diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index ab91ec99baf..ad98c7df191 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a + - name: components-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2 value: '{"executorLabel":"exec-consume-ids","inputDefinitions":{"parameters":{"ids":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a + - name: implementations-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","consume_ids"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef consume_ids(ids: List[str]) -\u003e str:\n for id in ids:\n print(f''Consuming: - {id}'')\n return ''completed''\n\n"],"image":"python:3.9"}' - - name: components-9188bbf5b36c887074ca4eb40d70e9941b5a8c7907d6f07c5c30c0f5e2d5d77f + {id}'')\n return ''completed''\n\n"],"image":"python:3.11"}' + - name: components-7afbc0576341694fb223cf141107254be822a3b82ec9ee880741d0440246a0c0 value: '{"executorLabel":"exec-consume-single-id","inputDefinitions":{"parameters":{"id":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-9188bbf5b36c887074ca4eb40d70e9941b5a8c7907d6f07c5c30c0f5e2d5d77f + - name: implementations-7afbc0576341694fb223cf141107254be822a3b82ec9ee880741d0440246a0c0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","consume_single_id"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,10 +33,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef consume_single_id(id: str) -\u003e str:\n print(f''Consuming single: {id}'')\n return - ''completed''\n\n"],"image":"python:3.9"}' - - name: components-df2373cb8810b1a0954f3daeec49ae019a9421b35f7405c8e212dbfd47c1d48c + ''completed''\n\n"],"image":"python:3.11"}' + - name: components-579756e400e5e7eb1e9d6fabfb7a8e87a8971dbe30dcfb3fee4b6bf02148944a value: '{"executorLabel":"exec-prepend-id","inputDefinitions":{"parameters":{"content":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-df2373cb8810b1a0954f3daeec49ae019a9421b35f7405c8e212dbfd47c1d48c + - name: implementations-579756e400e5e7eb1e9d6fabfb7a8e87a8971dbe30dcfb3fee4b6bf02148944a value: '{"args":["--executor_input","{{$}}","--function_to_execute","prepend_id"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -47,12 +47,12 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef prepend_id(content: str) -\u003e str:\n print(f\"prepending: {content} - with ''model_id''\")\n return f''model_id_{content}''\n\n"],"image":"python:3.9"}' + with ''model_id''\")\n return f''model_id_{content}''\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--prepend-id-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"prepend-id"}}}},"tasks":{"consume-single-id":{"cachingOptions":{},"componentRef":{"name":"comp-consume-single-id"},"dependentTasks":["prepend-id"],"inputs":{"parameters":{"id":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"prepend-id"}}}},"taskInfo":{"name":"consume-single-id"}},"prepend-id":{"cachingOptions":{},"componentRef":{"name":"comp-prepend-id"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"prepend-id"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--split-ids-Output":{"parameterType":"LIST"},"pipelinechannel--split-ids-Output-loop-item":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--prepend-id-Output":{"parameterType":"LIST"}}}}' - - name: components-4dc5ff1bee8506864b281bf8b8add721ff9352165ca4734c110638f55de15497 + - name: components-675759a85fbbe43a31184fa3998f5ad3d5231281f0a7b9bea7eae605a3f8f570 value: '{"executorLabel":"exec-split-ids","inputDefinitions":{"parameters":{"ids":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-4dc5ff1bee8506864b281bf8b8add721ff9352165ca4734c110638f55de15497 + - name: implementations-675759a85fbbe43a31184fa3998f5ad3d5231281f0a7b9bea7eae605a3f8f570 value: '{"args":["--executor_input","{{$}}","--function_to_execute","split_ids"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -62,7 +62,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - split_ids(ids: str) -\u003e list:\n return ids.split('','')\n\n"],"image":"python:3.9"}' + split_ids(ids: str) -\u003e list:\n return ids.split('','')\n\n"],"image":"python:3.11"}' - name: components-comp-collecting-parameters value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--prepend-id-Output","producerSubtask":"for-loop-1"}}}},"tasks":{"consume-ids":{"cachingOptions":{},"componentRef":{"name":"comp-consume-ids"},"dependentTasks":["for-loop-1"],"inputs":{"parameters":{"ids":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--prepend-id-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"consume-ids"}},"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-ids"],"inputs":{"parameters":{"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}},"split-ids":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-split-ids"},"inputs":{"parameters":{"ids":{"componentInputParameter":"model_ids"}}},"taskInfo":{"name":"split-ids"}}}},"inputDefinitions":{"parameters":{"model_ids":{"defaultValue":"","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - name: components-root @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9188bbf5b36c887074ca4eb40d70e9941b5a8c7907d6f07c5c30c0f5e2d5d77f}}' + value: '{{workflow.parameters.components-7afbc0576341694fb223cf141107254be822a3b82ec9ee880741d0440246a0c0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-consume-single-id"},"dependentTasks":["prepend-id"],"inputs":{"parameters":{"id":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"prepend-id"}}}},"taskInfo":{"name":"consume-single-id"}}' - name: container - value: '{{workflow.parameters.implementations-9188bbf5b36c887074ca4eb40d70e9941b5a8c7907d6f07c5c30c0f5e2d5d77f}}' + value: '{{workflow.parameters.implementations-7afbc0576341694fb223cf141107254be822a3b82ec9ee880741d0440246a0c0}}' - name: task-name value: consume-single-id - name: parent-dag-id @@ -272,11 +272,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-df2373cb8810b1a0954f3daeec49ae019a9421b35f7405c8e212dbfd47c1d48c}}' + value: '{{workflow.parameters.components-579756e400e5e7eb1e9d6fabfb7a8e87a8971dbe30dcfb3fee4b6bf02148944a}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-prepend-id"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"taskInfo":{"name":"prepend-id"}}' - name: container - value: '{{workflow.parameters.implementations-df2373cb8810b1a0954f3daeec49ae019a9421b35f7405c8e212dbfd47c1d48c}}' + value: '{{workflow.parameters.implementations-579756e400e5e7eb1e9d6fabfb7a8e87a8971dbe30dcfb3fee4b6bf02148944a}}' - name: task-name value: prepend-id - name: parent-dag-id @@ -440,11 +440,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a}}' + value: '{{workflow.parameters.components-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-consume-ids"},"dependentTasks":["for-loop-1"],"inputs":{"parameters":{"ids":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--prepend-id-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"consume-ids"}}' - name: container - value: '{{workflow.parameters.implementations-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a}}' + value: '{{workflow.parameters.implementations-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2}}' - name: task-name value: consume-ids - name: parent-dag-id @@ -472,11 +472,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4dc5ff1bee8506864b281bf8b8add721ff9352165ca4734c110638f55de15497}}' + value: '{{workflow.parameters.components-675759a85fbbe43a31184fa3998f5ad3d5231281f0a7b9bea7eae605a3f8f570}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-split-ids"},"inputs":{"parameters":{"ids":{"componentInputParameter":"model_ids"}}},"taskInfo":{"name":"split-ids"}}' - name: container - value: '{{workflow.parameters.implementations-4dc5ff1bee8506864b281bf8b8add721ff9352165ca4734c110638f55de15497}}' + value: '{{workflow.parameters.implementations-675759a85fbbe43a31184fa3998f5ad3d5231281f0a7b9bea7eae605a3f8f570}}' - name: task-name value: split-ids - name: parent-dag-id @@ -525,11 +525,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a}}' + value: '{{workflow.parameters.components-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-consume-ids-2"},"dependentTasks":["collecting-parameters"],"inputs":{"parameters":{"ids":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"collecting-parameters"}}}},"taskInfo":{"name":"consume-ids"}}' - name: container - value: '{{workflow.parameters.implementations-7a820e79c6578d36752f6fda7d48b8a137a50faa9bd35b332acc7fb844f2a95a}}' + value: '{{workflow.parameters.implementations-690594f98b930e924d092cdddb1c8379f51c5ea407e8ff519feae9d7947be1c2}}' - name: task-name value: consume-ids - name: parent-dag-id diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index 922c34bb1d8..958191d7308 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -6,13 +6,13 @@ metadata: spec: arguments: parameters: - - name: components-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f + - name: components-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff value: '{"executorLabel":"exec-dataset-joiner","inputDefinitions":{"artifacts":{"dataset_a":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"First dataset."},"dataset_b":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"Second dataset."}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"The concatenated dataset."}},"parameters":{"Output":{"description":"The concatenated string.","parameterType":"STRING"}}}}' - - name: implementations-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f + - name: implementations-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff value: '{"args":["--executor_input","{{$}}","--function_to_execute","dataset_joiner"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -29,7 +29,7 @@ spec: The concatenated string.\n \"\"\"\n with open(dataset_a.path) as f:\n content_a = f.read()\n\n with open(dataset_b.path) as f:\n content_b = f.read()\n\n concatenated_string = content_a + content_b\n with open(out_dataset.path, ''w'') as f:\n f.write(concatenated_string)\n\n return - concatenated_string\n\n"],"image":"python:3.9"}' + concatenated_string\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"out_dataset":{"artifactSelectors":[{"outputArtifactKey":"out_dataset","producerSubtask":"dataset-joiner"}]}},"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"dataset-joiner"}}}},"tasks":{"dataset-joiner":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-joiner"},"inputs":{"artifacts":{"dataset_a":{"componentInputArtifact":"dataset_a"},"dataset_b":{"componentInputArtifact":"dataset_b"}}},"taskInfo":{"name":"dataset-joiner"}}}},"inputDefinitions":{"artifacts":{"dataset_a":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"First dataset."},"dataset_b":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"Second @@ -216,11 +216,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f}}' + value: '{{workflow.parameters.components-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-joiner"},"inputs":{"artifacts":{"dataset_a":{"componentInputArtifact":"dataset_a"},"dataset_b":{"componentInputArtifact":"dataset_b"}}},"taskInfo":{"name":"dataset-joiner"}}' - name: container - value: '{{workflow.parameters.implementations-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f}}' + value: '{{workflow.parameters.implementations-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff}}' - name: task-name value: dataset-joiner - name: parent-dag-id diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index cd0c46a2ee7..2fb5d515521 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -6,10 +6,10 @@ metadata: spec: arguments: parameters: - - name: components-305fccf43860a79b150cabe2f8d8140dd3c6539588f7e9c3f1788e775bd62887 + - name: components-8a37febc0bf19dffb0c33aa550ff8996f9e410b3f2b4c7f8f527b0c40618bb22 value: '{"executorLabel":"exec-component-op","inputDefinitions":{"parameters":{"input1":{"defaultValue":"default value","isOptional":true,"parameterType":"STRING"},"input2":{"isOptional":true,"parameterType":"STRING"},"input3":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-305fccf43860a79b150cabe2f8d8140dd3c6539588f7e9c3f1788e775bd62887 + - name: implementations-8a37febc0bf19dffb0c33aa550ff8996f9e410b3f2b4c7f8f527b0c40618bb22 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -22,7 +22,7 @@ spec: component_op(\n input1: str = ''default value'',\n input2: Optional[str] = None,\n input3: Optional[str] = None,\n):\n print(f''input1: {input1}, type: {type(input1)}'')\n print(f''input2: {input2}, type: {type(input2)}'')\n print(f''input3: - {input3}, type: {type(input3)}'')\n\n"],"image":"python:3.9"}' + {input3}, type: {type(input3)}'')\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"component-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-op"},"inputs":{"parameters":{"input1":{"runtimeValue":{"constant":"Hello"}},"input2":{"runtimeValue":{"constant":"World"}}}},"taskInfo":{"name":"component-op"}}}}}' entrypoint: entrypoint @@ -205,11 +205,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-305fccf43860a79b150cabe2f8d8140dd3c6539588f7e9c3f1788e775bd62887}}' + value: '{{workflow.parameters.components-8a37febc0bf19dffb0c33aa550ff8996f9e410b3f2b4c7f8f527b0c40618bb22}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-op"},"inputs":{"parameters":{"input1":{"runtimeValue":{"constant":"Hello"}},"input2":{"runtimeValue":{"constant":"World"}}}},"taskInfo":{"name":"component-op"}}' - name: container - value: '{{workflow.parameters.implementations-305fccf43860a79b150cabe2f8d8140dd3c6539588f7e9c3f1788e775bd62887}}' + value: '{{workflow.parameters.implementations-8a37febc0bf19dffb0c33aa550ff8996f9e410b3f2b4c7f8f527b0c40618bb22}}' - name: task-name value: component-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index a68ff8ab866..dcacdb44a4f 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-193733112d59e5469b4142adfbe2c9deed6bad8e824a72821f68347f1923663f + - name: components-47cfdcdf2aba86202a9b65c5f3a504c58eeb1fbd3e74c26e64e10f9fd0f3b8b9 value: '{"executorLabel":"exec-component-op"}' - - name: implementations-193733112d59e5469b4142adfbe2c9deed6bad8e824a72821f68347f1923663f + - name: implementations-47cfdcdf2aba86202a9b65c5f3a504c58eeb1fbd3e74c26e64e10f9fd0f3b8b9 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,7 +20,7 @@ spec: -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - component_op():\n import yapf\n print(dir(yapf))\n\n"],"image":"python:3.9"}' + component_op():\n import yapf\n print(dir(yapf))\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"component-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-op"},"taskInfo":{"name":"component-op"}}}}}' entrypoint: entrypoint @@ -203,11 +203,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-193733112d59e5469b4142adfbe2c9deed6bad8e824a72821f68347f1923663f}}' + value: '{{workflow.parameters.components-47cfdcdf2aba86202a9b65c5f3a504c58eeb1fbd3e74c26e64e10f9fd0f3b8b9}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-op"},"taskInfo":{"name":"component-op"}}' - name: container - value: '{{workflow.parameters.implementations-193733112d59e5469b4142adfbe2c9deed6bad8e824a72821f68347f1923663f}}' + value: '{{workflow.parameters.implementations-47cfdcdf2aba86202a9b65c5f3a504c58eeb1fbd3e74c26e64e10f9fd0f3b8b9}}' - name: task-name value: component-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index 54c58d83ef0..2ac9f4910e6 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-802eb316f0b783824e20b61d8170192b475638db1d585e4b891cfe17e7c76602 + - name: components-76f86a3dc9bc2983c5f9c392d26e715eac83f100d1612e8841cab55696a13ec4 value: '{"executorLabel":"exec-component-with-pip-install"}' - - name: implementations-802eb316f0b783824e20b61d8170192b475638db1d585e4b891cfe17e7c76602 + - name: implementations-76f86a3dc9bc2983c5f9c392d26e715eac83f100d1612e8841cab55696a13ec4 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_with_pip_install"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,7 +20,7 @@ spec: -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - component_with_pip_install():\n import yapf\n print(dir(yapf))\n\n"],"image":"python:3.9"}' + component_with_pip_install():\n import yapf\n print(dir(yapf))\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"component-with-pip-install":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-with-pip-install"},"taskInfo":{"name":"component-with-pip-install"}}}}}' entrypoint: entrypoint @@ -203,11 +203,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-802eb316f0b783824e20b61d8170192b475638db1d585e4b891cfe17e7c76602}}' + value: '{{workflow.parameters.components-76f86a3dc9bc2983c5f9c392d26e715eac83f100d1612e8841cab55696a13ec4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-with-pip-install"},"taskInfo":{"name":"component-with-pip-install"}}' - name: container - value: '{{workflow.parameters.implementations-802eb316f0b783824e20b61d8170192b475638db1d585e4b891cfe17e7c76602}}' + value: '{{workflow.parameters.implementations-76f86a3dc9bc2983c5f9c392d26e715eac83f100d1612e8841cab55696a13ec4}}' - name: task-name value: component-with-pip-install - name: parent-dag-id diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 6fdfc820759..16c1d98b3a8 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-2074f0535aaa76760a0be660778c79824ceacab159cd4d83a847c6a3876650e0 + - name: components-153629da06fc36cdcd8f3c6e00a4b975ee79bcb08aac69fd0c9e23e727c0e838 value: '{"executorLabel":"exec-component-with-pip-install"}' - - name: implementations-2074f0535aaa76760a0be660778c79824ceacab159cd4d83a847c6a3876650e0 + - name: implementations-153629da06fc36cdcd8f3c6e00a4b975ee79bcb08aac69fd0c9e23e727c0e838 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_with_pip_install"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,7 +21,7 @@ spec: -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - component_with_pip_install():\n import yapf\n\n print(dir(yapf))\n\n"],"image":"python:3.9"}' + component_with_pip_install():\n import yapf\n\n print(dir(yapf))\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"component-with-pip-install":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-with-pip-install"},"taskInfo":{"name":"component-with-pip-install"}}}}}' entrypoint: entrypoint @@ -204,11 +204,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2074f0535aaa76760a0be660778c79824ceacab159cd4d83a847c6a3876650e0}}' + value: '{{workflow.parameters.components-153629da06fc36cdcd8f3c6e00a4b975ee79bcb08aac69fd0c9e23e727c0e838}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-with-pip-install"},"taskInfo":{"name":"component-with-pip-install"}}' - name: container - value: '{{workflow.parameters.implementations-2074f0535aaa76760a0be660778c79824ceacab159cd4d83a847c6a3876650e0}}' + value: '{{workflow.parameters.implementations-153629da06fc36cdcd8f3c6e00a4b975ee79bcb08aac69fd0c9e23e727c0e838}}' - name: task-name value: component-with-pip-install - name: parent-dag-id diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index 8a015239970..546b32b3897 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -16,9 +16,9 @@ spec: value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: implementations-comp-importer value: '{"artifactUri":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"},"typeSchema":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}' - - name: components-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994 + - name: components-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97 value: '{"executorLabel":"exec-python-artifact-printer","inputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"},"isOptional":true}}}}' - - name: implementations-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994 + - name: implementations-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97 value: '{"args":["--executor_input","{{$}}","--function_to_execute","python_artifact_printer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -30,7 +30,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef python_artifact_printer(artifact: Optional[Input[Artifact]] = None):\n if artifact is not None:\n print(artifact.name)\n print(artifact.uri)\n print(artifact.metadata)\n else:\n print(''No - artifact provided!'')\n\n"],"image":"python:3.9"}' + artifact provided!'')\n\n"],"image":"python:3.11"}' - name: components-comp-inner-pipeline value: '{"dag":{"tasks":{"python-artifact-printer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-python-artifact-printer"},"inputs":{"artifacts":{"artifact":{"componentInputArtifact":"dataset"}}},"taskInfo":{"name":"python-artifact-printer"}}}},"inputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isOptional":true}}}}' - name: components-comp-inner-pipeline-2 @@ -217,11 +217,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994}}' + value: '{{workflow.parameters.components-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-python-artifact-printer"},"inputs":{"artifacts":{"artifact":{"componentInputArtifact":"dataset"}}},"taskInfo":{"name":"python-artifact-printer"}}' - name: container - value: '{{workflow.parameters.implementations-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994}}' + value: '{{workflow.parameters.implementations-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97}}' - name: task-name value: python-artifact-printer - name: parent-dag-id @@ -249,11 +249,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994}}' + value: '{{workflow.parameters.components-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-python-artifact-printer-2"},"inputs":{"artifacts":{"artifact":{"componentInputArtifact":"dataset"}}},"taskInfo":{"name":"python-artifact-printer"}}' - name: container - value: '{{workflow.parameters.implementations-b375628bfbe3297e8615b48608c3bb9e5702145c781f5d43ce35416859ad7994}}' + value: '{{workflow.parameters.implementations-538891c59305f39fa7d0dc686fc88b88affbb35a10fd4ada0fe6e7b5e9136e97}}' - name: task-name value: python-artifact-printer - name: parent-dag-id diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index b683eeaf674..5f78700b94a 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-7af5009b3d45dcf8a4432f8c675e83fecaf54125510a1a4ec938ecaadc07eac0 + - name: components-5738fa6a8034d33cd840d7fed5885fd1870a2c63f8e79df3adc50a53e949c021 value: '{"executorLabel":"exec-concat-message","inputDefinitions":{"parameters":{"message1":{"parameterType":"STRING"},"message2":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-7af5009b3d45dcf8a4432f8c675e83fecaf54125510a1a4ec938ecaadc07eac0 + - name: implementations-5738fa6a8034d33cd840d7fed5885fd1870a2c63f8e79df3adc50a53e949c021 value: '{"args":["--executor_input","{{$}}","--function_to_execute","concat_message"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,7 +19,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef concat_message(message1: str, message2: str) -\u003e str:\n return message1 - + message2\n\n"],"image":"python:3.9"}' + + message2\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"concat-message"}}}},"tasks":{"concat-message":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-message"},"inputs":{"parameters":{"message1":{"componentInputParameter":"message1"},"message2":{"componentInputParameter":"message2"}}},"taskInfo":{"name":"concat-message"}}}},"inputDefinitions":{"parameters":{"message1":{"parameterType":"STRING"},"message2":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -202,11 +202,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7af5009b3d45dcf8a4432f8c675e83fecaf54125510a1a4ec938ecaadc07eac0}}' + value: '{{workflow.parameters.components-5738fa6a8034d33cd840d7fed5885fd1870a2c63f8e79df3adc50a53e949c021}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-message"},"inputs":{"parameters":{"message1":{"componentInputParameter":"message1"},"message2":{"componentInputParameter":"message2"}}},"taskInfo":{"name":"concat-message"}}' - name: container - value: '{{workflow.parameters.implementations-7af5009b3d45dcf8a4432f8c675e83fecaf54125510a1a4ec938ecaadc07eac0}}' + value: '{{workflow.parameters.implementations-5738fa6a8034d33cd840d7fed5885fd1870a2c63f8e79df3adc50a53e949c021}}' - name: task-name value: concat-message - name: parent-dag-id diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index f630cb4121d..de6da4fcd9a 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428 + - name: components-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3 value: '{"executorLabel":"exec-add","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428 + - name: implementations-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,12 +18,12 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.9"}' + add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-4 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--add-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"add"}}}},"tasks":{"add":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-Output"}}},"taskInfo":{"name":"add"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--for-loop-2-pipelinechannel--double-Output":{"parameterType":"LIST"},"pipelinechannel--threshold":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--add-Output":{"parameterType":"LIST"}}}}' - - name: components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,7 +33,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.9"}' + double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.11"}' - name: components-comp-condition-3 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--double-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"double"}}}},"tasks":{"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--threshold":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--double-Output":{"parameterType":"LIST"}}}}' - name: components-comp-for-loop-2 @@ -223,11 +223,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428}}' + value: '{{workflow.parameters.components-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-Output"}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428}}' + value: '{{workflow.parameters.implementations-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3}}' - name: task-name value: add - name: parent-dag-id @@ -255,11 +255,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 5ee2195e24f..988c29f1c95 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -6,10 +6,10 @@ metadata: spec: arguments: parameters: - - name: components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d + - name: components-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093 value: '{"executorLabel":"exec-hello-world-container"}' - - name: implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d - value: '{"command":["echo","hello world"],"image":"python:3.9"}' + - name: implementations-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093 + value: '{"command":["echo","hello world"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"hello-world-container":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world-container"},"taskInfo":{"name":"hello-world-container"}}}}}' entrypoint: entrypoint @@ -192,11 +192,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' + value: '{{workflow.parameters.components-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world-container"},"taskInfo":{"name":"hello-world-container"}}' - name: container - value: '{{workflow.parameters.implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' + value: '{{workflow.parameters.implementations-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093}}' - name: task-name value: hello-world-container - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index 5cc4408ad8c..8153c4eeded 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -6,10 +6,10 @@ metadata: spec: arguments: parameters: - - name: components-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105 + - name: components-685d68a4c481be1028be8fd6b26bb6eb8f637d4d7984d7a9f1a767bdafbc0275 value: '{"executorLabel":"exec-container-io","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"output_path":{"parameterType":"STRING"}}}}' - - name: implementations-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105 - value: '{"args":["--output_path","{{$.outputs.parameters[''output_path''].output_file}}"],"command":["my_program","{{$.inputs.parameters[''text'']}}"],"image":"python:3.9"}' + - name: implementations-685d68a4c481be1028be8fd6b26bb6eb8f637d4d7984d7a9f1a767bdafbc0275 + value: '{"args":["--output_path","{{$.outputs.parameters[''output_path''].output_file}}"],"command":["my_program","{{$.inputs.parameters[''text'']}}"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"output_path":{"valueFromParameter":{"outputParameterKey":"output_path","producerSubtask":"container-io"}}}},"tasks":{"container-io":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-io"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"container-io"}}}},"inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"output_path":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -192,11 +192,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' + value: '{{workflow.parameters.components-685d68a4c481be1028be8fd6b26bb6eb8f637d4d7984d7a9f1a767bdafbc0275}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-io"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"container-io"}}' - name: container - value: '{{workflow.parameters.implementations-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' + value: '{{workflow.parameters.implementations-685d68a4c481be1028be8fd6b26bb6eb8f637d4d7984d7a9f1a767bdafbc0275}}' - name: task-name value: container-io - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 34520baec79..c6016ecf100 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -6,10 +6,10 @@ metadata: spec: arguments: parameters: - - name: components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d + - name: components-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093 value: '{"executorLabel":"exec-container-no-input"}' - - name: implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d - value: '{"command":["echo","hello world"],"image":"python:3.9"}' + - name: implementations-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093 + value: '{"command":["echo","hello world"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"container-no-input":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-no-input"},"taskInfo":{"name":"container-no-input"}}}}}' entrypoint: entrypoint @@ -192,11 +192,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' + value: '{{workflow.parameters.components-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-no-input"},"taskInfo":{"name":"container-no-input"}}' - name: container - value: '{{workflow.parameters.implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' + value: '{{workflow.parameters.implementations-38c5a29d4ca763681a94a05486f63b9b66ef520343be3e17ca8cd892801aa093}}' - name: task-name value: container-no-input - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index e16120c6d4e..def6e2f6868 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -6,11 +6,11 @@ metadata: spec: arguments: parameters: - - name: components-b8fd25671b4c5e37ac7a8a2dd9fddc33d348d63dbc1c43627760eaa412e161d7 + - name: components-0f31c7277c0aefdce4b0699271966a6e11bb78bb3df17d5dc950a54509d02679 value: '{"executorLabel":"exec-container-with-concat-placeholder","inputDefinitions":{"parameters":{"text1":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"text2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_path":{"parameterType":"STRING"}}}}' - - name: implementations-b8fd25671b4c5e37ac7a8a2dd9fddc33d348d63dbc1c43627760eaa412e161d7 + - name: implementations-0f31c7277c0aefdce4b0699271966a6e11bb78bb3df17d5dc950a54509d02679 value: '{"args":["--output_path","{{$.outputs.parameters[''output_path''].output_file}}"],"command":["my_program","{\"Concat\": - [\"prefix-\", \"{{$.inputs.parameters[''text1'']}}\", \"{{$.outputs.artifacts[''text2''].uri}}\"]}"],"image":"python:3.9"}' + [\"prefix-\", \"{{$.inputs.parameters[''text1'']}}\", \"{{$.outputs.artifacts[''text2''].uri}}\"]}"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"text2":{"artifactSelectors":[{"outputArtifactKey":"text2","producerSubtask":"container-with-concat-placeholder"}]}},"parameters":{"output_path":{"valueFromParameter":{"outputParameterKey":"output_path","producerSubtask":"container-with-concat-placeholder"}}}},"tasks":{"container-with-concat-placeholder":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-concat-placeholder"},"inputs":{"parameters":{"text1":{"componentInputParameter":"text1"}}},"taskInfo":{"name":"container-with-concat-placeholder"}}}},"inputDefinitions":{"parameters":{"text1":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"text2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_path":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -193,11 +193,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b8fd25671b4c5e37ac7a8a2dd9fddc33d348d63dbc1c43627760eaa412e161d7}}' + value: '{{workflow.parameters.components-0f31c7277c0aefdce4b0699271966a6e11bb78bb3df17d5dc950a54509d02679}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-concat-placeholder"},"inputs":{"parameters":{"text1":{"componentInputParameter":"text1"}}},"taskInfo":{"name":"container-with-concat-placeholder"}}' - name: container - value: '{{workflow.parameters.implementations-b8fd25671b4c5e37ac7a8a2dd9fddc33d348d63dbc1c43627760eaa412e161d7}}' + value: '{{workflow.parameters.implementations-0f31c7277c0aefdce4b0699271966a6e11bb78bb3df17d5dc950a54509d02679}}' - name: task-name value: container-with-concat-placeholder - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 3d2c02d06b4..07b6bfcea51 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -6,13 +6,13 @@ metadata: spec: arguments: parameters: - - name: components-9bb02daefc2bb7fc6a8bb6f5d1f2c8b7d7feafc2aba8e1117287f6a4a6dfb89f + - name: components-d97b94290cc248e70e263031ba3a32938896cb7df7937b14caf50d8967631256 value: '{"executorLabel":"exec-container-with-if-placeholder","inputDefinitions":{"parameters":{"optional_input":{"defaultValue":"default","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_path":{"parameterType":"STRING"}}}}' - - name: implementations-9bb02daefc2bb7fc6a8bb6f5d1f2c8b7d7feafc2aba8e1117287f6a4a6dfb89f + - name: implementations-d97b94290cc248e70e263031ba3a32938896cb7df7937b14caf50d8967631256 value: '{"args":["--output_path","{{$.outputs.parameters[''output_path''].output_file}}"],"command":["my_program","{\"IfPresent\": {\"InputName\": \"optional_input\", \"Then\": [\"{{$.inputs.parameters[''optional_input'']}}\"], \"Else\": [\"bye\"]}}","--dataset","{\"IfPresent\": {\"InputName\": \"optional_input\", - \"Then\": [\"{{$.outputs.artifacts[''dataset''].uri}}\"], \"Else\": [\"bye\"]}}"],"image":"python:3.9"}' + \"Then\": [\"{{$.outputs.artifacts[''dataset''].uri}}\"], \"Else\": [\"bye\"]}}"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"dataset":{"artifactSelectors":[{"outputArtifactKey":"dataset","producerSubtask":"container-with-if-placeholder"}]}},"parameters":{"output_path":{"valueFromParameter":{"outputParameterKey":"output_path","producerSubtask":"container-with-if-placeholder"}}}},"tasks":{"container-with-if-placeholder":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-if-placeholder"},"inputs":{"parameters":{"optional_input":{"componentInputParameter":"optional_input"}}},"taskInfo":{"name":"container-with-if-placeholder"}}}},"inputDefinitions":{"parameters":{"optional_input":{"defaultValue":"default","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_path":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -195,11 +195,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9bb02daefc2bb7fc6a8bb6f5d1f2c8b7d7feafc2aba8e1117287f6a4a6dfb89f}}' + value: '{{workflow.parameters.components-d97b94290cc248e70e263031ba3a32938896cb7df7937b14caf50d8967631256}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-if-placeholder"},"inputs":{"parameters":{"optional_input":{"componentInputParameter":"optional_input"}}},"taskInfo":{"name":"container-with-if-placeholder"}}' - name: container - value: '{{workflow.parameters.implementations-9bb02daefc2bb7fc6a8bb6f5d1f2c8b7d7feafc2aba8e1117287f6a4a6dfb89f}}' + value: '{{workflow.parameters.implementations-d97b94290cc248e70e263031ba3a32938896cb7df7937b14caf50d8967631256}}' - name: task-name value: container-with-if-placeholder - name: parent-dag-id diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 867fccdf842..2742a7d49b7 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -6,10 +6,10 @@ metadata: spec: arguments: parameters: - - name: components-f76f16c424ed80e87d38bb55b27a483700ad3a6f1fe4c5deccb44812db875f5f + - name: components-79db8839d07686ed239d9057400d121034026b267540e3591bf32d0e1237cb52 value: '{"executorLabel":"exec-container-with-placeholder-in-fstring","inputDefinitions":{"parameters":{"text1":{"defaultValue":"text!","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-f76f16c424ed80e87d38bb55b27a483700ad3a6f1fe4c5deccb44812db875f5f - value: '{"command":["my_program","prefix-{{$.inputs.parameters[''text1'']}}","{{$.outputs.artifacts[''output_artifact''].uri}}/0"],"image":"python:3.9"}' + - name: implementations-79db8839d07686ed239d9057400d121034026b267540e3591bf32d0e1237cb52 + value: '{"command":["my_program","prefix-{{$.inputs.parameters[''text1'']}}","{{$.outputs.artifacts[''output_artifact''].uri}}/0"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"output_artifact":{"artifactSelectors":[{"outputArtifactKey":"output_artifact","producerSubtask":"container-with-placeholder-in-fstring"}]}}},"tasks":{"container-with-placeholder-in-fstring":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-placeholder-in-fstring"},"inputs":{"parameters":{"text1":{"componentInputParameter":"text1"}}},"taskInfo":{"name":"container-with-placeholder-in-fstring"}}}},"inputDefinitions":{"parameters":{"text1":{"defaultValue":"text!","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_artifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' entrypoint: entrypoint @@ -192,11 +192,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f76f16c424ed80e87d38bb55b27a483700ad3a6f1fe4c5deccb44812db875f5f}}' + value: '{{workflow.parameters.components-79db8839d07686ed239d9057400d121034026b267540e3591bf32d0e1237cb52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-with-placeholder-in-fstring"},"inputs":{"parameters":{"text1":{"componentInputParameter":"text1"}}},"taskInfo":{"name":"container-with-placeholder-in-fstring"}}' - name: container - value: '{{workflow.parameters.implementations-f76f16c424ed80e87d38bb55b27a483700ad3a6f1fe4c5deccb44812db875f5f}}' + value: '{{workflow.parameters.implementations-79db8839d07686ed239d9057400d121034026b267540e3591bf32d0e1237cb52}}' - name: task-name value: container-with-placeholder-in-fstring - name: parent-dag-id diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index f60c847db20..5d4ab18da5c 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-c78f6e9b1302a8641c86310a2b303eb6fe40bc7225ccb38e15cd89cd412a77e1 + - name: components-ddf5c44b745eff77bd01b2badb8eb0ad37a114f732779104288f59e67701512f value: '{"executorLabel":"exec-validate-no-pod-metadata","inputDefinitions":{"parameters":{"annotation_path":{"parameterType":"STRING"},"label_path":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-c78f6e9b1302a8641c86310a2b303eb6fe40bc7225ccb38e15cd89cd412a77e1 + - name: implementations-ddf5c44b745eff77bd01b2badb8eb0ad37a114f732779104288f59e67701512f value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_no_pod_metadata"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -23,12 +23,12 @@ spec: ValueError(\n f\"Pod annotation is {annotation} but is supposed to be None.\"\n )\n label = os.getenv(label_path)\n if label != \"\":\n raise ValueError(\n f\"Pod label is {label} but - is supposed to be None.\"\n )\n return True\n\n"],"image":"python:3.9"}' + is supposed to be None.\"\n )\n return True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation'']","name":"POD_TASK_ANNOTATION"},{"fieldPath":"metadata.labels[''task-label-1'']","name":"POD_TASK_LABEL_1"},{"fieldPath":"metadata.labels[''task-label-2'']","name":"POD_TASK_LABEL_2"}],"podMetadata":{"annotations":{"task-annotation":"annotation"},"labels":{"task-label-1":"label-1","task-label-2":"label-2"}}}' - - name: components-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc + - name: components-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735 value: '{"executorLabel":"exec-validate-pod-metadata","inputDefinitions":{"parameters":{"annotation_exp_val_1":{"isOptional":true,"parameterType":"STRING"},"annotation_exp_val_2":{"isOptional":true,"parameterType":"STRING"},"annotation_path_1":{"isOptional":true,"parameterType":"STRING"},"annotation_path_2":{"isOptional":true,"parameterType":"STRING"},"label_exp_val_1":{"isOptional":true,"parameterType":"STRING"},"label_exp_val_2":{"isOptional":true,"parameterType":"STRING"},"label_path_1":{"isOptional":true,"parameterType":"STRING"},"label_path_2":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc + - name: implementations-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_pod_metadata"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -60,7 +60,7 @@ spec: ValueError(\n f\"Pod label is {label_val_2} but is supposed to be {label_exp_val_2}.\"\n )\n validated_metadata_count+=1\n if validated_metadata_count \u003c1:\n raise RuntimeError(f\"No pod metadata - found to validate.\")\n return True\n\n"],"image":"python:3.9"}' + found to validate.\")\n return True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata-2 value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation-1'']","name":"POD_TASK_ANNOTATION_1"},{"fieldPath":"metadata.annotations[''task-annotation-2'']","name":"POD_TASK_ANNOTATION_2"}],"podMetadata":{"annotations":{"task-annotation-1":"annotation-1","task-annotation-2":"annotation-2"}}}' - name: components-root @@ -473,11 +473,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c78f6e9b1302a8641c86310a2b303eb6fe40bc7225ccb38e15cd89cd412a77e1}}' + value: '{{workflow.parameters.components-ddf5c44b745eff77bd01b2badb8eb0ad37a114f732779104288f59e67701512f}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-no-pod-metadata"},"inputs":{"parameters":{"annotation_path":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION"}},"label_path":{"runtimeValue":{"constant":"POD_TASK_LABEL"}}}},"taskInfo":{"name":"validate-no-pod-metadata"}}' - name: container - value: '{{workflow.parameters.implementations-c78f6e9b1302a8641c86310a2b303eb6fe40bc7225ccb38e15cd89cd412a77e1}}' + value: '{{workflow.parameters.implementations-ddf5c44b745eff77bd01b2badb8eb0ad37a114f732779104288f59e67701512f}}' - name: task-name value: validate-no-pod-metadata - name: parent-dag-id @@ -497,11 +497,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc}}' + value: '{{workflow.parameters.components-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION"}},"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}}}},"taskInfo":{"name":"validate-pod-metadata"}}' - name: container - value: '{{workflow.parameters.implementations-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc}}' + value: '{{workflow.parameters.implementations-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735}}' - name: task-name value: validate-pod-metadata - name: parent-dag-id @@ -535,11 +535,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc}}' + value: '{{workflow.parameters.components-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-2"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation-1"}},"annotation_exp_val_2":{"runtimeValue":{"constant":"annotation-2"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_1"}},"annotation_path_2":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_2"}}}},"taskInfo":{"name":"validate-pod-metadata-2"}}' - name: container - value: '{{workflow.parameters.implementations-41f6db675508ce85c4c81ffc8f4f63358027f85ec0bf31f36946cf1ede3c43cc}}' + value: '{{workflow.parameters.implementations-c18d45fda14ac35e1b8c0d82357e972f4d416ae8d9ce79aef5cd5f31cb0e7735}}' - name: task-name value: validate-pod-metadata-2 - name: parent-dag-id diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index 922d86dab45..4f3b1699329 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac + - name: components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b value: '{"executorLabel":"exec-print-op-5","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac + - name: implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str):\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str):\n print(message)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-10 value: '{"dag":{"tasks":{"print-op-5":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"five"}}}},"taskInfo":{"name":"print-op-5"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-9":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-comp-for-loop-14 @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"five"}}}},"taskInfo":{"name":"print-op-5"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-5 - name: parent-dag-id @@ -254,11 +254,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-7"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"seven"}}}},"taskInfo":{"name":"print-op-7"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-7 - name: parent-dag-id @@ -430,11 +430,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-8"},"dependentTasks":["for-loop-14"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"eight"}}}},"taskInfo":{"name":"print-op-8"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-8 - name: parent-dag-id @@ -463,11 +463,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"one"}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op - name: parent-dag-id @@ -495,11 +495,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"two"}}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-2 - name: parent-dag-id @@ -527,11 +527,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"four"}}}},"taskInfo":{"name":"print-op-4"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-4 - name: parent-dag-id @@ -627,11 +627,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"three"}}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-3 - name: parent-dag-id @@ -995,11 +995,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.components-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-6"},"dependentTasks":["for-loop-10"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"six"}}}},"taskInfo":{"name":"print-op-6"}}' - name: container - value: '{{workflow.parameters.implementations-e12164b597ed5229b5b77b574180112348cfc79cceecf6121b99dfb6b742f5ac}}' + value: '{{workflow.parameters.implementations-2a367e5abfe5ee4a63da037b9312bc5427fac22774eda49990b500ddc4de005b}}' - name: task-name value: print-op-6 - name: parent-dag-id diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 2f441916122..30dc39ed68d 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-e11c23af71b1b107d01e28b0c31e9994727d3eff7eef1dbb87fdd8868ff80e84 + - name: components-35671413d3dd82981b2a90982070716c0f9c16927eb0c60333b097b2d5a6774b value: '{"executorLabel":"exec-dict-input","inputDefinitions":{"parameters":{"struct":{"parameterType":"STRUCT"}}}}' - - name: implementations-e11c23af71b1b107d01e28b0c31e9994727d3eff7eef1dbb87fdd8868ff80e84 + - name: implementations-35671413d3dd82981b2a90982070716c0f9c16927eb0c60333b097b2d5a6774b value: '{"args":["--executor_input","{{$}}","--function_to_execute","dict_input"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - dict_input(struct: Dict):\n print(struct)\n\n"],"image":"python:3.9"}' + dict_input(struct: Dict):\n print(struct)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"dict-input":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dict-input"},"inputs":{"parameters":{"struct":{"componentInputParameter":"struct"}}},"taskInfo":{"name":"dict-input"}}}},"inputDefinitions":{"parameters":{"struct":{"parameterType":"STRUCT"}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e11c23af71b1b107d01e28b0c31e9994727d3eff7eef1dbb87fdd8868ff80e84}}' + value: '{{workflow.parameters.components-35671413d3dd82981b2a90982070716c0f9c16927eb0c60333b097b2d5a6774b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dict-input"},"inputs":{"parameters":{"struct":{"componentInputParameter":"struct"}}},"taskInfo":{"name":"dict-input"}}' - name: container - value: '{{workflow.parameters.implementations-e11c23af71b1b107d01e28b0c31e9994727d3eff7eef1dbb87fdd8868ff80e84}}' + value: '{{workflow.parameters.implementations-35671413d3dd82981b2a90982070716c0f9c16927eb0c60333b097b2d5a6774b}}' - name: task-name value: dict-input - name: parent-dag-id diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index b0bb3f5ee63..3cc641ca925 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-228cf9424bf53a0e969ad4299e33fa7d2c0cebe754627c59bd00c710b00f88fa + - name: components-ffbce4c56102eddc51d0cb2aaa352ac140f1bbf010042621313e0144a79c7a20 value: '{"executorLabel":"exec-read-embedded-artifact-dir"}' - - name: implementations-228cf9424bf53a0e969ad4299e33fa7d2c0cebe754627c59bd00c710b00f88fa + - name: implementations-ffbce4c56102eddc51d0cb2aaa352ac140f1bbf010042621313e0144a79c7a20 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_embedded_artifact_dir"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,10 +32,10 @@ spec: __KFP_EMBEDDED_ASSET_DIR)\n\n\n\n\ndef read_embedded_artifact_dir(artifact: dsl.EmbeddedInput[dsl.Dataset]):\n import os\n\n with open(os.path.join(artifact.path, \"log.txt\"), \"r\", encoding=\"utf-8\") as f:\n log = f.read()\n\n assert - log == \"Hello, world!\"\n\n"],"image":"python:3.9"}' - - name: components-302f28ee42e44910a1eb45593566a12b430150b643fb02ded3ae7838c0cea331 + log == \"Hello, world!\"\n\n"],"image":"python:3.11"}' + - name: components-51efdf1bda01221dc8fa240e519ad3100e7bd478949411c505a8925e19f655a1 value: '{"executorLabel":"exec-read-embedded-artifact-file"}' - - name: implementations-302f28ee42e44910a1eb45593566a12b430150b643fb02ded3ae7838c0cea331 + - name: implementations-51efdf1bda01221dc8fa240e519ad3100e7bd478949411c505a8925e19f655a1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_embedded_artifact_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -59,7 +59,7 @@ spec: __KFP_EMBEDDED_ASSET_DIR)\n\n__KFP_EMBEDDED_ASSET_FILE = __kfp_os.path.join(__KFP_EMBEDDED_ASSET_DIR, ''log.txt'')\n\n\n\n\ndef read_embedded_artifact_file(artifact: dsl.EmbeddedInput[dsl.Dataset]):\n with open(artifact.path, \"r\", encoding=\"utf-8\") as f:\n log = f.read()\n\n assert - log == \"Hello, world!\"\n\n"],"image":"python:3.9"}' + log == \"Hello, world!\"\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"read-embedded-artifact-dir":{"cachingOptions":{},"componentRef":{"name":"comp-read-embedded-artifact-dir"},"taskInfo":{"name":"read-embedded-artifact-dir"}},"read-embedded-artifact-file":{"cachingOptions":{},"componentRef":{"name":"comp-read-embedded-artifact-file"},"taskInfo":{"name":"read-embedded-artifact-file"}}}}}' entrypoint: entrypoint @@ -242,11 +242,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-228cf9424bf53a0e969ad4299e33fa7d2c0cebe754627c59bd00c710b00f88fa}}' + value: '{{workflow.parameters.components-ffbce4c56102eddc51d0cb2aaa352ac140f1bbf010042621313e0144a79c7a20}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-embedded-artifact-dir"},"taskInfo":{"name":"read-embedded-artifact-dir"}}' - name: container - value: '{{workflow.parameters.implementations-228cf9424bf53a0e969ad4299e33fa7d2c0cebe754627c59bd00c710b00f88fa}}' + value: '{{workflow.parameters.implementations-ffbce4c56102eddc51d0cb2aaa352ac140f1bbf010042621313e0144a79c7a20}}' - name: task-name value: read-embedded-artifact-dir - name: parent-dag-id @@ -266,11 +266,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-302f28ee42e44910a1eb45593566a12b430150b643fb02ded3ae7838c0cea331}}' + value: '{{workflow.parameters.components-51efdf1bda01221dc8fa240e519ad3100e7bd478949411c505a8925e19f655a1}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-embedded-artifact-file"},"taskInfo":{"name":"read-embedded-artifact-file"}}' - name: container - value: '{{workflow.parameters.implementations-302f28ee42e44910a1eb45593566a12b430150b643fb02ded3ae7838c0cea331}}' + value: '{{workflow.parameters.implementations-51efdf1bda01221dc8fa240e519ad3100e7bd478949411c505a8925e19f655a1}}' - name: task-name value: read-embedded-artifact-file - name: parent-dag-id diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index 4235a603fed..b025c45c82f 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-b568fbdaaed9b6c215b65a277dc1107af4099ba40aa61cffdbae497666a77ba4 + - name: components-29331b5087411692b16ea3f55f802cd64bc8edfcbcf0db7f6592cc1b91685487 value: '{"executorLabel":"exec-fail"}' - - name: implementations-b568fbdaaed9b6c215b65a277dc1107af4099ba40aa61cffdbae497666a77ba4 + - name: implementations-29331b5087411692b16ea3f55f802cd64bc8edfcbcf0db7f6592cc1b91685487 value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - fail():\n ''''''Fails''''''\n import sys\n sys.exit(1)\n\n"],"image":"python:3.9"}' + fail():\n ''''''Fails''''''\n import sys\n sys.exit(1)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"fail":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail"},"taskInfo":{"name":"fail"}}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b568fbdaaed9b6c215b65a277dc1107af4099ba40aa61cffdbae497666a77ba4}}' + value: '{{workflow.parameters.components-29331b5087411692b16ea3f55f802cd64bc8edfcbcf0db7f6592cc1b91685487}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail"},"taskInfo":{"name":"fail"}}' - name: container - value: '{{workflow.parameters.implementations-b568fbdaaed9b6c215b65a277dc1107af4099ba40aa61cffdbae497666a77ba4}}' + value: '{{workflow.parameters.implementations-29331b5087411692b16ea3f55f802cd64bc8edfcbcf0db7f6592cc1b91685487}}' - name: task-name value: fail - name: parent-dag-id diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index 6ca409a0b31..3267c16b16b 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766 + - name: components-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582 value: '{"executorLabel":"exec-print-msg","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766 + - name: implementations-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_msg"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.9"}' + print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"tasks":{"print-msg":{"cachingOptions":{},"componentRef":{"name":"comp-print-msg"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"heads and {{$.inputs.parameters[''pipelinechannel--random-num-Output'']}} \u003e @@ -27,9 +27,9 @@ spec: value: '{"dag":{"tasks":{"print-msg-2":{"cachingOptions":{},"componentRef":{"name":"comp-print-msg-2"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"heads and {{$.inputs.parameters[''pipelinechannel--random-num-Output'']}} \u003c= 5!"}},"pipelinechannel--random-num-Output":{"componentInputParameter":"pipelinechannel--random-num-Output"}}},"taskInfo":{"name":"print-msg-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"},"pipelinechannel--random-num-Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: components-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b + - name: components-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0 value: '{"executorLabel":"exec-random-num","inputDefinitions":{"parameters":{"high":{"parameterType":"NUMBER_INTEGER"},"low":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b + - name: implementations-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","random_num"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -41,7 +41,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef random_num(low: int, high: int) -\u003e int:\n \"\"\"Generate a random number between low and high.\"\"\"\n import random # noqa: PLC0415\n\n result - = random.randint(low, high)\n print(result)\n return result\n\n"],"image":"python:3.9"}' + = random.randint(low, high)\n print(result)\n return result\n\n"],"image":"python:3.11"}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"condition-2":{"componentRef":{"name":"comp-condition-2"},"dependentTasks":["random-num"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"},"pipelinechannel--random-num-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"random-num"}}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--random-num-Output'']) \u003e 5"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"dependentTasks":["random-num"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"},"pipelinechannel--random-num-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"random-num"}}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--random-num-Output'']) @@ -58,9 +58,9 @@ spec: value: '{"dag":{"tasks":{"condition-5":{"componentRef":{"name":"comp-condition-5"},"dependentTasks":["random-num-2"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"},"pipelinechannel--random-num-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"random-num-2"}}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--random-num-2-Output'']) \u003e 15"}},"condition-6":{"componentRef":{"name":"comp-condition-6"},"dependentTasks":["random-num-2"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"},"pipelinechannel--random-num-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"random-num-2"}}}},"taskInfo":{"name":"condition-6"},"triggerPolicy":{"condition":"int(inputs.parameter_values[''pipelinechannel--random-num-2-Output'']) \u003c= 15"}},"random-num-2":{"cachingOptions":{},"componentRef":{"name":"comp-random-num-2"},"inputs":{"parameters":{"high":{"runtimeValue":{"constant":19}},"low":{"runtimeValue":{"constant":10}}}},"taskInfo":{"name":"random-num-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}}}' - - name: components-8febb76ff31d4bcdea4ae0c2f886b818c1d766cfe9bec311de0f57970d0ccb9b + - name: components-6175eb183c8ba935a4c42b40e65a5c3e4623a9c16b38fd5c4bbfc23275f0e9bd value: '{"executorLabel":"exec-flip-coin","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-8febb76ff31d4bcdea4ae0c2f886b818c1d766cfe9bec311de0f57970d0ccb9b + - name: implementations-6175eb183c8ba935a4c42b40e65a5c3e4623a9c16b38fd5c4bbfc23275f0e9bd value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -73,7 +73,7 @@ spec: flip_coin() -\u003e str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random # noqa: PLC0415\n\n result = \"heads\" if random.randint(0, 1) == 0 else \"tails\"\n print(result)\n return - result\n\n"],"image":"python:3.9"}' + result\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"condition-1":{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["flip-coin"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin"}}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads''"}},"condition-4":{"componentRef":{"name":"comp-condition-4"},"dependentTasks":["flip-coin"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin"}}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-coin-Output''] @@ -258,13 +258,13 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.components-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-msg"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"heads and {{$.inputs.parameters[''pipelinechannel--random-num-Output'']}} \u003e 5!"}},"pipelinechannel--random-num-Output":{"componentInputParameter":"pipelinechannel--random-num-Output"}}},"taskInfo":{"name":"print-msg"}}' - name: container - value: '{{workflow.parameters.implementations-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.implementations-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task-name value: print-msg - name: parent-dag-id @@ -292,13 +292,13 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.components-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-msg-2"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"heads and {{$.inputs.parameters[''pipelinechannel--random-num-Output'']}} \u003c= 5!"}},"pipelinechannel--random-num-Output":{"componentInputParameter":"pipelinechannel--random-num-Output"}}},"taskInfo":{"name":"print-msg-2"}}' - name: container - value: '{{workflow.parameters.implementations-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.implementations-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task-name value: print-msg-2 - name: parent-dag-id @@ -451,11 +451,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b}}' + value: '{{workflow.parameters.components-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-random-num"},"inputs":{"parameters":{"high":{"runtimeValue":{"constant":9}},"low":{"runtimeValue":{"constant":0}}}},"taskInfo":{"name":"random-num"}}' - name: container - value: '{{workflow.parameters.implementations-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b}}' + value: '{{workflow.parameters.implementations-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0}}' - name: task-name value: random-num - name: parent-dag-id @@ -483,13 +483,13 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.components-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-msg-3"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"tails and {{$.inputs.parameters[''pipelinechannel--random-num-2-Output'']}} \u003e 15!"}},"pipelinechannel--random-num-2-Output":{"componentInputParameter":"pipelinechannel--random-num-2-Output"}}},"taskInfo":{"name":"print-msg-3"}}' - name: container - value: '{{workflow.parameters.implementations-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.implementations-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task-name value: print-msg-3 - name: parent-dag-id @@ -517,13 +517,13 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.components-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-msg-4"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"tails and {{$.inputs.parameters[''pipelinechannel--random-num-2-Output'']}} \u003c= 15!"}},"pipelinechannel--random-num-2-Output":{"componentInputParameter":"pipelinechannel--random-num-2-Output"}}},"taskInfo":{"name":"print-msg-4"}}' - name: container - value: '{{workflow.parameters.implementations-66f7b70943b22b0e93ab97e737d33d87c19e743933c63d7573ec6f55617cb766}}' + value: '{{workflow.parameters.implementations-6d9fd4a4d18480bfc98d1bbb174a6aaea617e354b4ed0efcc0eadc2414b1e582}}' - name: task-name value: print-msg-4 - name: parent-dag-id @@ -599,11 +599,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b}}' + value: '{{workflow.parameters.components-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-random-num-2"},"inputs":{"parameters":{"high":{"runtimeValue":{"constant":19}},"low":{"runtimeValue":{"constant":10}}}},"taskInfo":{"name":"random-num-2"}}' - name: container - value: '{{workflow.parameters.implementations-e6fdb657c103ed2cbd67c3611182d3961f70724e37166e8b94304cf95b0c736b}}' + value: '{{workflow.parameters.implementations-2c52a38e80b2a374e3ca25334ecf08a91b2565a0eb9fe88129b5d3b4392493b0}}' - name: task-name value: random-num-2 - name: parent-dag-id @@ -679,11 +679,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8febb76ff31d4bcdea4ae0c2f886b818c1d766cfe9bec311de0f57970d0ccb9b}}' + value: '{{workflow.parameters.components-6175eb183c8ba935a4c42b40e65a5c3e4623a9c16b38fd5c4bbfc23275f0e9bd}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}}' - name: container - value: '{{workflow.parameters.implementations-8febb76ff31d4bcdea4ae0c2f886b818c1d766cfe9bec311de0f57970d0ccb9b}}' + value: '{{workflow.parameters.implementations-6175eb183c8ba935a4c42b40e65a5c3e4623a9c16b38fd5c4bbfc23275f0e9bd}}' - name: task-name value: flip-coin - name: parent-dag-id diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index fd3aaa1099c..4901bdfd87c 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-ba6f3c33ed7f8f19467251611934d428f27109368234070d61411c092f2ccf11 + - name: components-4f72b1b68ec8567b06989de2c1a51aa5a00125003bb0b1c7dd1bb2eb07970287 value: '{"executorLabel":"exec-identity","inputDefinitions":{"parameters":{"value":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-ba6f3c33ed7f8f19467251611934d428f27109368234070d61411c092f2ccf11 + - name: implementations-4f72b1b68ec8567b06989de2c1a51aa5a00125003bb0b1c7dd1bb2eb07970287 value: '{"args":["--executor_input","{{$}}","--function_to_execute","identity"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - identity(value: str) -\u003e str:\n return value\n\n"],"image":"python:3.9"}' + identity(value: str) -\u003e str:\n return value\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"identity"}}}},"tasks":{"identity":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-identity"},"inputs":{"parameters":{"value":{"componentInputParameter":"value"}}},"taskInfo":{"name":"identity"}}}},"inputDefinitions":{"parameters":{"value":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ba6f3c33ed7f8f19467251611934d428f27109368234070d61411c092f2ccf11}}' + value: '{{workflow.parameters.components-4f72b1b68ec8567b06989de2c1a51aa5a00125003bb0b1c7dd1bb2eb07970287}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-identity"},"inputs":{"parameters":{"value":{"componentInputParameter":"value"}}},"taskInfo":{"name":"identity"}}' - name: container - value: '{{workflow.parameters.implementations-ba6f3c33ed7f8f19467251611934d428f27109368234070d61411c092f2ccf11}}' + value: '{{workflow.parameters.implementations-4f72b1b68ec8567b06989de2c1a51aa5a00125003bb0b1c7dd1bb2eb07970287}}' - name: task-name value: identity - name: parent-dag-id diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 3ce7e18ad1f..7f09346c05b 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"executorLabel":"exec-print-and-return","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_and_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.9"}' + print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.11"}' - name: components-comp-condition-3 value: '{"dag":{"tasks":{"print-and-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Adding drumroll on last trial!"}}}},"taskInfo":{"name":"print-and-return"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--add_drumroll":{"parameterType":"BOOLEAN"},"pipelinechannel--trials-loop-item":{"parameterType":"NUMBER_INTEGER"}}}}' @@ -45,9 +45,9 @@ spec: value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--condition-branches-5-oneof-1":{"valueFromOneof":{"parameterSelectors":[{"outputParameterKey":"pipelinechannel--print-and-return-2-Output","producerSubtask":"condition-6"},{"outputParameterKey":"pipelinechannel--print-and-return-3-Output","producerSubtask":"condition-7"}]}}}},"tasks":{"condition-6":{"componentRef":{"name":"comp-condition-6"},"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"},"pipelinechannel--is-even-or-odd-Output":{"componentInputParameter":"pipelinechannel--is-even-or-odd-Output"}}},"taskInfo":{"name":"condition-6"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--is-even-or-odd-Output''] == ''even''"}},"condition-7":{"componentRef":{"name":"comp-condition-7"},"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"},"pipelinechannel--is-even-or-odd-Output":{"componentInputParameter":"pipelinechannel--is-even-or-odd-Output"}}},"taskInfo":{"name":"condition-7"},"triggerPolicy":{"condition":"!(inputs.parameter_values[''pipelinechannel--is-even-or-odd-Output''] == ''even'')"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--is-even-or-odd-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--condition-branches-5-oneof-1":{"parameterType":"STRING"}}}}' - - name: components-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f + - name: components-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb value: '{"executorLabel":"exec-is-even-or-odd","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f + - name: implementations-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb value: '{"args":["--executor_input","{{$}}","--function_to_execute","is_even_or_odd"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -58,7 +58,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef is_even_or_odd(num: int) -\u003e str:\n return ''odd'' if num % 2 else - ''even''\n\n"],"image":"python:3.9"}' + ''even''\n\n"],"image":"python:3.11"}' - name: components-comp-condition-8 value: '{"dag":{"tasks":{"condition-branches-5":{"componentRef":{"name":"comp-condition-branches-5"},"dependentTasks":["is-even-or-odd"],"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"},"pipelinechannel--is-even-or-odd-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"is-even-or-odd"}}}},"taskInfo":{"name":"condition-branches-5"}},"is-even-or-odd":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-is-even-or-odd"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"}}},"taskInfo":{"name":"is-even-or-odd"}},"print-and-return-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-4"},"dependentTasks":["condition-branches-5"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--condition-branches-5-oneof-1","producerTask":"condition-branches-5"}}}},"taskInfo":{"name":"print-and-return-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-comp-condition-11 @@ -80,9 +80,9 @@ spec: \u003c 5000"}},"condition-9":{"componentRef":{"name":"comp-condition-9"},"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"}}},"taskInfo":{"name":"condition-9"},"triggerPolicy":{"condition":"!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) \u003c 5000) \u0026\u0026 int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) \u003e 5000"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--repeat_if_lucky_number":{"parameterType":"BOOLEAN"}}}}' - - name: components-dc6d5e95e0c0be0e93cfc8471078d7b7474f5d54cf510b6f3643e0364e578c0e + - name: components-037286a2481e6e06b20c68a503b3734f6d588a2e617f51124ada32d3bfbaebc9 value: '{"executorLabel":"exec-int-0-to-9999","outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-dc6d5e95e0c0be0e93cfc8471078d7b7474f5d54cf510b6f3643e0364e578c0e + - name: implementations-037286a2481e6e06b20c68a503b3734f6d588a2e617f51124ada32d3bfbaebc9 value: '{"args":["--executor_input","{{$}}","--function_to_execute","int_0_to_9999"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -93,13 +93,13 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef int_0_to_9999() -\u003e int:\n import random\n return random.randint(0, - 9999)\n\n"],"image":"python:3.9"}' + 9999)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"int-0-to-9999"}}}},"tasks":{"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--add_drumroll":{"componentInputParameter":"pipelinechannel--add_drumroll"},"pipelinechannel--trials-loop-item":{"componentInputParameter":"pipelinechannel--trials-loop-item"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--add_drumroll''] == true"}},"condition-branches-4":{"componentRef":{"name":"comp-condition-branches-4"},"dependentTasks":["int-0-to-9999"],"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"int-0-to-9999"}},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"pipelinechannel--repeat_if_lucky_number"}}},"taskInfo":{"name":"condition-branches-4"}},"int-0-to-9999":{"cachingOptions":{},"componentRef":{"name":"comp-int-0-to-9999"},"taskInfo":{"name":"int-0-to-9999"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--add_drumroll":{"parameterType":"BOOLEAN"},"pipelinechannel--repeat_if_lucky_number":{"parameterType":"BOOLEAN"},"pipelinechannel--trials":{"parameterType":"LIST"},"pipelinechannel--trials-loop-item":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"parameterType":"LIST"}}}}' - - name: components-c48160ebe824a2b5355816c99d1f335ad55b8ba40e2f53c260c844c16bd2f6df + - name: components-7eb3c92dc3692d5ff7c6a3fee6f5d875746948a93fc517922b5ce5d9c8fb8e41 value: '{"executorLabel":"exec-print-ints","inputDefinitions":{"parameters":{"ints":{"parameterType":"LIST"}}}}' - - name: implementations-c48160ebe824a2b5355816c99d1f335ad55b8ba40e2f53c260c844c16bd2f6df + - name: implementations-7eb3c92dc3692d5ff7c6a3fee6f5d875746948a93fc517922b5ce5d9c8fb8e41 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_ints"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -109,7 +109,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_ints(ints: List[int]):\n print(ints)\n\n"],"image":"python:3.9"}' + print_ints(ints: List[int]):\n print(ints)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--add_drumroll":{"componentInputParameter":"add_drumroll"},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"repeat_if_lucky_number"},"pipelinechannel--trials":{"componentInputParameter":"trials"}}},"parameterIterator":{"itemInput":"pipelinechannel--trials-loop-item","items":{"inputParameter":"pipelinechannel--trials"}},"taskInfo":{"name":"for-loop-1"}},"print-ints":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-ints"},"dependentTasks":["for-loop-1"],"inputs":{"parameters":{"ints":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--int-0-to-9999-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"print-ints"}}}},"inputDefinitions":{"parameters":{"add_drumroll":{"defaultValue":true,"isOptional":true,"parameterType":"BOOLEAN"},"repeat_if_lucky_number":{"defaultValue":true,"isOptional":true,"parameterType":"BOOLEAN"},"trials":{"defaultValue":[1,2,3],"isOptional":true,"parameterType":"LIST"}}}}' entrypoint: entrypoint @@ -292,12 +292,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Adding drumroll on last trial!"}}}},"taskInfo":{"name":"print-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return - name: parent-dag-id @@ -433,12 +433,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-9"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Announcing again: Got the lucky number 5000! A one in 10,000 chance."}}}},"taskInfo":{"name":"print-and-return-9"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-9 - name: parent-dag-id @@ -564,12 +564,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-8"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Announcing: Got the lucky number 5000! A one in 10,000 chance."}}}},"taskInfo":{"name":"print-and-return-8"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-8 - name: parent-dag-id @@ -597,12 +597,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-2"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got a low even number!"}}}},"taskInfo":{"name":"print-and-return-2"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-2 - name: parent-dag-id @@ -630,12 +630,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-3"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got a low odd number!"}}}},"taskInfo":{"name":"print-and-return-3"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-3 - name: parent-dag-id @@ -739,11 +739,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f}}' + value: '{{workflow.parameters.components-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-is-even-or-odd"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"}}},"taskInfo":{"name":"is-even-or-odd"}}' - name: container - value: '{{workflow.parameters.implementations-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f}}' + value: '{{workflow.parameters.implementations-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb}}' - name: task-name value: is-even-or-odd - name: parent-dag-id @@ -763,11 +763,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-4"},"dependentTasks":["condition-branches-5"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--condition-branches-5-oneof-1","producerTask":"condition-branches-5"}}}},"taskInfo":{"name":"print-and-return-4"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-4 - name: parent-dag-id @@ -796,12 +796,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-5"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got a high even number!"}}}},"taskInfo":{"name":"print-and-return-5"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-5 - name: parent-dag-id @@ -829,12 +829,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-6"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got a high odd number!"}}}},"taskInfo":{"name":"print-and-return-6"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-6 - name: parent-dag-id @@ -938,11 +938,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f}}' + value: '{{workflow.parameters.components-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-is-even-or-odd-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"}}},"taskInfo":{"name":"is-even-or-odd-2"}}' - name: container - value: '{{workflow.parameters.implementations-07f4c47d015aff01f4f9e8a60254c059f980d3b1c2afd8798b34b0fece648e3f}}' + value: '{{workflow.parameters.implementations-9f03385cde2d0dd36c4de829527d6ece0aa8acc96cfc09cd66b2f670fb6c12eb}}' - name: task-name value: is-even-or-odd-2 - name: parent-dag-id @@ -962,11 +962,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-7"},"dependentTasks":["condition-branches-10"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--condition-branches-10-oneof-1","producerTask":"condition-branches-10"}}}},"taskInfo":{"name":"print-and-return-7"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-7 - name: parent-dag-id @@ -1119,11 +1119,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-dc6d5e95e0c0be0e93cfc8471078d7b7474f5d54cf510b6f3643e0364e578c0e}}' + value: '{{workflow.parameters.components-037286a2481e6e06b20c68a503b3734f6d588a2e617f51124ada32d3bfbaebc9}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-int-0-to-9999"},"taskInfo":{"name":"int-0-to-9999"}}' - name: container - value: '{{workflow.parameters.implementations-dc6d5e95e0c0be0e93cfc8471078d7b7474f5d54cf510b6f3643e0364e578c0e}}' + value: '{{workflow.parameters.implementations-037286a2481e6e06b20c68a503b3734f6d588a2e617f51124ada32d3bfbaebc9}}' - name: task-name value: int-0-to-9999 - name: parent-dag-id @@ -1216,11 +1216,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c48160ebe824a2b5355816c99d1f335ad55b8ba40e2f53c260c844c16bd2f6df}}' + value: '{{workflow.parameters.components-7eb3c92dc3692d5ff7c6a3fee6f5d875746948a93fc517922b5ce5d9c8fb8e41}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-ints"},"dependentTasks":["for-loop-1"],"inputs":{"parameters":{"ints":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--int-0-to-9999-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"print-ints"}}' - name: container - value: '{{workflow.parameters.implementations-c48160ebe824a2b5355816c99d1f335ad55b8ba40e2f53c260c844c16bd2f6df}}' + value: '{{workflow.parameters.implementations-7eb3c92dc3692d5ff7c6a3fee6f5d875746948a93fc517922b5ce5d9c8fb8e41}}' - name: task-name value: print-ints - name: parent-dag-id diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index c1c8f80d730..d523def9741 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"executorLabel":"exec-print-and-return-3","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_and_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,16 +18,16 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.9"}' + print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--print-and-return-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"print-and-return"}}}},"tasks":{"print-and-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"print-and-return"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--print-and-return-Output":{"parameterType":"STRING"}}}}' - name: components-comp-condition-3 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--print-and-return-2-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"print-and-return-2"}}}},"tasks":{"print-and-return-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-2"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got tails!"}}}},"taskInfo":{"name":"print-and-return-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--print-and-return-2-Output":{"parameterType":"STRING"}}}}' - - name: components-6dff0af7f5de9b87fab66963c0544a7a050f6a297f26187648f401e15ba49cfd + - name: components-8db1b3d3b7e6823165c17afdab16d298bdef05326ec345e8eded255ee61b4c30 value: '{"executorLabel":"exec-special-print-and-return","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"output_key":{"parameterType":"STRING"}}}}' - - name: implementations-6dff0af7f5de9b87fab66963c0544a7a050f6a297f26187648f401e15ba49cfd + - name: implementations-8db1b3d3b7e6823165c17afdab16d298bdef05326ec345e8eded255ee61b4c30 value: '{"args":["--executor_input","{{$}}","--function_to_execute","special_print_and_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -38,7 +38,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef special_print_and_return(text: str, output_key: dsl.OutputPath(str)):\n print(''Got - the special state:'', text)\n with open(output_key, ''w'') as f:\n f.write(text)\n\n"],"image":"python:3.9"}' + the special state:'', text)\n with open(output_key, ''w'') as f:\n f.write(text)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-4 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--special-print-and-return-output_key":{"valueFromParameter":{"outputParameterKey":"output_key","producerSubtask":"special-print-and-return"}}}},"tasks":{"special-print-and-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-special-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Draw!"}}}},"taskInfo":{"name":"special-print-and-return"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--special-print-and-return-output_key":{"parameterType":"STRING"}}}}' - name: components-comp-condition-branches-1 @@ -48,9 +48,9 @@ spec: == ''tails''"}},"condition-4":{"componentRef":{"name":"comp-condition-4"},"inputs":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"componentInputParameter":"pipelinechannel--flip-three-sided-die-Output"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"!(inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] == ''heads'') \u0026\u0026 !(inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] == ''tails'')"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--condition-branches-1-oneof-1":{"parameterType":"STRING"}}}}' - - name: components-9a6ceddb3a88b2813013f31610e27e091dab5dc13510b28f7b468fdc3ccdfb86 + - name: components-4c0f0fdecdf01e223e03a16bfbb999078ae2981e08cc552cfc2460f569d6ff5b value: '{"executorLabel":"exec-flip-three-sided-die","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-9a6ceddb3a88b2813013f31610e27e091dab5dc13510b28f7b468fdc3ccdfb86 + - name: implementations-4c0f0fdecdf01e223e03a16bfbb999078ae2981e08cc552cfc2460f569d6ff5b value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_three_sided_die"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -62,7 +62,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_three_sided_die() -\u003e str:\n import random\n val = random.randint(0, 2)\n\n if val == 0:\n return ''heads''\n elif val == 1:\n return - ''tails''\n else:\n return ''draw''\n\n"],"image":"python:3.9"}' + ''tails''\n else:\n return ''draw''\n\n"],"image":"python:3.11"}' - name: components-comp-roll-die-pipeline value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--condition-branches-1-oneof-1","producerSubtask":"condition-branches-1"}}}},"tasks":{"condition-branches-1":{"componentRef":{"name":"comp-condition-branches-1"},"dependentTasks":["flip-three-sided-die"],"inputs":{"parameters":{"pipelinechannel--flip-three-sided-die-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-three-sided-die"}}}},"taskInfo":{"name":"condition-branches-1"}},"flip-three-sided-die":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-three-sided-die"},"taskInfo":{"name":"flip-three-sided-die"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: components-root @@ -247,12 +247,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"print-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return - name: parent-dag-id @@ -280,12 +280,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-2"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got tails!"}}}},"taskInfo":{"name":"print-and-return-2"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-2 - name: parent-dag-id @@ -313,11 +313,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-6dff0af7f5de9b87fab66963c0544a7a050f6a297f26187648f401e15ba49cfd}}' + value: '{{workflow.parameters.components-8db1b3d3b7e6823165c17afdab16d298bdef05326ec345e8eded255ee61b4c30}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-special-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Draw!"}}}},"taskInfo":{"name":"special-print-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-6dff0af7f5de9b87fab66963c0544a7a050f6a297f26187648f401e15ba49cfd}}' + value: '{{workflow.parameters.implementations-8db1b3d3b7e6823165c17afdab16d298bdef05326ec345e8eded255ee61b4c30}}' - name: task-name value: special-print-and-return - name: parent-dag-id @@ -523,11 +523,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9a6ceddb3a88b2813013f31610e27e091dab5dc13510b28f7b468fdc3ccdfb86}}' + value: '{{workflow.parameters.components-4c0f0fdecdf01e223e03a16bfbb999078ae2981e08cc552cfc2460f569d6ff5b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-three-sided-die"},"taskInfo":{"name":"flip-three-sided-die"}}' - name: container - value: '{{workflow.parameters.implementations-9a6ceddb3a88b2813013f31610e27e091dab5dc13510b28f7b468fdc3ccdfb86}}' + value: '{{workflow.parameters.implementations-4c0f0fdecdf01e223e03a16bfbb999078ae2981e08cc552cfc2460f569d6ff5b}}' - name: task-name value: flip-three-sided-die - name: parent-dag-id @@ -555,11 +555,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-3"},"dependentTasks":["roll-die-pipeline"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"roll-die-pipeline"}}}},"taskInfo":{"name":"print-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return - name: parent-dag-id diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index f7f431286a2..cb5bfe87126 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc + - name: components-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44 value: '{"executorLabel":"exec-param-to-artifact","inputDefinitions":{"parameters":{"val":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"a":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc + - name: implementations-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44 value: '{"args":["--executor_input","{{$}}","--function_to_execute","param_to_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,7 +19,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef param_to_artifact(val: str, a: Output[Artifact]):\n with open(a.path, ''w'') - as f:\n f.write(val)\n\n"],"image":"python:3.9"}' + as f:\n f.write(val)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--param-to-artifact-a":{"artifactSelectors":[{"outputArtifactKey":"a","producerSubtask":"param-to-artifact"}]}}},"tasks":{"param-to-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-param-to-artifact"},"inputs":{"parameters":{"val":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"param-to-artifact"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--param-to-artifact-a":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: components-comp-condition-3 @@ -28,9 +28,9 @@ spec: value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--condition-branches-1-oneof-1":{"artifactSelectors":[{"outputArtifactKey":"pipelinechannel--param-to-artifact-a","producerSubtask":"condition-2"},{"outputArtifactKey":"pipelinechannel--param-to-artifact-2-a","producerSubtask":"condition-3"}]}}},"tasks":{"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads''"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"!(inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads'')"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--condition-branches-1-oneof-1":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"executorLabel":"exec-flip-coin","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -41,10 +41,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin() -\u003e str:\n import random\n return ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n\n"],"image":"python:3.9"}' - - name: components-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe + 1) == 0 else ''tails''\n\n"],"image":"python:3.11"}' + - name: components-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430 value: '{"executorLabel":"exec-print-artifact","inputDefinitions":{"artifacts":{"a":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe + - name: implementations-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -54,7 +54,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_artifact(a: Input[Artifact]):\n with open(a.path) as f:\n print(f.read())\n\n"],"image":"python:3.9"}' + print_artifact(a: Input[Artifact]):\n with open(a.path) as f:\n print(f.read())\n\n"],"image":"python:3.11"}' - name: components-comp-flip-coin-pipeline value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"pipelinechannel--condition-branches-1-oneof-1","producerSubtask":"condition-branches-1"}]}}},"tasks":{"condition-branches-1":{"componentRef":{"name":"comp-condition-branches-1"},"dependentTasks":["flip-coin"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin"}}}},"taskInfo":{"name":"condition-branches-1"}},"flip-coin":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}},"print-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-artifact"},"dependentTasks":["condition-branches-1"],"inputs":{"artifacts":{"a":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--condition-branches-1-oneof-1","producerTask":"condition-branches-1"}}}},"taskInfo":{"name":"print-artifact"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: components-root @@ -239,11 +239,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc}}' + value: '{{workflow.parameters.components-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-param-to-artifact"},"inputs":{"parameters":{"val":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"param-to-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc}}' + value: '{{workflow.parameters.implementations-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44}}' - name: task-name value: param-to-artifact - name: parent-dag-id @@ -271,11 +271,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc}}' + value: '{{workflow.parameters.components-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-param-to-artifact-2"},"inputs":{"parameters":{"val":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"param-to-artifact-2"}}' - name: container - value: '{{workflow.parameters.implementations-c3727009f00beb8b5919fc00c4c334adbf50ee929c9f2bb4f16182c1cd2e3cbc}}' + value: '{{workflow.parameters.implementations-548a24cf68d38c6b9cd5ffa0cf3bff0ebc8c28e453df033e43d174b1b0799e44}}' - name: task-name value: param-to-artifact-2 - name: parent-dag-id @@ -456,11 +456,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}}' - name: container - value: '{{workflow.parameters.implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task-name value: flip-coin - name: parent-dag-id @@ -480,11 +480,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe}}' + value: '{{workflow.parameters.components-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-artifact"},"dependentTasks":["condition-branches-1"],"inputs":{"artifacts":{"a":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--condition-branches-1-oneof-1","producerTask":"condition-branches-1"}}}},"taskInfo":{"name":"print-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe}}' + value: '{{workflow.parameters.implementations-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430}}' - name: task-name value: print-artifact - name: parent-dag-id @@ -534,11 +534,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe}}' + value: '{{workflow.parameters.components-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-artifact-2"},"dependentTasks":["flip-coin-pipeline"],"inputs":{"artifacts":{"a":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"flip-coin-pipeline"}}}},"taskInfo":{"name":"print-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-4b25ff0a49eab000c257f8f4a0117403198c888052ec86137349df44a41d04fe}}' + value: '{{workflow.parameters.implementations-f4e8ee10b02bd6b6b90fdf2bb11d355cbb8b814138f9ef2935d245797f7bf430}}' - name: task-name value: print-artifact - name: parent-dag-id diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index 084998ddb22..6c3acaaa30a 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"executorLabel":"exec-print-and-return","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac + - name: implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_and_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.9"}' + print_and_return(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--print-and-return-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"print-and-return"}}}},"tasks":{"print-and-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"print-and-return"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--print-and-return-Output":{"parameterType":"STRING"}}}}' @@ -29,9 +29,9 @@ spec: value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--condition-branches-1-oneof-1":{"valueFromOneof":{"parameterSelectors":[{"outputParameterKey":"pipelinechannel--print-and-return-Output","producerSubtask":"condition-2"},{"outputParameterKey":"pipelinechannel--print-and-return-2-Output","producerSubtask":"condition-3"}]}}}},"tasks":{"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads''"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"!(inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads'')"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--condition-branches-1-oneof-1":{"parameterType":"STRING"}}}}' - - name: components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"executorLabel":"exec-flip-coin","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -42,7 +42,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin() -\u003e str:\n import random\n return ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n\n"],"image":"python:3.9"}' + 1) == 0 else ''tails''\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--condition-branches-1-oneof-1","producerSubtask":"condition-branches-1"}}}},"tasks":{"condition-branches-1":{"componentRef":{"name":"comp-condition-branches-1"},"dependentTasks":["flip-coin"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin"}}}},"taskInfo":{"name":"condition-branches-1"}},"flip-coin":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}},"print-and-return-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-3"},"dependentTasks":["condition-branches-1"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--condition-branches-1-oneof-1","producerTask":"condition-branches-1"}}}},"taskInfo":{"name":"print-and-return-3"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -225,12 +225,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"print-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return - name: parent-dag-id @@ -258,12 +258,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-2"},"inputs":{"parameters":{"text":{"runtimeValue":{"constant":"Got tails!"}}}},"taskInfo":{"name":"print-and-return-2"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-2 - name: parent-dag-id @@ -444,11 +444,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}}' - name: container - value: '{{workflow.parameters.implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task-name value: flip-coin - name: parent-dag-id @@ -468,11 +468,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.components-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-and-return-3"},"dependentTasks":["condition-branches-1"],"inputs":{"parameters":{"text":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--condition-branches-1-oneof-1","producerTask":"condition-branches-1"}}}},"taskInfo":{"name":"print-and-return-3"}}' - name: container - value: '{{workflow.parameters.implementations-d4d51356e36512e82fa2af8e14375269db87299669cfb9b3cfa0baacf11fecac}}' + value: '{{workflow.parameters.implementations-cecb98cac1de5c44c981fe9bc4ca2eaa0b650231b182536eb374f37a191a09d2}}' - name: task-name value: print-and-return-3 - name: parent-dag-id diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index 100bf2f46af..89d16286c2b 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-da3c6808bf551f2963106f4d6e053e4d70b827b3960f6e6d581fa72f4c1191fe + - name: components-c0e9d1cc85ce81739c3677ec2338cb29b8bad1837b123a910f0aa723a5fde46c value: '{"executorLabel":"exec-input-artifact","inputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-da3c6808bf551f2963106f4d6e053e4d70b827b3960f6e6d581fa72f4c1191fe + - name: implementations-c0e9d1cc85ce81739c3677ec2338cb29b8bad1837b123a910f0aa723a5fde46c value: '{"args":["--executor_input","{{$}}","--function_to_execute","input_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - input_artifact(data: Input[Dataset]):\n print(data.name)\n print(data.uri)\n print(data.metadata)\n\n"],"image":"python:3.9"}' + input_artifact(data: Input[Dataset]):\n print(data.name)\n print(data.uri)\n print(data.metadata)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"input-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-input-artifact"},"inputs":{"artifacts":{"data":{"componentInputArtifact":"data"}}},"taskInfo":{"name":"input-artifact"}}}},"inputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-da3c6808bf551f2963106f4d6e053e4d70b827b3960f6e6d581fa72f4c1191fe}}' + value: '{{workflow.parameters.components-c0e9d1cc85ce81739c3677ec2338cb29b8bad1837b123a910f0aa723a5fde46c}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-input-artifact"},"inputs":{"artifacts":{"data":{"componentInputArtifact":"data"}}},"taskInfo":{"name":"input-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-da3c6808bf551f2963106f4d6e053e4d70b827b3960f6e6d581fa72f4c1191fe}}' + value: '{{workflow.parameters.implementations-c0e9d1cc85ce81739c3677ec2338cb29b8bad1837b123a910f0aa723a5fde46c}}' - name: task-name value: input-artifact - name: parent-dag-id diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 21e727eda75..00bb6e0f83a 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-2af7169887551fdca1e52d152b52ec11ea10d8009f09023395873eb7f04f41c2 + - name: components-46b83eacfc26cd94d4579de62f52c8eb251e7093888e545381b90fcf7e1b26c8 value: '{"executorLabel":"exec-create-dataset","outputDefinitions":{"artifacts":{"iris_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-2af7169887551fdca1e52d152b52ec11ea10d8009f09023395873eb7f04f41c2 + - name: implementations-46b83eacfc26cd94d4579de62f52c8eb251e7093888e545381b90fcf7e1b26c8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","create_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -23,10 +23,10 @@ spec: noqa: PLC0415\n\n import pandas as pd # noqa: PLC0415\n\n data = \"\"\"\n 5.1,3.5,1.4,0.2,Iris-setosa\n 4.9,3.0,1.4,0.2,Iris-setosa\n 4.7,3.2,1.3,0.2,Iris-setosa\n 4.6,3.1,1.5,0.2,Iris-setosa\n 5.0,3.6,1.4,0.2,Iris-setosa\n 5.7,3.8,1.7,0.3,Iris-setosa\n 5.1,3.8,1.5,0.3,Iris-setosa\n 5.4,3.4,1.7,0.2,Iris-setosa\n 5.1,3.7,1.5,0.4,Iris-setosa\n 5.1,3.4,1.5,0.2,Iris-setosa\n 5.0,3.5,1.3,0.3,Iris-setosa\n 4.5,2.3,1.3,0.3,Iris-setosa\n 4.4,3.2,1.3,0.2,Iris-setosa\n 5.0,3.5,1.6,0.6,Iris-setosa\n 5.1,3.8,1.9,0.4,Iris-setosa\n 4.8,3.0,1.4,0.3,Iris-setosa\n 5.1,3.8,1.6,0.2,Iris-setosa\n 4.6,3.2,1.4,0.2,Iris-setosa\n 5.3,3.7,1.5,0.2,Iris-setosa\n 5.0,3.3,1.4,0.2,Iris-setosa\n 7.0,3.2,4.7,1.4,Iris-versicolor\n 6.4,3.2,4.5,1.5,Iris-versicolor\n 6.9,3.1,4.9,1.5,Iris-versicolor\n 5.5,2.3,4.0,1.3,Iris-versicolor\n 6.5,2.8,4.6,1.5,Iris-versicolor\n 6.2,2.2,4.5,1.5,Iris-versicolor\n 5.6,2.5,3.9,1.1,Iris-versicolor\n 5.9,3.2,4.8,1.8,Iris-versicolor\n 6.1,2.8,4.0,1.3,Iris-versicolor\n 6.3,2.5,4.9,1.5,Iris-versicolor\n 6.1,2.8,4.7,1.2,Iris-versicolor\n 6.4,2.9,4.3,1.3,Iris-versicolor\n 6.6,3.0,4.4,1.4,Iris-versicolor\n 5.6,2.7,4.2,1.3,Iris-versicolor\n 5.7,3.0,4.2,1.2,Iris-versicolor\n 5.7,2.9,4.2,1.3,Iris-versicolor\n 6.2,2.9,4.3,1.3,Iris-versicolor\n 5.1,2.5,3.0,1.1,Iris-versicolor\n 5.7,2.8,4.1,1.3,Iris-versicolor\n 6.3,3.3,6.0,2.5,Iris-virginica\n 5.8,2.7,5.1,1.9,Iris-virginica\n 7.1,3.0,5.9,2.1,Iris-virginica\n 6.3,2.9,5.6,1.8,Iris-virginica\n 6.5,3.0,5.8,2.2,Iris-virginica\n 6.9,3.1,5.1,2.3,Iris-virginica\n 5.8,2.7,5.1,1.9,Iris-virginica\n 6.8,3.2,5.9,2.3,Iris-virginica\n 6.7,3.3,5.7,2.5,Iris-virginica\n 6.7,3.0,5.2,2.3,Iris-virginica\n 6.3,2.5,5.0,1.9,Iris-virginica\n 6.5,3.0,5.2,2.0,Iris-virginica\n 6.2,3.4,5.4,2.3,Iris-virginica\n 5.9,3.0,5.1,1.8,Iris-virginica\n \"\"\"\n col_names = [\"Sepal_Length\", \"Sepal_Width\", \"Petal_Length\", \"Petal_Width\", \"Labels\"]\n df = pd.read_csv(StringIO(data), names=col_names)\n\n with open(iris_dataset.path, - \"w\") as f:\n df.to_csv(f)\n\n"],"image":"python:3.9"}' - - name: components-e21502083257f62c5b8d5e6dddd74ed2fa77a7f3e5ebb318e474eb94b51d71bb + \"w\") as f:\n df.to_csv(f)\n\n"],"image":"python:3.11"}' + - name: components-f666f111ab69031b13ddbe7fb9b938dc0c077c50cf3b7b5472f4d97d36627cb0 value: '{"executorLabel":"exec-normalize-dataset","inputDefinitions":{"artifacts":{"input_iris_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"standard_scaler":{"parameterType":"BOOLEAN"}}},"outputDefinitions":{"artifacts":{"normalized_iris_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-e21502083257f62c5b8d5e6dddd74ed2fa77a7f3e5ebb318e474eb94b51d71bb + - name: implementations-f666f111ab69031b13ddbe7fb9b938dc0c077c50cf3b7b5472f4d97d36627cb0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","normalize_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -44,10 +44,10 @@ spec: = pd.read_csv(f)\n labels = df.pop(\"Labels\")\n\n scaler = StandardScaler() if standard_scaler else MinMaxScaler()\n\n df = pd.DataFrame(scaler.fit_transform(df))\n df[\"Labels\"] = labels\n normalized_iris_dataset.metadata[\"state\"] = \"Normalized\"\n with - open(normalized_iris_dataset.path, \"w\") as f:\n df.to_csv(f)\n\n"],"image":"python:3.9"}' - - name: components-e6e50647ba805f535e80fde81f76312e593390b70b360c1069a9b7766bfb5997 + open(normalized_iris_dataset.path, \"w\") as f:\n df.to_csv(f)\n\n"],"image":"python:3.11"}' + - name: components-0aef93036ec0e04e8c952f416203a009f7b67c1736c73e0a57cf513297ac4add value: '{"executorLabel":"exec-train-model","inputDefinitions":{"artifacts":{"normalized_iris_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"n_neighbors":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.ClassificationMetrics","schemaVersion":"0.0.1"}},"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-e6e50647ba805f535e80fde81f76312e593390b70b360c1069a9b7766bfb5997 + - name: implementations-0aef93036ec0e04e8c952f416203a009f7b67c1736c73e0a57cf513297ac4add value: '{"args":["--executor_input","{{$}}","--function_to_execute","train_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -71,7 +71,7 @@ spec: \"Iris-Versicolour\", \"Iris-Virginica\"],\n confusion_matrix(y_train, predictions).tolist(), # .tolist() to convert np array to list.\n )\n\n model.metadata[\"framework\"] = \"scikit-learn\"\n with open(model.path, \"wb\") as f:\n pickle.dump(clf, - f)\n\n"],"image":"python:3.9"}' + f)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"create-dataset":{"cachingOptions":{},"componentRef":{"name":"comp-create-dataset"},"taskInfo":{"name":"create-dataset"}},"normalize-dataset":{"cachingOptions":{},"componentRef":{"name":"comp-normalize-dataset"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"input_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"iris_dataset","producerTask":"create-dataset"}}},"parameters":{"standard_scaler":{"componentInputParameter":"standard_scaler"}}},"taskInfo":{"name":"normalize-dataset"}},"train-model":{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["normalize-dataset"],"inputs":{"artifacts":{"normalized_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"normalized_iris_dataset","producerTask":"normalize-dataset"}}},"parameters":{"n_neighbors":{"componentInputParameter":"neighbors"}}},"taskInfo":{"name":"train-model"}}}},"inputDefinitions":{"parameters":{"neighbors":{"defaultValue":3,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"standard_scaler":{"defaultValue":true,"isOptional":true,"parameterType":"BOOLEAN"}}}}' entrypoint: entrypoint @@ -254,11 +254,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2af7169887551fdca1e52d152b52ec11ea10d8009f09023395873eb7f04f41c2}}' + value: '{{workflow.parameters.components-46b83eacfc26cd94d4579de62f52c8eb251e7093888e545381b90fcf7e1b26c8}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-create-dataset"},"taskInfo":{"name":"create-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-2af7169887551fdca1e52d152b52ec11ea10d8009f09023395873eb7f04f41c2}}' + value: '{{workflow.parameters.implementations-46b83eacfc26cd94d4579de62f52c8eb251e7093888e545381b90fcf7e1b26c8}}' - name: task-name value: create-dataset - name: parent-dag-id @@ -278,11 +278,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e21502083257f62c5b8d5e6dddd74ed2fa77a7f3e5ebb318e474eb94b51d71bb}}' + value: '{{workflow.parameters.components-f666f111ab69031b13ddbe7fb9b938dc0c077c50cf3b7b5472f4d97d36627cb0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-normalize-dataset"},"dependentTasks":["create-dataset"],"inputs":{"artifacts":{"input_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"iris_dataset","producerTask":"create-dataset"}}},"parameters":{"standard_scaler":{"componentInputParameter":"standard_scaler"}}},"taskInfo":{"name":"normalize-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-e21502083257f62c5b8d5e6dddd74ed2fa77a7f3e5ebb318e474eb94b51d71bb}}' + value: '{{workflow.parameters.implementations-f666f111ab69031b13ddbe7fb9b938dc0c077c50cf3b7b5472f4d97d36627cb0}}' - name: task-name value: normalize-dataset - name: parent-dag-id @@ -303,11 +303,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e6e50647ba805f535e80fde81f76312e593390b70b360c1069a9b7766bfb5997}}' + value: '{{workflow.parameters.components-0aef93036ec0e04e8c952f416203a009f7b67c1736c73e0a57cf513297ac4add}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["normalize-dataset"],"inputs":{"artifacts":{"normalized_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"normalized_iris_dataset","producerTask":"normalize-dataset"}}},"parameters":{"n_neighbors":{"componentInputParameter":"neighbors"}}},"taskInfo":{"name":"train-model"}}' - name: container - value: '{{workflow.parameters.implementations-e6e50647ba805f535e80fde81f76312e593390b70b360c1069a9b7766bfb5997}}' + value: '{{workflow.parameters.implementations-0aef93036ec0e04e8c952f416203a009f7b67c1736c73e0a57cf513297ac4add}}' - name: task-name value: train-model - name: parent-dag-id diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index 6d46528f7b8..51b70783254 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f + - name: components-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4 value: '{"executorLabel":"exec-preprocess","inputDefinitions":{"parameters":{"input_dict_parameter":{"parameterType":"STRUCT"},"input_list_parameter":{"parameterType":"LIST"},"message":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_dataset_one":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"output_dataset_two_path":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_bool_parameter_path":{"parameterType":"BOOLEAN"},"output_dict_parameter_path":{"parameterType":"STRUCT"},"output_list_parameter_path":{"parameterType":"LIST"},"output_parameter_path":{"parameterType":"STRING"}}}}' - - name: implementations-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f + - name: implementations-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4 value: '{"args":["--executor_input","{{$}}","--function_to_execute","preprocess"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -40,10 +40,10 @@ spec: open(output_bool_parameter_path, ''w'') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, ''w'') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with - open(output_list_parameter_path, ''w'') as f:\n f.write(json.dumps(input_list_parameter))\n\n"],"image":"python:3.9"}' - - name: components-c146bad05c050c176402e7a5065dafbcb507ddacd77264720b5199dd9a8b1711 + open(output_list_parameter_path, ''w'') as f:\n f.write(json.dumps(input_list_parameter))\n\n"],"image":"python:3.11"}' + - name: components-b36bc37867b11dd27d7e36a6abd032012300f22cad3a1d2c85ed1f7d868a4ae1 value: '{"executorLabel":"exec-train","inputDefinitions":{"artifacts":{"dataset_one_path":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"dataset_two":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"input_bool":{"parameterType":"BOOLEAN"},"input_dict":{"parameterType":"STRUCT"},"input_list":{"parameterType":"LIST"},"message":{"parameterType":"STRING"},"num_steps":{"defaultValue":100,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-c146bad05c050c176402e7a5065dafbcb507ddacd77264720b5199dd9a8b1711 + - name: implementations-b36bc37867b11dd27d7e36a6abd032012300f22cad3a1d2c85ed1f7d868a4ae1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","train"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -73,7 +73,7 @@ spec: open(model.path, ''w'') as output_file:\n for i in range(num_steps):\n output_file.write(f''Step {i}\\n{line}\\n=====\\n'')\n\n # model is an instance of Model artifact, which has a .metadata dictionary\n # to store arbitrary metadata for the - output artifact.\n model.metadata[''accuracy''] = 0.9\n\n"],"image":"python:3.9"}' + output artifact.\n model.metadata[''accuracy''] = 0.9\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"preprocess":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"input_dict_parameter":{"componentInputParameter":"input_dict"},"input_list_parameter":{"runtimeValue":{"constant":["a","b","c"]}},"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"preprocess"}},"train":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-train"},"dependentTasks":["preprocess"],"inputs":{"artifacts":{"dataset_one_path":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset_one","producerTask":"preprocess"}},"dataset_two":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset_two_path","producerTask":"preprocess"}}},"parameters":{"input_bool":{"taskOutputParameter":{"outputParameterKey":"output_bool_parameter_path","producerTask":"preprocess"}},"input_dict":{"taskOutputParameter":{"outputParameterKey":"output_dict_parameter_path","producerTask":"preprocess"}},"input_list":{"taskOutputParameter":{"outputParameterKey":"output_list_parameter_path","producerTask":"preprocess"}},"message":{"taskOutputParameter":{"outputParameterKey":"output_parameter_path","producerTask":"preprocess"}}}},"taskInfo":{"name":"train"}}}},"inputDefinitions":{"parameters":{"input_dict":{"defaultValue":{"A":1,"B":2},"isOptional":true,"parameterType":"STRUCT"},"message":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -256,11 +256,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f}}' + value: '{{workflow.parameters.components-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"input_dict_parameter":{"componentInputParameter":"input_dict"},"input_list_parameter":{"runtimeValue":{"constant":["a","b","c"]}},"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"preprocess"}}' - name: container - value: '{{workflow.parameters.implementations-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f}}' + value: '{{workflow.parameters.implementations-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4}}' - name: task-name value: preprocess - name: parent-dag-id @@ -280,11 +280,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c146bad05c050c176402e7a5065dafbcb507ddacd77264720b5199dd9a8b1711}}' + value: '{{workflow.parameters.components-b36bc37867b11dd27d7e36a6abd032012300f22cad3a1d2c85ed1f7d868a4ae1}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-train"},"dependentTasks":["preprocess"],"inputs":{"artifacts":{"dataset_one_path":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset_one","producerTask":"preprocess"}},"dataset_two":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset_two_path","producerTask":"preprocess"}}},"parameters":{"input_bool":{"taskOutputParameter":{"outputParameterKey":"output_bool_parameter_path","producerTask":"preprocess"}},"input_dict":{"taskOutputParameter":{"outputParameterKey":"output_dict_parameter_path","producerTask":"preprocess"}},"input_list":{"taskOutputParameter":{"outputParameterKey":"output_list_parameter_path","producerTask":"preprocess"}},"message":{"taskOutputParameter":{"outputParameterKey":"output_parameter_path","producerTask":"preprocess"}}}},"taskInfo":{"name":"train"}}' - name: container - value: '{{workflow.parameters.implementations-c146bad05c050c176402e7a5065dafbcb507ddacd77264720b5199dd9a8b1711}}' + value: '{{workflow.parameters.implementations-b36bc37867b11dd27d7e36a6abd032012300f22cad3a1d2c85ed1f7d868a4ae1}}' - name: task-name value: train - name: parent-dag-id diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index 3f27d291acb..69ac40a7575 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-8870f0a2f68d8cd4568555a23e74ba662d25e8d5d7206d8c0b3d03e9d3afce64 + - name: components-64327f64eaa6ee6d1d3875fb132b2b0220e21b5ab70d2e725bd77cefe61af74a value: '{"executorLabel":"exec-add-numbers","inputDefinitions":{"parameters":{"first":{"parameterType":"NUMBER_INTEGER"},"second":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-8870f0a2f68d8cd4568555a23e74ba662d25e8d5d7206d8c0b3d03e9d3afce64 + - name: implementations-64327f64eaa6ee6d1d3875fb132b2b0220e21b5ab70d2e725bd77cefe61af74a value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add_numbers(first: int, second: int) -\u003e int:\n return first + second\n\n"],"image":"python:3.9"}' - - name: components-f927c43072fb8baf32c3241fad2cbf970008046a26a48f20f75c4226e4a16cd8 + add_numbers(first: int, second: int) -\u003e int:\n return first + second\n\n"],"image":"python:3.11"}' + - name: components-22305325c661206b4123f78ea46d0c5195ad2ec0f2cb1c12e7d3796adfea3272 value: '{"executorLabel":"exec-concat-message","inputDefinitions":{"parameters":{"first":{"parameterType":"STRING"},"second":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-f927c43072fb8baf32c3241fad2cbf970008046a26a48f20f75c4226e4a16cd8 + - name: implementations-22305325c661206b4123f78ea46d0c5195ad2ec0f2cb1c12e7d3796adfea3272 value: '{"args":["--executor_input","{{$}}","--function_to_execute","concat_message"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,10 +31,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - concat_message(first: str, second: str) -\u003e str:\n return first + second\n\n"],"image":"python:3.9"}' - - name: components-057891112980ba235e08bd912537a46f32f24c44d2eadd1f97e1f5f4ea3094c2 + concat_message(first: str, second: str) -\u003e str:\n return first + second\n\n"],"image":"python:3.11"}' + - name: components-8a22c1e65c445da5e19295ad1c0852d6baa362cde8d17969d80f385ffb3f744b value: '{"executorLabel":"exec-output-artifact","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"},"number":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-057891112980ba235e08bd912537a46f32f24c44d2eadd1f97e1f5f4ea3094c2 + - name: implementations-8a22c1e65c445da5e19295ad1c0852d6baa362cde8d17969d80f385ffb3f744b value: '{"args":["--executor_input","{{$}}","--function_to_execute","output_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -45,10 +45,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_artifact(number: int, message: str) -\u003e Dataset:\n result = - [message for _ in range(number)]\n return ''\\n''.join(result)\n\n"],"image":"python:3.9"}' - - name: components-88a0f7f1dfae7eede46c958ca3094899fb994cfa854fd93e13a067becc26bac6 + [message for _ in range(number)]\n return ''\\n''.join(result)\n\n"],"image":"python:3.11"}' + - name: components-8b7c37a05dbc36da22206bd274fccfa32b8ca2751e413c2d48e41d700d278796 value: '{"executorLabel":"exec-output-named-tuple","inputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}},"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}},"parameters":{"scalar":{"parameterType":"STRING"}}}}' - - name: implementations-88a0f7f1dfae7eede46c958ca3094899fb994cfa854fd93e13a067becc26bac6 + - name: implementations-8b7c37a05dbc36da22206bd274fccfa32b8ca2751e413c2d48e41d700d278796 value: '{"args":["--executor_input","{{$}}","--function_to_execute","output_named_tuple"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -65,7 +65,7 @@ spec: ''PERCENTAGE'',\n }]\n })\n\n with open(artifact.path) as f:\n artifact_contents = f.read()\n model = ''Model contents: '' + artifact_contents\n\n from collections import namedtuple\n output = namedtuple(''Outputs'', [''scalar'', - ''metrics'', ''model''])\n return output(scalar, metrics, model)\n\n"],"image":"python:3.9"}' + ''metrics'', ''model''])\n return output(scalar, metrics, model)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"add-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"first":{"componentInputParameter":"first_number"},"second":{"componentInputParameter":"second_number"}}},"taskInfo":{"name":"add-numbers"}},"concat-message":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-message"},"inputs":{"parameters":{"first":{"componentInputParameter":"first_message"},"second":{"componentInputParameter":"second_message"}}},"taskInfo":{"name":"concat-message"}},"output-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-artifact"},"dependentTasks":["add-numbers","concat-message"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"concat-message"}},"number":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers"}}}},"taskInfo":{"name":"output-artifact"}},"output-named-tuple":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-named-tuple"},"dependentTasks":["output-artifact"],"inputs":{"artifacts":{"artifact":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"output-artifact"}}}},"taskInfo":{"name":"output-named-tuple"}}}},"inputDefinitions":{"parameters":{"first_message":{"parameterType":"STRING"},"first_number":{"parameterType":"NUMBER_INTEGER"},"second_message":{"parameterType":"STRING"},"second_number":{"parameterType":"NUMBER_INTEGER"}}}}' entrypoint: entrypoint @@ -248,11 +248,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8870f0a2f68d8cd4568555a23e74ba662d25e8d5d7206d8c0b3d03e9d3afce64}}' + value: '{{workflow.parameters.components-64327f64eaa6ee6d1d3875fb132b2b0220e21b5ab70d2e725bd77cefe61af74a}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"first":{"componentInputParameter":"first_number"},"second":{"componentInputParameter":"second_number"}}},"taskInfo":{"name":"add-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-8870f0a2f68d8cd4568555a23e74ba662d25e8d5d7206d8c0b3d03e9d3afce64}}' + value: '{{workflow.parameters.implementations-64327f64eaa6ee6d1d3875fb132b2b0220e21b5ab70d2e725bd77cefe61af74a}}' - name: task-name value: add-numbers - name: parent-dag-id @@ -272,11 +272,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f927c43072fb8baf32c3241fad2cbf970008046a26a48f20f75c4226e4a16cd8}}' + value: '{{workflow.parameters.components-22305325c661206b4123f78ea46d0c5195ad2ec0f2cb1c12e7d3796adfea3272}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-concat-message"},"inputs":{"parameters":{"first":{"componentInputParameter":"first_message"},"second":{"componentInputParameter":"second_message"}}},"taskInfo":{"name":"concat-message"}}' - name: container - value: '{{workflow.parameters.implementations-f927c43072fb8baf32c3241fad2cbf970008046a26a48f20f75c4226e4a16cd8}}' + value: '{{workflow.parameters.implementations-22305325c661206b4123f78ea46d0c5195ad2ec0f2cb1c12e7d3796adfea3272}}' - name: task-name value: concat-message - name: parent-dag-id @@ -296,11 +296,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-057891112980ba235e08bd912537a46f32f24c44d2eadd1f97e1f5f4ea3094c2}}' + value: '{{workflow.parameters.components-8a22c1e65c445da5e19295ad1c0852d6baa362cde8d17969d80f385ffb3f744b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-artifact"},"dependentTasks":["add-numbers","concat-message"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"concat-message"}},"number":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers"}}}},"taskInfo":{"name":"output-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-057891112980ba235e08bd912537a46f32f24c44d2eadd1f97e1f5f4ea3094c2}}' + value: '{{workflow.parameters.implementations-8a22c1e65c445da5e19295ad1c0852d6baa362cde8d17969d80f385ffb3f744b}}' - name: task-name value: output-artifact - name: parent-dag-id @@ -321,11 +321,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-88a0f7f1dfae7eede46c958ca3094899fb994cfa854fd93e13a067becc26bac6}}' + value: '{{workflow.parameters.components-8b7c37a05dbc36da22206bd274fccfa32b8ca2751e413c2d48e41d700d278796}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-named-tuple"},"dependentTasks":["output-artifact"],"inputs":{"artifacts":{"artifact":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"output-artifact"}}}},"taskInfo":{"name":"output-named-tuple"}}' - name: container - value: '{{workflow.parameters.implementations-88a0f7f1dfae7eede46c958ca3094899fb994cfa854fd93e13a067becc26bac6}}' + value: '{{workflow.parameters.implementations-8b7c37a05dbc36da22206bd274fccfa32b8ca2751e413c2d48e41d700d278796}}' - name: task-name value: output-named-tuple - name: parent-dag-id diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index ab34fb6385e..9cd30149337 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-888809574b5309659ec40b386743894e8953566f7d2130458f4fb24e267dfb9c + - name: components-83903e2b86109679dccb14a8d66f84f860b49eb946f971670d98e0dc63e93cf6 value: '{"executorLabel":"exec-print-message","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-888809574b5309659ec40b386743894e8953566f7d2130458f4fb24e267dfb9c + - name: implementations-83903e2b86109679dccb14a8d66f84f860b49eb946f971670d98e0dc63e93cf6 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_message"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,7 +21,7 @@ spec: print_message(message: str):\n import datetime # noqa: PLC0415\n import time # noqa: PLC0415\n\n t_end = time.time() + 60\n while time.time() \u003c t_end:\n print(message + \" (\" + str(datetime.datetime.now()) - + \")\")\n\n"],"image":"python:3.9"}' + + \")\")\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"print-message":{"cachingOptions":{},"componentRef":{"name":"comp-print-message"},"inputs":{"parameters":{"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"print-message"}}}},"inputDefinitions":{"parameters":{"message":{"defaultValue":"Hello world","isOptional":true,"parameterType":"STRING"}}}}' @@ -205,11 +205,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-888809574b5309659ec40b386743894e8953566f7d2130458f4fb24e267dfb9c}}' + value: '{{workflow.parameters.components-83903e2b86109679dccb14a8d66f84f860b49eb946f971670d98e0dc63e93cf6}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-message"},"inputs":{"parameters":{"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"print-message"}}' - name: container - value: '{{workflow.parameters.implementations-888809574b5309659ec40b386743894e8953566f7d2130458f4fb24e267dfb9c}}' + value: '{{workflow.parameters.implementations-83903e2b86109679dccb14a8d66f84f860b49eb946f971670d98e0dc63e93cf6}}' - name: task-name value: print-message - name: parent-dag-id diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 2168729a408..c10b9e84e7f 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-ed5ffb9d2b8fff6bccdc76c9370a0e8bcdb7e239acce180cc6ce4a8eca4d1972 + - name: components-e95f5ecfe88450a38e0fd6e153fc9259d010d9ab920e3fe0dd141bf675c413df value: '{"executorLabel":"exec-create-file","inputDefinitions":{"parameters":{"content":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"file":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-ed5ffb9d2b8fff6bccdc76c9370a0e8bcdb7e239acce180cc6ce4a8eca4d1972 + - name: implementations-e95f5ecfe88450a38e0fd6e153fc9259d010d9ab920e3fe0dd141bf675c413df value: '{"args":["--executor_input","{{$}}","--function_to_execute","create_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef create_file(file: Output[Artifact], content: str):\n with open(file.path, - ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.9"}' - - name: components-711e8b6e3ae4585024167b39bc927d87c5c6dd39e2ab20b0613ea96fd049dc4a + ''w'') as f:\n f.write(content)\n\n"],"image":"python:3.11"}' + - name: components-f59c851c3c0a1e03b3e2065384eb3c4665ce5f234998fed18cb6afb34ce005a7 value: '{"executorLabel":"exec-read-file","inputDefinitions":{"artifacts":{"file":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-711e8b6e3ae4585024167b39bc927d87c5c6dd39e2ab20b0613ea96fd049dc4a + - name: implementations-f59c851c3c0a1e03b3e2065384eb3c4665ce5f234998fed18cb6afb34ce005a7 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,14 +33,14 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef read_file(file: Input[Artifact]) -\u003e str:\n with open(file.path, ''r'') - as f:\n print(f.read())\n return file.path\n\n"],"image":"python:3.9"}' + as f:\n print(f.read())\n return file.path\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"tasks":{"create-file":{"cachingOptions":{},"componentRef":{"name":"comp-create-file"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-input-Output-loop-item"}}},"taskInfo":{"name":"same display name"}},"read-file":{"cachingOptions":{},"componentRef":{"name":"comp-read-file"},"dependentTasks":["create-file"],"inputs":{"artifacts":{"file":{"taskOutputArtifact":{"outputArtifactKey":"file","producerTask":"create-file"}}}},"taskInfo":{"name":"same display name"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--split-input-Output":{"parameterType":"LIST"},"pipelinechannel--split-input-Output-loop-item":{"parameterType":"STRING"}}}}' - - name: components-ca2a87b6895df4cb47f1b6631daadcc4a5634c47ac3aadb0d316b2910f3b7794 + - name: components-b4e9d76b938751b0b1edf9cce7cc5817c0b9c54d34a1e06c3a226a46cc2c3f1e value: '{"executorLabel":"exec-print-input","inputDefinitions":{"parameters":{"input":{"parameterType":"LIST"}}}}' - - name: implementations-ca2a87b6895df4cb47f1b6631daadcc4a5634c47ac3aadb0d316b2910f3b7794 + - name: implementations-b4e9d76b938751b0b1edf9cce7cc5817c0b9c54d34a1e06c3a226a46cc2c3f1e value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_input"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -51,10 +51,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_input(input: list):\n for item in input:\n print(f''Input - item: {item}'')\n\n"],"image":"python:3.9"}' - - name: components-ed7019fac34ed83d85bd9c5b5fd36b6f73f1211ad397845e5f71e73680a4d5a6 + item: {item}'')\n\n"],"image":"python:3.11"}' + - name: components-fc5dd8cf7a18cc02226309bb936fb376a98741769b2faba10d1bfc9960de8802 value: '{"executorLabel":"exec-split-input","inputDefinitions":{"parameters":{"input":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-ed7019fac34ed83d85bd9c5b5fd36b6f73f1211ad397845e5f71e73680a4d5a6 + - name: implementations-fc5dd8cf7a18cc02226309bb936fb376a98741769b2faba10d1bfc9960de8802 value: '{"args":["--executor_input","{{$}}","--function_to_execute","split_input"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -64,7 +64,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - split_input(input: str) -\u003e list:\n return input.split('','')\n\n"],"image":"python:3.9"}' + split_input(input: str) -\u003e list:\n return input.split('','')\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-input"],"inputs":{"parameters":{"pipelinechannel--split-input-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-input"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-input-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-input-Output"}},"taskInfo":{"name":"for-loop-1"}},"print-input":{"cachingOptions":{},"componentRef":{"name":"comp-print-input"},"dependentTasks":["split-input"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-input"}}}},"taskInfo":{"name":"same display name"}},"split-input":{"cachingOptions":{},"componentRef":{"name":"comp-split-input"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"component1,component2,component3"}}}},"taskInfo":{"name":"same @@ -249,12 +249,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ed5ffb9d2b8fff6bccdc76c9370a0e8bcdb7e239acce180cc6ce4a8eca4d1972}}' + value: '{{workflow.parameters.components-e95f5ecfe88450a38e0fd6e153fc9259d010d9ab920e3fe0dd141bf675c413df}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-create-file"},"inputs":{"parameters":{"content":{"componentInputParameter":"pipelinechannel--split-input-Output-loop-item"}}},"taskInfo":{"name":"same display name"}}' - name: container - value: '{{workflow.parameters.implementations-ed5ffb9d2b8fff6bccdc76c9370a0e8bcdb7e239acce180cc6ce4a8eca4d1972}}' + value: '{{workflow.parameters.implementations-e95f5ecfe88450a38e0fd6e153fc9259d010d9ab920e3fe0dd141bf675c413df}}' - name: task-name value: create-file - name: parent-dag-id @@ -274,12 +274,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-711e8b6e3ae4585024167b39bc927d87c5c6dd39e2ab20b0613ea96fd049dc4a}}' + value: '{{workflow.parameters.components-f59c851c3c0a1e03b3e2065384eb3c4665ce5f234998fed18cb6afb34ce005a7}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-read-file"},"dependentTasks":["create-file"],"inputs":{"artifacts":{"file":{"taskOutputArtifact":{"outputArtifactKey":"file","producerTask":"create-file"}}}},"taskInfo":{"name":"same display name"}}' - name: container - value: '{{workflow.parameters.implementations-711e8b6e3ae4585024167b39bc927d87c5c6dd39e2ab20b0613ea96fd049dc4a}}' + value: '{{workflow.parameters.implementations-f59c851c3c0a1e03b3e2065384eb3c4665ce5f234998fed18cb6afb34ce005a7}}' - name: task-name value: read-file - name: parent-dag-id @@ -451,12 +451,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ca2a87b6895df4cb47f1b6631daadcc4a5634c47ac3aadb0d316b2910f3b7794}}' + value: '{{workflow.parameters.components-b4e9d76b938751b0b1edf9cce7cc5817c0b9c54d34a1e06c3a226a46cc2c3f1e}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-input"},"dependentTasks":["split-input"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-input"}}}},"taskInfo":{"name":"same display name"}}' - name: container - value: '{{workflow.parameters.implementations-ca2a87b6895df4cb47f1b6631daadcc4a5634c47ac3aadb0d316b2910f3b7794}}' + value: '{{workflow.parameters.implementations-b4e9d76b938751b0b1edf9cce7cc5817c0b9c54d34a1e06c3a226a46cc2c3f1e}}' - name: task-name value: print-input - name: parent-dag-id @@ -477,12 +477,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ed7019fac34ed83d85bd9c5b5fd36b6f73f1211ad397845e5f71e73680a4d5a6}}' + value: '{{workflow.parameters.components-fc5dd8cf7a18cc02226309bb936fb376a98741769b2faba10d1bfc9960de8802}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-split-input"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"component1,component2,component3"}}}},"taskInfo":{"name":"same display name"}}' - name: container - value: '{{workflow.parameters.implementations-ed7019fac34ed83d85bd9c5b5fd36b6f73f1211ad397845e5f71e73680a4d5a6}}' + value: '{{workflow.parameters.implementations-fc5dd8cf7a18cc02226309bb936fb376a98741769b2faba10d1bfc9960de8802}}' - name: task-name value: split-input - name: parent-dag-id diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index edf5b04fc70..f374e5e8b10 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-1e13061c086c82fca98c3bec1d9f3c373acd890377eb9145cfdb7f69de426019 + - name: components-f3f63f614ffceea8307ebdcd5f7c7a8ba2bf88301e6b6f4ef2771a43a1ac2e55 value: '{"executorLabel":"exec-digit-classification","outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-1e13061c086c82fca98c3bec1d9f3c373acd890377eb9145cfdb7f69de426019 + - name: implementations-f3f63f614ffceea8307ebdcd5f7c7a8ba2bf88301e6b6f4ef2771a43a1ac2e55 value: '{"args":["--executor_input","{{$}}","--function_to_execute","digit_classification"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,10 +31,10 @@ spec: y_train, y_test = model_selection.train_test_split(\n X, y, test_size=test_size, random_state=seed)\n #fit model\n model.fit(X_train, y_train)\n\n #accuracy on test set\n result = model.score(X_test, y_test)\n metrics.log_metric(''accuracy'', - (result * 100.0))\n\n"],"image":"python:3.9"}' - - name: components-507141586ba39388ea617640b16454b71ab71dfefbfda1d7add5fd6b13fc6f36 + (result * 100.0))\n\n"],"image":"python:3.11"}' + - name: components-b3c7dd98d963c8b8e8f52c8c62b593f94e99c900733c383027ffba46765ac7fb value: '{"executorLabel":"exec-html-visualization","outputDefinitions":{"artifacts":{"html_artifact":{"artifactType":{"schemaTitle":"system.HTML","schemaVersion":"0.0.1"}}}}}' - - name: implementations-507141586ba39388ea617640b16454b71ab71dfefbfda1d7add5fd6b13fc6f36 + - name: implementations-b3c7dd98d963c8b8e8f52c8c62b593f94e99c900733c383027ffba46765ac7fb value: '{"args":["--executor_input","{{$}}","--function_to_execute","html_visualization"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -46,10 +46,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef html_visualization(html_artifact: Output[HTML]):\n html_content = ''\u003c!DOCTYPE html\u003e\u003chtml\u003e\u003cbody\u003e\u003ch1\u003eHello world\u003c/h1\u003e\u003c/body\u003e\u003c/html\u003e''\n with - open(html_artifact.path, ''w'') as f:\n f.write(html_content)\n\n"],"image":"python:3.9"}' - - name: components-070c6090f000ab4579ea9970fdbe93fd311d1b7a3d4e4c9735898c872ed91a8f + open(html_artifact.path, ''w'') as f:\n f.write(html_content)\n\n"],"image":"python:3.11"}' + - name: components-9f3641fea11e44f30b9e256d336098510b78bd0f423c003ee0ce74da34c7fdc1 value: '{"executorLabel":"exec-iris-sgdclassifier","inputDefinitions":{"parameters":{"test_samples_fraction":{"parameterType":"NUMBER_DOUBLE"}}},"outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.ClassificationMetrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-070c6090f000ab4579ea9970fdbe93fd311d1b7a3d4e4c9735898c872ed91a8f + - name: implementations-9f3641fea11e44f30b9e256d336098510b78bd0f423c003ee0ce74da34c7fdc1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","iris_sgdclassifier"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -69,10 +69,10 @@ spec: model_selection.cross_val_predict(\n classifier, train_x, train_y, cv=3)\n metrics.log_confusion_matrix(\n [''Setosa'', ''Versicolour'', ''Virginica''],\n confusion_matrix(\n train_y,\n predictions).tolist() # - .tolist() to convert np array to list.\n )\n\n"],"image":"python:3.9"}' - - name: components-837e3b5eebdb2f6e1895391076d11f242320d5ee805a3712c681d87af27a0322 + .tolist() to convert np array to list.\n )\n\n"],"image":"python:3.11"}' + - name: components-b4e81501f4cd8d1ff945c7fd6ae18a8abb3f66daa3fe9e5d75df12714a185308 value: '{"executorLabel":"exec-markdown-visualization","outputDefinitions":{"artifacts":{"markdown_artifact":{"artifactType":{"schemaTitle":"system.Markdown","schemaVersion":"0.0.1"}}}}}' - - name: implementations-837e3b5eebdb2f6e1895391076d11f242320d5ee805a3712c681d87af27a0322 + - name: implementations-b4e81501f4cd8d1ff945c7fd6ae18a8abb3f66daa3fe9e5d75df12714a185308 value: '{"args":["--executor_input","{{$}}","--function_to_execute","markdown_visualization"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -84,10 +84,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef markdown_visualization(markdown_artifact: Output[Markdown]):\n markdown_content = ''## Hello world \\n\\n Markdown content''\n with open(markdown_artifact.path, - ''w'') as f:\n f.write(markdown_content)\n\n"],"image":"python:3.9"}' - - name: components-cedc2096e3940b72ca9724db5ab3d0ab9680a848e3c724b60de8adea53f2b7c8 + ''w'') as f:\n f.write(markdown_content)\n\n"],"image":"python:3.11"}' + - name: components-58d11750818090dbfe31db33200e5a2dfc0b5b9d742ccdd90fea504248ab4dba value: '{"executorLabel":"exec-wine-classification","outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.ClassificationMetrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-cedc2096e3940b72ca9724db5ab3d0ab9680a848e3c724b60de8adea53f2b7c8 + - name: implementations-58d11750818090dbfe31db33200e5a2dfc0b5b9d742ccdd90fea504248ab4dba value: '{"args":["--executor_input","{{$}}","--function_to_execute","wine_classification"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -110,7 +110,7 @@ spec: y_train, cv=3, method=''predict'')\n fpr, tpr, thresholds = roc_curve(\n y_true=y_train, y_score=y_scores[:, 1], pos_label=True)\n\n # avoid inf thresholds\n epsilon = 1e-6\n thresholds = [1 - epsilon if t == float(''inf'') else t for t - in thresholds]\n\n metrics.log_roc_curve(fpr, tpr, thresholds)\n\n"],"image":"python:3.9"}' + in thresholds]\n\n metrics.log_roc_curve(fpr, tpr, thresholds)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"digit-classification":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-digit-classification"},"taskInfo":{"name":"digit-classification"}},"html-visualization":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-html-visualization"},"taskInfo":{"name":"html-visualization"}},"iris-sgdclassifier":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-iris-sgdclassifier"},"inputs":{"parameters":{"test_samples_fraction":{"runtimeValue":{"constant":0.3}}}},"taskInfo":{"name":"iris-sgdclassifier"}},"markdown-visualization":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-markdown-visualization"},"taskInfo":{"name":"markdown-visualization"}},"wine-classification":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-wine-classification"},"taskInfo":{"name":"wine-classification"}}}}}' entrypoint: entrypoint @@ -293,11 +293,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-1e13061c086c82fca98c3bec1d9f3c373acd890377eb9145cfdb7f69de426019}}' + value: '{{workflow.parameters.components-f3f63f614ffceea8307ebdcd5f7c7a8ba2bf88301e6b6f4ef2771a43a1ac2e55}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-digit-classification"},"taskInfo":{"name":"digit-classification"}}' - name: container - value: '{{workflow.parameters.implementations-1e13061c086c82fca98c3bec1d9f3c373acd890377eb9145cfdb7f69de426019}}' + value: '{{workflow.parameters.implementations-f3f63f614ffceea8307ebdcd5f7c7a8ba2bf88301e6b6f4ef2771a43a1ac2e55}}' - name: task-name value: digit-classification - name: parent-dag-id @@ -317,11 +317,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-507141586ba39388ea617640b16454b71ab71dfefbfda1d7add5fd6b13fc6f36}}' + value: '{{workflow.parameters.components-b3c7dd98d963c8b8e8f52c8c62b593f94e99c900733c383027ffba46765ac7fb}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-html-visualization"},"taskInfo":{"name":"html-visualization"}}' - name: container - value: '{{workflow.parameters.implementations-507141586ba39388ea617640b16454b71ab71dfefbfda1d7add5fd6b13fc6f36}}' + value: '{{workflow.parameters.implementations-b3c7dd98d963c8b8e8f52c8c62b593f94e99c900733c383027ffba46765ac7fb}}' - name: task-name value: html-visualization - name: parent-dag-id @@ -341,11 +341,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-070c6090f000ab4579ea9970fdbe93fd311d1b7a3d4e4c9735898c872ed91a8f}}' + value: '{{workflow.parameters.components-9f3641fea11e44f30b9e256d336098510b78bd0f423c003ee0ce74da34c7fdc1}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-iris-sgdclassifier"},"inputs":{"parameters":{"test_samples_fraction":{"runtimeValue":{"constant":0.3}}}},"taskInfo":{"name":"iris-sgdclassifier"}}' - name: container - value: '{{workflow.parameters.implementations-070c6090f000ab4579ea9970fdbe93fd311d1b7a3d4e4c9735898c872ed91a8f}}' + value: '{{workflow.parameters.implementations-9f3641fea11e44f30b9e256d336098510b78bd0f423c003ee0ce74da34c7fdc1}}' - name: task-name value: iris-sgdclassifier - name: parent-dag-id @@ -365,11 +365,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-837e3b5eebdb2f6e1895391076d11f242320d5ee805a3712c681d87af27a0322}}' + value: '{{workflow.parameters.components-b4e81501f4cd8d1ff945c7fd6ae18a8abb3f66daa3fe9e5d75df12714a185308}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-markdown-visualization"},"taskInfo":{"name":"markdown-visualization"}}' - name: container - value: '{{workflow.parameters.implementations-837e3b5eebdb2f6e1895391076d11f242320d5ee805a3712c681d87af27a0322}}' + value: '{{workflow.parameters.implementations-b4e81501f4cd8d1ff945c7fd6ae18a8abb3f66daa3fe9e5d75df12714a185308}}' - name: task-name value: markdown-visualization - name: parent-dag-id @@ -389,11 +389,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-cedc2096e3940b72ca9724db5ab3d0ab9680a848e3c724b60de8adea53f2b7c8}}' + value: '{{workflow.parameters.components-58d11750818090dbfe31db33200e5a2dfc0b5b9d742ccdd90fea504248ab4dba}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-wine-classification"},"taskInfo":{"name":"wine-classification"}}' - name: container - value: '{{workflow.parameters.implementations-cedc2096e3940b72ca9724db5ab3d0ab9680a848e3c724b60de8adea53f2b7c8}}' + value: '{{workflow.parameters.implementations-58d11750818090dbfe31db33200e5a2dfc0b5b9d742ccdd90fea504248ab4dba}}' - name: task-name value: wine-classification - name: parent-dag-id diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index a6eab5cc4e3..1f0eaba674d 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-b5a2e54f451e167f623c082d02fe4e7719610dd2654a83c57f404de3898fc086 + - name: components-bada9af0362ed45ffb9dac372e0475396c75331f2b09988c0122ea4131e18462 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"parameters":{"x":{"parameterType":"NUMBER_INTEGER"},"y":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-b5a2e54f451e167f623c082d02fe4e7719610dd2654a83c57f404de3898fc086 + - name: implementations-bada9af0362ed45ffb9dac372e0475396c75331f2b09988c0122ea4131e18462 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - crust_comp(x: int, y: int):\n print(''sum :'', x + y)\n\n"],"image":"python:3.9"}' - - name: components-551cca49c1bbb847a0aaa9b53908a3b2c671ec76f99094d68fcb82705869d508 + crust_comp(x: int, y: int):\n print(''sum :'', x + y)\n\n"],"image":"python:3.11"}' + - name: components-58716d11d7152c994e16a93768ef172bce97a8d239a026d258c0ef81751e3168 value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-551cca49c1bbb847a0aaa9b53908a3b2c671ec76f99094d68fcb82705869d508 + - name: implementations-58716d11d7152c994e16a93768ef172bce97a8d239a026d258c0ef81751e3168 value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - core_comp() -\u003e int:\n return 1\n\n"],"image":"python:3.9"}' + core_comp() -\u003e int:\n return 1\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"core-comp"}}}},"tasks":{"core-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-comp-mantle @@ -218,11 +218,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-551cca49c1bbb847a0aaa9b53908a3b2c671ec76f99094d68fcb82705869d508}}' + value: '{{workflow.parameters.components-58716d11d7152c994e16a93768ef172bce97a8d239a026d258c0ef81751e3168}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-551cca49c1bbb847a0aaa9b53908a3b2c671ec76f99094d68fcb82705869d508}}' + value: '{{workflow.parameters.implementations-58716d11d7152c994e16a93768ef172bce97a8d239a026d258c0ef81751e3168}}' - name: task-name value: core-comp - name: parent-dag-id @@ -356,11 +356,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b5a2e54f451e167f623c082d02fe4e7719610dd2654a83c57f404de3898fc086}}' + value: '{{workflow.parameters.components-bada9af0362ed45ffb9dac372e0475396c75331f2b09988c0122ea4131e18462}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"x":{"runtimeValue":{"constant":2}},"y":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-b5a2e54f451e167f623c082d02fe4e7719610dd2654a83c57f404de3898fc086}}' + value: '{{workflow.parameters.implementations-bada9af0362ed45ffb9dac372e0475396c75331f2b09988c0122ea4131e18462}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index e60fdef1e42..212ff064f02 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-00fe5acf7a3f52f8f54524edb73f7d6272194b96173074dd3cdd140327308371 + - name: components-4f60483a1bee7f772ee3001cf05964083d2f9641615161626b6146636da04fe0 value: '{"executorLabel":"exec-build-model-car","outputDefinitions":{"artifacts":{"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-00fe5acf7a3f52f8f54524edb73f7d6272194b96173074dd3cdd140327308371 + - name: implementations-4f60483a1bee7f772ee3001cf05964083d2f9641615161626b6146636da04fe0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","build_model_car"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef build_model_car(model: dsl.Output[dsl.Model]):\n # Simulate pushing the - Modelcar to an OCI registry\n model.uri = \"oci://registry.domain.local/org/repo:v1.0\"\n\n"],"image":"python:3.9"}' - - name: components-426fe71cde5719d1162c36199e1686f1b2df4551d1fa8fa13a093333ea09a456 + Modelcar to an OCI registry\n model.uri = \"oci://registry.domain.local/org/repo:v1.0\"\n\n"],"image":"python:3.11"}' + - name: components-fc08120adea24f34a0c74fbd8e185e63caa9eca74abf250b23e4ef9c8a3d6878 value: '{"executorLabel":"exec-get-model-files-list","inputDefinitions":{"artifacts":{"input_model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-426fe71cde5719d1162c36199e1686f1b2df4551d1fa8fa13a093333ea09a456 + - name: implementations-fc08120adea24f34a0c74fbd8e185e63caa9eca74abf250b23e4ef9c8a3d6878 value: '{"args":["--executor_input","{{$}}","--function_to_execute","get_model_files_list"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -39,7 +39,7 @@ spec: = set(os.listdir(input_model.path))\n\n if not filesInPath.issuperset(expected_files):\n raise RuntimeError(\n \"The model does not have expected files: \"\n + \", \".join(sorted(expected_files.difference(filesInPath)))\n )\n\n return - \", \".join(sorted(filesInPath))\n\n"],"image":"python:3.9"}' + \", \".join(sorted(filesInPath))\n\n"],"image":"python:3.11"}' - name: components-comp-importer value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - name: implementations-comp-importer @@ -283,11 +283,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-00fe5acf7a3f52f8f54524edb73f7d6272194b96173074dd3cdd140327308371}}' + value: '{{workflow.parameters.components-4f60483a1bee7f772ee3001cf05964083d2f9641615161626b6146636da04fe0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-build-model-car"},"taskInfo":{"name":"build-model-car"}}' - name: container - value: '{{workflow.parameters.implementations-00fe5acf7a3f52f8f54524edb73f7d6272194b96173074dd3cdd140327308371}}' + value: '{{workflow.parameters.implementations-4f60483a1bee7f772ee3001cf05964083d2f9641615161626b6146636da04fe0}}' - name: task-name value: build-model-car - name: parent-dag-id @@ -307,11 +307,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-426fe71cde5719d1162c36199e1686f1b2df4551d1fa8fa13a093333ea09a456}}' + value: '{{workflow.parameters.components-fc08120adea24f34a0c74fbd8e185e63caa9eca74abf250b23e4ef9c8a3d6878}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-get-model-files-list"},"dependentTasks":["importer"],"inputs":{"artifacts":{"input_model":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"get-model-files-list"}}' - name: container - value: '{{workflow.parameters.implementations-426fe71cde5719d1162c36199e1686f1b2df4551d1fa8fa13a093333ea09a456}}' + value: '{{workflow.parameters.implementations-fc08120adea24f34a0c74fbd8e185e63caa9eca74abf250b23e4ef9c8a3d6878}}' - name: task-name value: get-model-files-list - name: parent-dag-id diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index 0c649c65ad1..3c908b0dc8c 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-415b11a96a5d71255274b3e811e186d556046f1ffe8a8dfd4e13193f62447e13 + - name: components-41d6f43a77336abfc42373262a0d3f817036dea59881c80ac128ebff19d9ac06 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"artifacts":{"ds1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"ds2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-415b11a96a5d71255274b3e811e186d556046f1ffe8a8dfd4e13193f62447e13 + - name: implementations-41d6f43a77336abfc42373262a0d3f817036dea59881c80ac128ebff19d9ac06 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef crust_comp(\n ds1: dsl.Dataset,\n ds2: dsl.Dataset,\n):\n with open(ds1.path, ''r'') as f:\n print(''ds1: '', f.read())\n with open(ds2.path, - ''r'') as f:\n print(''ds2: '', f.read())\n\n"],"image":"python:3.9"}' - - name: components-0ac03a39a38b1a3cad2a272deff551384098d69749e6673beb6cfdc3a1e59015 + ''r'') as f:\n print(''ds2: '', f.read())\n\n"],"image":"python:3.11"}' + - name: components-5502b96ca3f6aca433522fdb2d6365445dee6a291a29d192a3665429bcba4eca value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"artifacts":{"ds1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"ds2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-0ac03a39a38b1a3cad2a272deff551384098d69749e6673beb6cfdc3a1e59015 + - name: implementations-5502b96ca3f6aca433522fdb2d6365445dee6a291a29d192a3665429bcba4eca value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -35,7 +35,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_comp(ds1: dsl.Output[dsl.Dataset], ds2: dsl.Output[dsl.Dataset]):\n with open(ds1.path, ''w'') as f:\n f.write(''foo'')\n with open(ds2.path, - ''w'') as f:\n f.write(''bar'')\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(''bar'')\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"artifacts":{"ds1":{"artifactSelectors":[{"outputArtifactKey":"ds1","producerSubtask":"core-comp"}]},"ds2":{"artifactSelectors":[{"outputArtifactKey":"ds2","producerSubtask":"core-comp"}]}}},"tasks":{"core-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"artifacts":{"ds1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"ds2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: components-comp-mantle @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0ac03a39a38b1a3cad2a272deff551384098d69749e6673beb6cfdc3a1e59015}}' + value: '{{workflow.parameters.components-5502b96ca3f6aca433522fdb2d6365445dee6a291a29d192a3665429bcba4eca}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-0ac03a39a38b1a3cad2a272deff551384098d69749e6673beb6cfdc3a1e59015}}' + value: '{{workflow.parameters.implementations-5502b96ca3f6aca433522fdb2d6365445dee6a291a29d192a3665429bcba4eca}}' - name: task-name value: core-comp - name: parent-dag-id @@ -360,11 +360,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-415b11a96a5d71255274b3e811e186d556046f1ffe8a8dfd4e13193f62447e13}}' + value: '{{workflow.parameters.components-41d6f43a77336abfc42373262a0d3f817036dea59881c80ac128ebff19d9ac06}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"artifacts":{"ds1":{"taskOutputArtifact":{"outputArtifactKey":"ds1","producerTask":"mantle"}},"ds2":{"taskOutputArtifact":{"outputArtifactKey":"ds2","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-415b11a96a5d71255274b3e811e186d556046f1ffe8a8dfd4e13193f62447e13}}' + value: '{{workflow.parameters.implementations-41d6f43a77336abfc42373262a0d3f817036dea59881c80ac128ebff19d9ac06}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 5b714a6cacc..6e86a053905 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-54461b7769754ecea4f755e24938448cd07c365048c2ee6e16271652b11205d8 + - name: components-4e3e2b26bcc3e5ebd2165f67c13916a89b3269c76c31b39a2e59f15e82b89020 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"parameters":{"val1":{"parameterType":"STRING"},"val2":{"parameterType":"STRING"}}}}' - - name: implementations-54461b7769754ecea4f755e24938448cd07c365048c2ee6e16271652b11205d8 + - name: implementations-4e3e2b26bcc3e5ebd2165f67c13916a89b3269c76c31b39a2e59f15e82b89020 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef crust_comp(val1: str, val2: str):\n print(''val1: '', val1)\n print(''val2: - '', val2)\n\n"],"image":"python:3.9"}' - - name: components-6d13d1db365eee59a2cdc2d89bfaa4a0c32572a64b1743f50115e2614b7b60bd + '', val2)\n\n"],"image":"python:3.11"}' + - name: components-1ebf161b377e2baf7660119b69476134f6b72ee853a2bf59b5c77ae20d35e4df value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"parameters":{"val1":{"parameterType":"STRING"},"val2":{"parameterType":"STRING"}}}}' - - name: implementations-6d13d1db365eee59a2cdc2d89bfaa4a0c32572a64b1743f50115e2614b7b60bd + - name: implementations-1ebf161b377e2baf7660119b69476134f6b72ee853a2bf59b5c77ae20d35e4df value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,7 +34,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_comp() -\u003e NamedTuple(''outputs'', val1=str, val2=str): # type: ignore\n outputs = NamedTuple(''outputs'', val1=str, val2=str)\n return - outputs(''foo'', ''bar'')\n\n"],"image":"python:3.9"}' + outputs(''foo'', ''bar'')\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"parameters":{"val1":{"valueFromParameter":{"outputParameterKey":"val1","producerSubtask":"core-comp"}},"val2":{"valueFromParameter":{"outputParameterKey":"val2","producerSubtask":"core-comp"}}}},"tasks":{"core-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"parameters":{"val1":{"parameterType":"STRING"},"val2":{"parameterType":"STRING"}}}}' - name: components-comp-mantle @@ -221,11 +221,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-6d13d1db365eee59a2cdc2d89bfaa4a0c32572a64b1743f50115e2614b7b60bd}}' + value: '{{workflow.parameters.components-1ebf161b377e2baf7660119b69476134f6b72ee853a2bf59b5c77ae20d35e4df}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-6d13d1db365eee59a2cdc2d89bfaa4a0c32572a64b1743f50115e2614b7b60bd}}' + value: '{{workflow.parameters.implementations-1ebf161b377e2baf7660119b69476134f6b72ee853a2bf59b5c77ae20d35e4df}}' - name: task-name value: core-comp - name: parent-dag-id @@ -359,11 +359,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-54461b7769754ecea4f755e24938448cd07c365048c2ee6e16271652b11205d8}}' + value: '{{workflow.parameters.components-4e3e2b26bcc3e5ebd2165f67c13916a89b3269c76c31b39a2e59f15e82b89020}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"val1":{"taskOutputParameter":{"outputParameterKey":"val1","producerTask":"mantle"}},"val2":{"taskOutputParameter":{"outputParameterKey":"val2","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-54461b7769754ecea4f755e24938448cd07c365048c2ee6e16271652b11205d8}}' + value: '{{workflow.parameters.implementations-4e3e2b26bcc3e5ebd2165f67c13916a89b3269c76c31b39a2e59f15e82b89020}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index 9375b4caf05..8c1067fef07 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-1592d3a8b70d65e90cbff58d5735d5d60ce6590d27348fa31fb79fd35cdadd33 + - name: components-67acbf0e9a3f6bfa27bfaf8979e8f91a80b1de04a3c0dffff6d475e5d8c8a2fc value: '{"executorLabel":"exec-component-a-bool","inputDefinitions":{"parameters":{"componentInputBool":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - - name: implementations-1592d3a8b70d65e90cbff58d5735d5d60ce6590d27348fa31fb79fd35cdadd33 + - name: implementations-67acbf0e9a3f6bfa27bfaf8979e8f91a80b1de04a3c0dffff6d475e5d8c8a2fc value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_a_bool"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_a_bool(componentInputBool: bool = None):\n if componentInputBool != True:\n raise ValueError(f\"componentInputBool should be True but - is {componentInputBool}\")\n\n"],"image":"python:3.9"}' - - name: components-0750c052d89cc0d094b20f49a157c30478a4f9d958c44d579ec072576221664e + is {componentInputBool}\")\n\n"],"image":"python:3.11"}' + - name: components-1b21465d9fecfb690a65b5d22e0ec7540a7652fb013f3386f36e8255dd2ab5ca value: '{"executorLabel":"exec-component-a-int","inputDefinitions":{"parameters":{"componentInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-0750c052d89cc0d094b20f49a157c30478a4f9d958c44d579ec072576221664e + - name: implementations-1b21465d9fecfb690a65b5d22e0ec7540a7652fb013f3386f36e8255dd2ab5ca value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_a_int"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,10 +34,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_a_int(componentInputInt: int = None):\n if componentInputInt - != 1:\n raise ValueError(f\"componentInputInt should be 1 but is {componentInputInt}\")\n\n"],"image":"python:3.9"}' - - name: components-7a7918ee3f5d6cb0b49dc19dfc1ce5915b1657ab8e4f1b5ebdc62dc7ec335694 + != 1:\n raise ValueError(f\"componentInputInt should be 1 but is {componentInputInt}\")\n\n"],"image":"python:3.11"}' + - name: components-e793c84d1e31d803ca345dd2c0bf915dd5e8c356537d91a002e90837789e73be value: '{"executorLabel":"exec-component-a-str","inputDefinitions":{"parameters":{"componentInputStr":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-7a7918ee3f5d6cb0b49dc19dfc1ce5915b1657ab8e4f1b5ebdc62dc7ec335694 + - name: implementations-e793c84d1e31d803ca345dd2c0bf915dd5e8c356537d91a002e90837789e73be value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_a_str"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -49,10 +49,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_a_str(componentInputStr: str = None):\n if componentInputStr != ''Input - pipeline'':\n raise ValueError(f\"componentInputStr should - be ''Input - pipeline'' but is {componentInputStr}\")\n\n"],"image":"python:3.9"}' - - name: components-95008ca85079a1497c324a0e7fb96cb610e8a736dcf259de169d00e6435c22fe + be ''Input - pipeline'' but is {componentInputStr}\")\n\n"],"image":"python:3.11"}' + - name: components-847405d2298b3810654100892c836106fe0397c48f59c14f51827ef975efcd95 value: '{"executorLabel":"exec-component-b-bool","inputDefinitions":{"parameters":{"componentInputBool":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - - name: implementations-95008ca85079a1497c324a0e7fb96cb610e8a736dcf259de169d00e6435c22fe + - name: implementations-847405d2298b3810654100892c836106fe0397c48f59c14f51827ef975efcd95 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_b_bool"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -64,10 +64,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_b_bool(componentInputBool: bool = None):\n if componentInputBool != False:\n raise ValueError(f\"componentInputBool should be False - but is {componentInputBool}\")\n\n"],"image":"python:3.9"}' - - name: components-35127cd40fb981e7d7fbeaec732ca8402a0254b636d841b3482369aaa0f0529f + but is {componentInputBool}\")\n\n"],"image":"python:3.11"}' + - name: components-fcf5d3023cec6f80952cd3ce9a483536e84ac6ae5f853a7178b710495776f2a5 value: '{"executorLabel":"exec-component-b-int","inputDefinitions":{"parameters":{"componentInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-35127cd40fb981e7d7fbeaec732ca8402a0254b636d841b3482369aaa0f0529f + - name: implementations-fcf5d3023cec6f80952cd3ce9a483536e84ac6ae5f853a7178b710495776f2a5 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_b_int"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -78,10 +78,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_b_int(componentInputInt: int = None):\n if componentInputInt - != 0:\n raise ValueError(f\"componentInputInt should be 0 but is {componentInputInt}\")\n\n"],"image":"python:3.9"}' - - name: components-f295cc45bba61d2a9dd59d4ee817c93c137713ba2b99356f4eb98907fbfde36e + != 0:\n raise ValueError(f\"componentInputInt should be 0 but is {componentInputInt}\")\n\n"],"image":"python:3.11"}' + - name: components-319fe909f6b49ecb503c73c273e622302238293692577f00c9d93d0071fef705 value: '{"executorLabel":"exec-component-b-str","inputDefinitions":{"parameters":{"componentInputStr":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-f295cc45bba61d2a9dd59d4ee817c93c137713ba2b99356f4eb98907fbfde36e + - name: implementations-319fe909f6b49ecb503c73c273e622302238293692577f00c9d93d0071fef705 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_b_str"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -93,7 +93,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_b_str(componentInputStr: str = None):\n if componentInputStr != ''Input 2 - nested pipeline'':\n raise ValueError(f\"componentInputStr - should be ''Input 2 - nested pipeline'' but is {componentInputStr}\")\n\n"],"image":"python:3.9"}' + should be ''Input 2 - nested pipeline'' but is {componentInputStr}\")\n\n"],"image":"python:3.11"}' - name: components-comp-nested-pipeline value: '{"dag":{"tasks":{"component-a-bool":{"cachingOptions":{},"componentRef":{"name":"comp-component-a-bool"},"inputs":{"parameters":{"componentInputBool":{"componentInputParameter":"nestedInputBool1"}}},"taskInfo":{"name":"component-a-bool"}},"component-a-int":{"cachingOptions":{},"componentRef":{"name":"comp-component-a-int"},"inputs":{"parameters":{"componentInputInt":{"componentInputParameter":"nestedInputInt1"}}},"taskInfo":{"name":"component-a-int"}},"component-a-str":{"cachingOptions":{},"componentRef":{"name":"comp-component-a-str"},"inputs":{"parameters":{"componentInputStr":{"componentInputParameter":"nestedInputStr1"}}},"taskInfo":{"name":"component-a-str"}},"component-b-bool":{"cachingOptions":{},"componentRef":{"name":"comp-component-b-bool"},"inputs":{"parameters":{"componentInputBool":{"componentInputParameter":"nestedInputBool2"}}},"taskInfo":{"name":"component-b-bool"}},"component-b-int":{"cachingOptions":{},"componentRef":{"name":"comp-component-b-int"},"inputs":{"parameters":{"componentInputInt":{"componentInputParameter":"nestedInputInt2"}}},"taskInfo":{"name":"component-b-int"}},"component-b-str":{"cachingOptions":{},"componentRef":{"name":"comp-component-b-str"},"inputs":{"parameters":{"componentInputStr":{"componentInputParameter":"nestedInputStr2"}}},"taskInfo":{"name":"component-b-str"}}}},"inputDefinitions":{"parameters":{"nestedInputBool1":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputBool2":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt1":{"defaultValue":0,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputInt2":{"defaultValue":0,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr1":{"defaultValue":"Input 1 - nested pipeline","isOptional":true,"parameterType":"STRING"},"nestedInputStr2":{"defaultValue":"Input @@ -281,11 +281,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-1592d3a8b70d65e90cbff58d5735d5d60ce6590d27348fa31fb79fd35cdadd33}}' + value: '{{workflow.parameters.components-67acbf0e9a3f6bfa27bfaf8979e8f91a80b1de04a3c0dffff6d475e5d8c8a2fc}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-a-bool"},"inputs":{"parameters":{"componentInputBool":{"componentInputParameter":"nestedInputBool1"}}},"taskInfo":{"name":"component-a-bool"}}' - name: container - value: '{{workflow.parameters.implementations-1592d3a8b70d65e90cbff58d5735d5d60ce6590d27348fa31fb79fd35cdadd33}}' + value: '{{workflow.parameters.implementations-67acbf0e9a3f6bfa27bfaf8979e8f91a80b1de04a3c0dffff6d475e5d8c8a2fc}}' - name: task-name value: component-a-bool - name: parent-dag-id @@ -305,11 +305,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0750c052d89cc0d094b20f49a157c30478a4f9d958c44d579ec072576221664e}}' + value: '{{workflow.parameters.components-1b21465d9fecfb690a65b5d22e0ec7540a7652fb013f3386f36e8255dd2ab5ca}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-a-int"},"inputs":{"parameters":{"componentInputInt":{"componentInputParameter":"nestedInputInt1"}}},"taskInfo":{"name":"component-a-int"}}' - name: container - value: '{{workflow.parameters.implementations-0750c052d89cc0d094b20f49a157c30478a4f9d958c44d579ec072576221664e}}' + value: '{{workflow.parameters.implementations-1b21465d9fecfb690a65b5d22e0ec7540a7652fb013f3386f36e8255dd2ab5ca}}' - name: task-name value: component-a-int - name: parent-dag-id @@ -329,11 +329,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7a7918ee3f5d6cb0b49dc19dfc1ce5915b1657ab8e4f1b5ebdc62dc7ec335694}}' + value: '{{workflow.parameters.components-e793c84d1e31d803ca345dd2c0bf915dd5e8c356537d91a002e90837789e73be}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-a-str"},"inputs":{"parameters":{"componentInputStr":{"componentInputParameter":"nestedInputStr1"}}},"taskInfo":{"name":"component-a-str"}}' - name: container - value: '{{workflow.parameters.implementations-7a7918ee3f5d6cb0b49dc19dfc1ce5915b1657ab8e4f1b5ebdc62dc7ec335694}}' + value: '{{workflow.parameters.implementations-e793c84d1e31d803ca345dd2c0bf915dd5e8c356537d91a002e90837789e73be}}' - name: task-name value: component-a-str - name: parent-dag-id @@ -353,11 +353,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-95008ca85079a1497c324a0e7fb96cb610e8a736dcf259de169d00e6435c22fe}}' + value: '{{workflow.parameters.components-847405d2298b3810654100892c836106fe0397c48f59c14f51827ef975efcd95}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-b-bool"},"inputs":{"parameters":{"componentInputBool":{"componentInputParameter":"nestedInputBool2"}}},"taskInfo":{"name":"component-b-bool"}}' - name: container - value: '{{workflow.parameters.implementations-95008ca85079a1497c324a0e7fb96cb610e8a736dcf259de169d00e6435c22fe}}' + value: '{{workflow.parameters.implementations-847405d2298b3810654100892c836106fe0397c48f59c14f51827ef975efcd95}}' - name: task-name value: component-b-bool - name: parent-dag-id @@ -377,11 +377,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-35127cd40fb981e7d7fbeaec732ca8402a0254b636d841b3482369aaa0f0529f}}' + value: '{{workflow.parameters.components-fcf5d3023cec6f80952cd3ce9a483536e84ac6ae5f853a7178b710495776f2a5}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-b-int"},"inputs":{"parameters":{"componentInputInt":{"componentInputParameter":"nestedInputInt2"}}},"taskInfo":{"name":"component-b-int"}}' - name: container - value: '{{workflow.parameters.implementations-35127cd40fb981e7d7fbeaec732ca8402a0254b636d841b3482369aaa0f0529f}}' + value: '{{workflow.parameters.implementations-fcf5d3023cec6f80952cd3ce9a483536e84ac6ae5f853a7178b710495776f2a5}}' - name: task-name value: component-b-int - name: parent-dag-id @@ -401,11 +401,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f295cc45bba61d2a9dd59d4ee817c93c137713ba2b99356f4eb98907fbfde36e}}' + value: '{{workflow.parameters.components-319fe909f6b49ecb503c73c273e622302238293692577f00c9d93d0071fef705}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-b-str"},"inputs":{"parameters":{"componentInputStr":{"componentInputParameter":"nestedInputStr2"}}},"taskInfo":{"name":"component-b-str"}}' - name: container - value: '{{workflow.parameters.implementations-f295cc45bba61d2a9dd59d4ee817c93c137713ba2b99356f4eb98907fbfde36e}}' + value: '{{workflow.parameters.implementations-319fe909f6b49ecb503c73c273e622302238293692577f00c9d93d0071fef705}}' - name: task-name value: component-b-str - name: parent-dag-id diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index 710aa68bff1..4a0d9a9abfe 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce + - name: components-ea55a4b32ea777d21165417782052ce723fbedd5f411eb3ea22cf9ef7e574167 value: '{"executorLabel":"exec-component-bool","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - - name: implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce + - name: implementations-ea55a4b32ea777d21165417782052ce723fbedd5f411eb3ea22cf9ef7e574167 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_bool"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_bool(componentInput: bool = None):\n if componentInput is not - None:\n raise ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934 + None:\n raise ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-edd8fe327d9525f18fce56ac94b03f295b8e9dff3dadf0403760de87f18ddcb2 value: '{"executorLabel":"exec-component-int","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934 + - name: implementations-edd8fe327d9525f18fce56ac94b03f295b8e9dff3dadf0403760de87f18ddcb2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_int"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,10 +33,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_int(componentInput: int = None):\n if componentInput is not None:\n raise - ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2 + ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-4a1ad43e53e46919879caa0a1100eef09d1fbcd99a4afae74fabbe11fcb381ce value: '{"executorLabel":"exec-component-str","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2 + - name: implementations-4a1ad43e53e46919879caa0a1100eef09d1fbcd99a4afae74fabbe11fcb381ce value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_str"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -47,7 +47,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_str(componentInput: str = None):\n if componentInput is not None:\n raise - ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.9"}' + ValueError(f\"componentInput should be None but is {componentInput}\")\n\n"],"image":"python:3.11"}' - name: components-comp-nested-pipeline value: '{"dag":{"tasks":{"component-bool":{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}},"component-int":{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}},"component-str":{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}}},"inputDefinitions":{"parameters":{"nestedInputBool":{"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr":{"isOptional":true,"parameterType":"STRING"}}}}' - name: components-root @@ -232,11 +232,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' + value: '{{workflow.parameters.components-ea55a4b32ea777d21165417782052ce723fbedd5f411eb3ea22cf9ef7e574167}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}}' - name: container - value: '{{workflow.parameters.implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' + value: '{{workflow.parameters.implementations-ea55a4b32ea777d21165417782052ce723fbedd5f411eb3ea22cf9ef7e574167}}' - name: task-name value: component-bool - name: parent-dag-id @@ -256,11 +256,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' + value: '{{workflow.parameters.components-edd8fe327d9525f18fce56ac94b03f295b8e9dff3dadf0403760de87f18ddcb2}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}}' - name: container - value: '{{workflow.parameters.implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' + value: '{{workflow.parameters.implementations-edd8fe327d9525f18fce56ac94b03f295b8e9dff3dadf0403760de87f18ddcb2}}' - name: task-name value: component-int - name: parent-dag-id @@ -280,11 +280,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' + value: '{{workflow.parameters.components-4a1ad43e53e46919879caa0a1100eef09d1fbcd99a4afae74fabbe11fcb381ce}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}' - name: container - value: '{{workflow.parameters.implementations-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' + value: '{{workflow.parameters.implementations-4a1ad43e53e46919879caa0a1100eef09d1fbcd99a4afae74fabbe11fcb381ce}}' - name: task-name value: component-str - name: parent-dag-id diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index 5af5f118f1c..cadf245eb26 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-4a30dca88fda7931eeb4feb50afec0eace855d5b8d3bbe5f2932c20743957731 + - name: components-3fb1a1c38ece9fdf50975220e7cfae8f23547ef21522393d7e590333a4dbfd75 value: '{"executorLabel":"exec-component-nil-bool-default","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - - name: implementations-4a30dca88fda7931eeb4feb50afec0eace855d5b8d3bbe5f2932c20743957731 + - name: implementations-3fb1a1c38ece9fdf50975220e7cfae8f23547ef21522393d7e590333a4dbfd75 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_nil_bool_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_nil_bool_default(componentInput: bool = None):\n if componentInput != True:\n raise ValueError(f\"componentInput should be True but is - {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-9df5116e23bd19dbe520aeb571b4e8e7d639536a628a8608988c6424e43e453e + {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-22fb7cbea158fa49c81562ab2996be3fe3b07c161e8979367050d728d411e29c value: '{"executorLabel":"exec-component-nil-int-default","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-9df5116e23bd19dbe520aeb571b4e8e7d639536a628a8608988c6424e43e453e + - name: implementations-22fb7cbea158fa49c81562ab2996be3fe3b07c161e8979367050d728d411e29c value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_nil_int_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,10 +34,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_nil_int_default(componentInput: int = None):\n if componentInput - != 1:\n raise ValueError(f\"componentInput should be 1 but is {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-4767fec1d25c0d4ccee124435b615e8f733fad285ae87247472a210f2f5d359d + != 1:\n raise ValueError(f\"componentInput should be 1 but is {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-af9421038bc7cb5eeb32efd4d2a0c050a4d7988646bac55e6aaaa06cabc9304d value: '{"executorLabel":"exec-component-nil-str-default","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-4767fec1d25c0d4ccee124435b615e8f733fad285ae87247472a210f2f5d359d + - name: implementations-af9421038bc7cb5eeb32efd4d2a0c050a4d7988646bac55e6aaaa06cabc9304d value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_nil_str_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -49,13 +49,13 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_nil_str_default(componentInput: str = None):\n if componentInput != ''Input - parent pipeline'':\n raise ValueError(f\"componentInput - should be ''Input - parent pipeline'' but is {componentInput}\")\n\n"],"image":"python:3.9"}' + should be ''Input - parent pipeline'' but is {componentInput}\")\n\n"],"image":"python:3.11"}' - name: components-comp-nested-pipeline-nil-defaults value: '{"dag":{"tasks":{"component-nil-bool-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-bool-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-nil-bool-default"}},"component-nil-int-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-int-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-nil-int-default"}},"component-nil-str-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-str-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-nil-str-default"}}}},"inputDefinitions":{"parameters":{"nestedInputBool":{"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr":{"defaultValue":"Input - nested pipeline","isOptional":true,"parameterType":"STRING"}}}}' - - name: components-9fd2d7643bca745728790c89fea0cb45165152855e2d2c76db913d0a46987c90 + - name: components-0471bffb66a33e2f351040cca3cec0486ec93f968cc8eae19f63a7c81e9f5916 value: '{"executorLabel":"exec-component-bool-default","inputDefinitions":{"parameters":{"componentInput":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"}}}}' - - name: implementations-9fd2d7643bca745728790c89fea0cb45165152855e2d2c76db913d0a46987c90 + - name: implementations-0471bffb66a33e2f351040cca3cec0486ec93f968cc8eae19f63a7c81e9f5916 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_bool_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -67,10 +67,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_bool_default(componentInput: bool = False):\n if componentInput != True:\n raise ValueError(f\"componentInput should be True but is - {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-f2229e13b10e8a9cba43225f9f37de307a58f77689b763ddca00e6c1a91ebeb9 + {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-0f1bb762821e1923f608b9b5c75f9fb3e4a628508c2343e9cc383eff2bc0ae3e value: '{"executorLabel":"exec-component-int-default","inputDefinitions":{"parameters":{"componentInput":{"defaultValue":0,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-f2229e13b10e8a9cba43225f9f37de307a58f77689b763ddca00e6c1a91ebeb9 + - name: implementations-0f1bb762821e1923f608b9b5c75f9fb3e4a628508c2343e9cc383eff2bc0ae3e value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_int_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -81,11 +81,11 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_int_default(componentInput: int = 0):\n if componentInput != - 1:\n raise ValueError(f\"componentInput should be 1 but is {componentInput}\")\n\n"],"image":"python:3.9"}' - - name: components-0a219828dd0af40be7391d96b166bbb7aea267342dde59869750e8b85c6306ff + 1:\n raise ValueError(f\"componentInput should be 1 but is {componentInput}\")\n\n"],"image":"python:3.11"}' + - name: components-fb41f663306ed858f33b40f076339dd4af802def6164ef138a8c278c8dd2647e value: '{"executorLabel":"exec-component-str-default","inputDefinitions":{"parameters":{"componentInput":{"defaultValue":"Input - component","isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-0a219828dd0af40be7391d96b166bbb7aea267342dde59869750e8b85c6306ff + - name: implementations-fb41f663306ed858f33b40f076339dd4af802def6164ef138a8c278c8dd2647e value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_str_default"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -97,7 +97,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef component_str_default(componentInput: str = ''Input - component''):\n if componentInput != ''Input - parent pipeline'':\n raise ValueError(f\"componentInput - should be ''Input - parent pipeline'' but is {componentInput}\")\n\n"],"image":"python:3.9"}' + should be ''Input - parent pipeline'' but is {componentInput}\")\n\n"],"image":"python:3.11"}' - name: components-comp-nested-pipeline-non-nil-defaults value: '{"dag":{"tasks":{"component-bool-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-bool-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool-default"}},"component-int-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-int-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int-default"}},"component-str-default":{"cachingOptions":{},"componentRef":{"name":"comp-component-str-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str-default"}}}},"inputDefinitions":{"parameters":{"nestedInputBool":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt":{"defaultValue":0,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr":{"defaultValue":"Input - nested pipeline","isOptional":true,"parameterType":"STRING"}}}}' @@ -284,11 +284,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4a30dca88fda7931eeb4feb50afec0eace855d5b8d3bbe5f2932c20743957731}}' + value: '{{workflow.parameters.components-3fb1a1c38ece9fdf50975220e7cfae8f23547ef21522393d7e590333a4dbfd75}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-bool-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-nil-bool-default"}}' - name: container - value: '{{workflow.parameters.implementations-4a30dca88fda7931eeb4feb50afec0eace855d5b8d3bbe5f2932c20743957731}}' + value: '{{workflow.parameters.implementations-3fb1a1c38ece9fdf50975220e7cfae8f23547ef21522393d7e590333a4dbfd75}}' - name: task-name value: component-nil-bool-default - name: parent-dag-id @@ -308,11 +308,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9df5116e23bd19dbe520aeb571b4e8e7d639536a628a8608988c6424e43e453e}}' + value: '{{workflow.parameters.components-22fb7cbea158fa49c81562ab2996be3fe3b07c161e8979367050d728d411e29c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-int-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-nil-int-default"}}' - name: container - value: '{{workflow.parameters.implementations-9df5116e23bd19dbe520aeb571b4e8e7d639536a628a8608988c6424e43e453e}}' + value: '{{workflow.parameters.implementations-22fb7cbea158fa49c81562ab2996be3fe3b07c161e8979367050d728d411e29c}}' - name: task-name value: component-nil-int-default - name: parent-dag-id @@ -332,11 +332,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4767fec1d25c0d4ccee124435b615e8f733fad285ae87247472a210f2f5d359d}}' + value: '{{workflow.parameters.components-af9421038bc7cb5eeb32efd4d2a0c050a4d7988646bac55e6aaaa06cabc9304d}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-nil-str-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-nil-str-default"}}' - name: container - value: '{{workflow.parameters.implementations-4767fec1d25c0d4ccee124435b615e8f733fad285ae87247472a210f2f5d359d}}' + value: '{{workflow.parameters.implementations-af9421038bc7cb5eeb32efd4d2a0c050a4d7988646bac55e6aaaa06cabc9304d}}' - name: task-name value: component-nil-str-default - name: parent-dag-id @@ -364,11 +364,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9fd2d7643bca745728790c89fea0cb45165152855e2d2c76db913d0a46987c90}}' + value: '{{workflow.parameters.components-0471bffb66a33e2f351040cca3cec0486ec93f968cc8eae19f63a7c81e9f5916}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-bool-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool-default"}}' - name: container - value: '{{workflow.parameters.implementations-9fd2d7643bca745728790c89fea0cb45165152855e2d2c76db913d0a46987c90}}' + value: '{{workflow.parameters.implementations-0471bffb66a33e2f351040cca3cec0486ec93f968cc8eae19f63a7c81e9f5916}}' - name: task-name value: component-bool-default - name: parent-dag-id @@ -388,11 +388,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f2229e13b10e8a9cba43225f9f37de307a58f77689b763ddca00e6c1a91ebeb9}}' + value: '{{workflow.parameters.components-0f1bb762821e1923f608b9b5c75f9fb3e4a628508c2343e9cc383eff2bc0ae3e}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-int-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int-default"}}' - name: container - value: '{{workflow.parameters.implementations-f2229e13b10e8a9cba43225f9f37de307a58f77689b763ddca00e6c1a91ebeb9}}' + value: '{{workflow.parameters.implementations-0f1bb762821e1923f608b9b5c75f9fb3e4a628508c2343e9cc383eff2bc0ae3e}}' - name: task-name value: component-int-default - name: parent-dag-id @@ -412,11 +412,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0a219828dd0af40be7391d96b166bbb7aea267342dde59869750e8b85c6306ff}}' + value: '{{workflow.parameters.components-fb41f663306ed858f33b40f076339dd4af802def6164ef138a8c278c8dd2647e}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-str-default"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str-default"}}' - name: container - value: '{{workflow.parameters.implementations-0a219828dd0af40be7391d96b166bbb7aea267342dde59869750e8b85c6306ff}}' + value: '{{workflow.parameters.implementations-fb41f663306ed858f33b40f076339dd4af802def6164ef138a8c278c8dd2647e}}' - name: task-name value: component-str-default - name: parent-dag-id diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index d088e35030f..2f97c738afb 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-9206f02c3bc8c7317ae618f681a47d92020ce2a4461722611ba616d3ca7af069 + - name: components-70e868fc6ce044c86e37af5f3bcd230d34c20381b89800a6273edd2f80a3798a value: '{"executorLabel":"exec-nested-return","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-9206f02c3bc8c7317ae618f681a47d92020ce2a4461722611ba616d3ca7af069 + - name: implementations-70e868fc6ce044c86e37af5f3bcd230d34c20381b89800a6273edd2f80a3798a value: '{"args":["--executor_input","{{$}}","--function_to_execute","nested_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,7 +19,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef nested_return() -\u003e List[Dict[str, str]]:\n return [{''A_a'': ''1'', - ''B_b'': ''2''}, {''A_a'': ''10'', ''B_b'': ''20''}]\n\n"],"image":"python:3.9"}' + ''B_b'': ''2''}, {''A_a'': ''10'', ''B_b'': ''20''}]\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"nested-return"}}}},"tasks":{"nested-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-return"},"taskInfo":{"name":"nested-return"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' entrypoint: entrypoint @@ -202,11 +202,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9206f02c3bc8c7317ae618f681a47d92020ce2a4461722611ba616d3ca7af069}}' + value: '{{workflow.parameters.components-70e868fc6ce044c86e37af5f3bcd230d34c20381b89800a6273edd2f80a3798a}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-return"},"taskInfo":{"name":"nested-return"}}' - name: container - value: '{{workflow.parameters.implementations-9206f02c3bc8c7317ae618f681a47d92020ce2a4461722611ba616d3ca7af069}}' + value: '{{workflow.parameters.implementations-70e868fc6ce044c86e37af5f3bcd230d34c20381b89800a6273edd2f80a3798a}}' - name: task-name value: nested-return - name: parent-dag-id diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 3c3cb5d01f9..0ccadef5840 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61 + - name: components-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3 value: '{"executorLabel":"exec-add","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61 + - name: implementations-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef add(nums: List[List[int]]) -\u003e int:\n import itertools\n return - sum(itertools.chain(*nums))\n\n"],"image":"python:3.9"}' - - name: components-354039a8e11ca05821b8894c847eb4ea3388b9500e7db0411d4a9e68acfe16c6 + sum(itertools.chain(*nums))\n\n"],"image":"python:3.11"}' + - name: components-a2ff9335a8c3625e01d0cf6e331727c113a989bfb48adfd433b90721c60b3bf7 value: '{"executorLabel":"exec-add-two-nums","inputDefinitions":{"parameters":{"x":{"parameterType":"NUMBER_INTEGER"},"y":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-354039a8e11ca05821b8894c847eb4ea3388b9500e7db0411d4a9e68acfe16c6 + - name: implementations-a2ff9335a8c3625e01d0cf6e331727c113a989bfb48adfd433b90721c60b3bf7 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_two_nums"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,10 +32,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add_two_nums(x: int, y: int) -\u003e int:\n return x + y\n\n"],"image":"python:3.9"}' - - name: components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + add_two_nums(x: int, y: int) -\u003e int:\n return x + y\n\n"],"image":"python:3.11"}' + - name: components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -45,7 +45,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.9"}' + double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-4 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--add-two-nums-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"add-two-nums"}}}},"tasks":{"add-two-nums":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-two-nums"},"dependentTasks":["double","double-2"],"inputs":{"parameters":{"x":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"double"}},"y":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"double-2"}}}},"taskInfo":{"name":"add-two-nums"}},"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}},"double-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--loop-item-param-3":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--add-two-nums-Output":{"parameterType":"LIST"}}}}' - name: components-comp-for-loop-2 @@ -234,11 +234,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-354039a8e11ca05821b8894c847eb4ea3388b9500e7db0411d4a9e68acfe16c6}}' + value: '{{workflow.parameters.components-a2ff9335a8c3625e01d0cf6e331727c113a989bfb48adfd433b90721c60b3bf7}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-two-nums"},"dependentTasks":["double","double-2"],"inputs":{"parameters":{"x":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"double"}},"y":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"double-2"}}}},"taskInfo":{"name":"add-two-nums"}}' - name: container - value: '{{workflow.parameters.implementations-354039a8e11ca05821b8894c847eb4ea3388b9500e7db0411d4a9e68acfe16c6}}' + value: '{{workflow.parameters.implementations-a2ff9335a8c3625e01d0cf6e331727c113a989bfb48adfd433b90721c60b3bf7}}' - name: task-name value: add-two-nums - name: parent-dag-id @@ -259,11 +259,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double - name: parent-dag-id @@ -283,11 +283,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double-2 - name: parent-dag-id @@ -528,11 +528,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61}}' + value: '{{workflow.parameters.components-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--add-two-nums-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61}}' + value: '{{workflow.parameters.implementations-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3}}' - name: task-name value: add - name: parent-dag-id diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index d3dc4f7dfc4..028dda7f617 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-43c0d13fb84d9ca119bdf5124174bd3678698b085ea5c314339233390502736a + - name: components-5e4fe14151999fbde09ad1cf2ea1ef08988b5557bb7ffb2f26e71b48051a0b24 value: '{"executorLabel":"exec-evaluate-model","inputDefinitions":{"artifacts":{"model_text":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-43c0d13fb84d9ca119bdf5124174bd3678698b085ea5c314339233390502736a + - name: implementations-5e4fe14151999fbde09ad1cf2ea1ef08988b5557bb7ffb2f26e71b48051a0b24 value: '{"args":["--executor_input","{{$}}","--function_to_execute","evaluate_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -119,10 +119,10 @@ spec: open(\"/tmp/kfp_nb_outputs/metrics.json\", \"r\", encoding=\"utf-8\") as f:\n metrics_dict = json.load(f)\n\n assert metrics_dict == {\"score\": float(len(model_text))}\n\n for metric_name, metric_value in metrics_dict.items():\n metrics.log_metric(metric_name, - metric_value)\n\n"],"image":"python:3.9"}' - - name: components-de426933840df1d5b1966245bfd35532f6a11ccaffa3dbb2311a6b7dbf3ec609 + metric_value)\n\n"],"image":"python:3.11"}' + - name: components-a389d0c5411da97fdf7ec55aea7ffb8d99e46f46230c9b8890c6ee0fed4c920c value: '{"executorLabel":"exec-preprocess","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-de426933840df1d5b1966245bfd35532f6a11ccaffa3dbb2311a6b7dbf3ec609 + - name: implementations-a389d0c5411da97fdf7ec55aea7ffb8d99e46f46230c9b8890c6ee0fed4c920c value: '{"args":["--executor_input","{{$}}","--function_to_execute","preprocess"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -134,10 +134,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\n\n\ndef preprocess(text: str, dataset: dsl.Output[dsl.Dataset]):\n import re\n\n cleaned_text = re.sub(r\"\\s+\", \" \", text).strip()\n with open(dataset.path, \"w\", - encoding=\"utf-8\") as f:\n f.write(cleaned_text)\n\n"],"image":"python:3.9"}' - - name: components-0e704495704924f0f854d4c513ac164f987781625d13c72aabf7e58c7fc1d08c + encoding=\"utf-8\") as f:\n f.write(cleaned_text)\n\n"],"image":"python:3.11"}' + - name: components-428abe73fd2527425a6d4eba6c35314506906ee691a0dbe032573b8c130f58d1 value: '{"executorLabel":"exec-train-model","inputDefinitions":{"artifacts":{"cleaned_text":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-0e704495704924f0f854d4c513ac164f987781625d13c72aabf7e58c7fc1d08c + - name: implementations-428abe73fd2527425a6d4eba6c35314506906ee691a0dbe032573b8c130f58d1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","train_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -247,7 +247,7 @@ spec: \"r\", encoding=\"utf-8\") as f:\n cleaned_text = f.read()\n\n dsl.run_notebook(cleaned_text=cleaned_text)\n\n # Notebook writes its model into /tmp/kfp_nb_outputs/model.txt\n shutil.copy(\"/tmp/kfp_nb_outputs/model.txt\", model.path)\n\n with open(model.path, \"r\", encoding=\"utf-8\") as f:\n model_text - = f.read()\n\n assert model_text == cleaned_text.upper()\n\n"],"image":"python:3.9"}' + = f.read()\n\n assert model_text == cleaned_text.upper()\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"evaluate-model":{"cachingOptions":{},"componentRef":{"name":"comp-evaluate-model"},"dependentTasks":["train-model"],"inputs":{"artifacts":{"model_text":{"taskOutputArtifact":{"outputArtifactKey":"model","producerTask":"train-model"}}}},"taskInfo":{"name":"evaluate-model"}},"preprocess":{"cachingOptions":{},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"preprocess"}},"train-model":{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["preprocess"],"inputs":{"artifacts":{"cleaned_text":{"taskOutputArtifact":{"outputArtifactKey":"dataset","producerTask":"preprocess"}}}},"taskInfo":{"name":"train-model"}}}},"inputDefinitions":{"parameters":{"text":{"defaultValue":"Hello world","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -430,11 +430,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-43c0d13fb84d9ca119bdf5124174bd3678698b085ea5c314339233390502736a}}' + value: '{{workflow.parameters.components-5e4fe14151999fbde09ad1cf2ea1ef08988b5557bb7ffb2f26e71b48051a0b24}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-evaluate-model"},"dependentTasks":["train-model"],"inputs":{"artifacts":{"model_text":{"taskOutputArtifact":{"outputArtifactKey":"model","producerTask":"train-model"}}}},"taskInfo":{"name":"evaluate-model"}}' - name: container - value: '{{workflow.parameters.implementations-43c0d13fb84d9ca119bdf5124174bd3678698b085ea5c314339233390502736a}}' + value: '{{workflow.parameters.implementations-5e4fe14151999fbde09ad1cf2ea1ef08988b5557bb7ffb2f26e71b48051a0b24}}' - name: task-name value: evaluate-model - name: parent-dag-id @@ -455,11 +455,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-de426933840df1d5b1966245bfd35532f6a11ccaffa3dbb2311a6b7dbf3ec609}}' + value: '{{workflow.parameters.components-a389d0c5411da97fdf7ec55aea7ffb8d99e46f46230c9b8890c6ee0fed4c920c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"preprocess"}}' - name: container - value: '{{workflow.parameters.implementations-de426933840df1d5b1966245bfd35532f6a11ccaffa3dbb2311a6b7dbf3ec609}}' + value: '{{workflow.parameters.implementations-a389d0c5411da97fdf7ec55aea7ffb8d99e46f46230c9b8890c6ee0fed4c920c}}' - name: task-name value: preprocess - name: parent-dag-id @@ -479,11 +479,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0e704495704924f0f854d4c513ac164f987781625d13c72aabf7e58c7fc1d08c}}' + value: '{{workflow.parameters.components-428abe73fd2527425a6d4eba6c35314506906ee691a0dbe032573b8c130f58d1}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["preprocess"],"inputs":{"artifacts":{"cleaned_text":{"taskOutputArtifact":{"outputArtifactKey":"dataset","producerTask":"preprocess"}}}},"taskInfo":{"name":"train-model"}}' - name: container - value: '{{workflow.parameters.implementations-0e704495704924f0f854d4c513ac164f987781625d13c72aabf7e58c7fc1d08c}}' + value: '{{workflow.parameters.implementations-428abe73fd2527425a6d4eba6c35314506906ee691a0dbe032573b8c130f58d1}}' - name: task-name value: train-model - name: parent-dag-id diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index f8f395155c6..d2b85c86866 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-6cb96369cb79941c813048ffcedc1b2b4dc48a280367899eadca221221c54b92 + - name: components-b5cadd2bcc71c6f13dc450182b4eb137eb62b94bdd5a170d964d6f8dec0260a5 value: '{"executorLabel":"exec-run-train-notebook","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}}}' - - name: implementations-6cb96369cb79941c813048ffcedc1b2b4dc48a280367899eadca221221c54b92 + - name: implementations-b5cadd2bcc71c6f13dc450182b4eb137eb62b94bdd5a170d964d6f8dec0260a5 value: '{"args":["--executor_input","{{$}}","--function_to_execute","run_train_notebook"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -116,7 +116,7 @@ spec: = kfp_run_notebook\n\n\ndef run_train_notebook(text: str):\n # text is not defined in the notebook but text2 is defined\n dsl.run_notebook(text=text)\n\n with open(\"/tmp/kfp_nb_outputs/log.txt\", \"r\", encoding=\"utf-8\") as f:\n log - = f.read()\n\n assert log == text + \" \" + \"default2\"\n\n"],"image":"python:3.9"}' + = f.read()\n\n assert log == text + \" \" + \"default2\"\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"run-train-notebook":{"cachingOptions":{},"componentRef":{"name":"comp-run-train-notebook"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"run-train-notebook"}}}},"inputDefinitions":{"parameters":{"text":{"defaultValue":"hello","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -299,11 +299,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-6cb96369cb79941c813048ffcedc1b2b4dc48a280367899eadca221221c54b92}}' + value: '{{workflow.parameters.components-b5cadd2bcc71c6f13dc450182b4eb137eb62b94bdd5a170d964d6f8dec0260a5}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-run-train-notebook"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"run-train-notebook"}}' - name: container - value: '{{workflow.parameters.implementations-6cb96369cb79941c813048ffcedc1b2b4dc48a280367899eadca221221c54b92}}' + value: '{{workflow.parameters.implementations-b5cadd2bcc71c6f13dc450182b4eb137eb62b94bdd5a170d964d6f8dec0260a5}}' - name: task-name value: run-train-notebook - name: parent-dag-id diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index da12a5448ca..99e72873e95 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4 + - name: components-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97 value: '{"executorLabel":"exec-output-metrics","outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4 + - name: implementations-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97 value: '{"args":["--executor_input","{{$}}","--function_to_execute","output_metrics"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,7 +20,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_metrics(metrics: Output[Metrics]):\n \"\"\"Dummy component that outputs metrics with a random accuracy.\"\"\"\n import random\n result - = random.randint(0, 100)\n metrics.log_metric(''accuracy'', result)\n\n"],"image":"python:3.9"}' + = random.randint(0, 100)\n metrics.log_metric(''accuracy'', result)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"metrics":{"artifactSelectors":[{"outputArtifactKey":"metrics","producerSubtask":"output-metrics"}]}}},"tasks":{"output-metrics":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-metrics"},"taskInfo":{"name":"output-metrics"}}}},"outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}}}}}' entrypoint: entrypoint @@ -203,11 +203,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.components-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-metrics"},"taskInfo":{"name":"output-metrics"}}' - name: container - value: '{{workflow.parameters.implementations-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.implementations-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task-name value: output-metrics - name: parent-dag-id diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index f93d0a4da86..81c0ad8e624 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5 + - name: components-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179 value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5 + - name: implementations-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str) -\u003e str:\n print(message)\n return message\n\n"],"image":"python:3.9"}' + print_op(message: str) -\u003e str:\n print(message)\n return message\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"foo"}}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-root @@ -204,11 +204,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.components-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"foo"}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.implementations-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task-name value: print-op - name: parent-dag-id @@ -380,11 +380,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.components-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"bar"}}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.implementations-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task-name value: print-op-2 - name: parent-dag-id @@ -405,11 +405,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.components-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baz"}}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-d58719d25d427e9e486606802de7140a76dc85a7da1c1753cd07c1433e7a90a5}}' + value: '{{workflow.parameters.implementations-968dd0fe10602246a7925b6c93838c43ef3e0773013d8f5c8cc15c5d7c6e1179}}' - name: task-name value: print-op-3 - name: parent-dag-id diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 482ff51da12..661ea64fa19 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d + - name: components-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"parameters":{"input":{"parameterType":"STRING"}}}}' - - name: implementations-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d + - name: implementations-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - crust_comp(input: str):\n print(''input :'', input)\n\n"],"image":"python:3.9"}' - - name: components-ee0778c93e3b1661b503b59214cb8f679694b2ecc4aa2e53dd25ba5c03529348 + crust_comp(input: str):\n print(''input :'', input)\n\n"],"image":"python:3.11"}' + - name: components-0f16b9e977a5e18f1fdc3e157b0997b82f1f8dc81f77d99e59f429e48800530c value: '{"executorLabel":"exec-core-comp","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-ee0778c93e3b1661b503b59214cb8f679694b2ecc4aa2e53dd25ba5c03529348 + - name: implementations-0f16b9e977a5e18f1fdc3e157b0997b82f1f8dc81f77d99e59f429e48800530c value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - core_comp() -\u003e str:\n return ''foo''\n\n"],"image":"python:3.9"}' + core_comp() -\u003e str:\n return ''foo''\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"core-comp"}}}},"tasks":{"core-comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: components-comp-mantle @@ -218,11 +218,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ee0778c93e3b1661b503b59214cb8f679694b2ecc4aa2e53dd25ba5c03529348}}' + value: '{{workflow.parameters.components-0f16b9e977a5e18f1fdc3e157b0997b82f1f8dc81f77d99e59f429e48800530c}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-ee0778c93e3b1661b503b59214cb8f679694b2ecc4aa2e53dd25ba5c03529348}}' + value: '{{workflow.parameters.implementations-0f16b9e977a5e18f1fdc3e157b0997b82f1f8dc81f77d99e59f429e48800530c}}' - name: task-name value: core-comp - name: parent-dag-id @@ -356,11 +356,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d}}' + value: '{{workflow.parameters.components-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d}}' + value: '{{workflow.parameters.implementations-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index 9a9ccbb8677..ec02a4cf23e 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d + - name: components-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786 value: '{"executorLabel":"exec-crust-comp","inputDefinitions":{"parameters":{"input":{"parameterType":"STRING"}}}}' - - name: implementations-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d + - name: implementations-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786 value: '{"args":["--executor_input","{{$}}","--function_to_execute","crust_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - crust_comp(input: str):\n print(''input :'', input)\n\n"],"image":"python:3.9"}' - - name: components-a7bb13e035d75f1941d4a901687d98eb7be27da4abc34edc24ecdb00bb99d745 + crust_comp(input: str):\n print(''input :'', input)\n\n"],"image":"python:3.11"}' + - name: components-f8f8a77c7beb5b190f9fb182fe0e0dd40b3a7381615f2c349f0d398819f532c3 value: '{"executorLabel":"exec-core-comp","inputDefinitions":{"parameters":{"input":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-a7bb13e035d75f1941d4a901687d98eb7be27da4abc34edc24ecdb00bb99d745 + - name: implementations-f8f8a77c7beb5b190f9fb182fe0e0dd40b3a7381615f2c349f0d398819f532c3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,13 +32,13 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_comp(input: str) -\u003e str:\n print(''input :'', input)\n return - input\n\n"],"image":"python:3.9"}' + input\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--core-comp-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"core-comp"}}}},"tasks":{"core-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"core-comp"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--core-comp-Output":{"parameterType":"STRING"}}}}' - - name: components-991bfbeac7bb7d91fb9d139f44d5e5b5f220227052409ae8a4c3f88b486f84dc + - name: components-c2e197395954c8d0af830daca22f180b3ccec8198dd5795ab265dec857a88316 value: '{"executorLabel":"exec-core-output-comp","inputDefinitions":{"parameters":{"input":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"output_key":{"parameterType":"STRING"}}}}' - - name: implementations-991bfbeac7bb7d91fb9d139f44d5e5b5f220227052409ae8a4c3f88b486f84dc + - name: implementations-c2e197395954c8d0af830daca22f180b3ccec8198dd5795ab265dec857a88316 value: '{"args":["--executor_input","{{$}}","--function_to_execute","core_output_comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -49,7 +49,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef core_output_comp(input: str, output_key: dsl.OutputPath(str)):\n print(''input - :'', input)\n with open(output_key, ''w'') as f:\n f.write(input)\n\n"],"image":"python:3.9"}' + :'', input)\n with open(output_key, ''w'') as f:\n f.write(input)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-3 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--core-output-comp-output_key":{"valueFromParameter":{"outputParameterKey":"output_key","producerSubtask":"core-output-comp"}}}},"tasks":{"core-output-comp":{"cachingOptions":{},"componentRef":{"name":"comp-core-output-comp"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"Got tails!"}}}},"taskInfo":{"name":"core-output-comp"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--core-output-comp-output_key":{"parameterType":"STRING"}}}}' @@ -57,9 +57,9 @@ spec: value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--condition-branches-1-oneof-1":{"valueFromOneof":{"parameterSelectors":[{"outputParameterKey":"pipelinechannel--core-comp-Output","producerSubtask":"condition-2"},{"outputParameterKey":"pipelinechannel--core-output-comp-output_key","producerSubtask":"condition-3"}]}}}},"tasks":{"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads''"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"componentInputParameter":"pipelinechannel--flip-coin-Output"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"!(inputs.parameter_values[''pipelinechannel--flip-coin-Output''] == ''heads'')"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-Output":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--condition-branches-1-oneof-1":{"parameterType":"STRING"}}}}' - - name: components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"executorLabel":"exec-flip-coin","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14 + - name: implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -70,7 +70,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin() -\u003e str:\n import random\n return ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n\n"],"image":"python:3.9"}' + 1) == 0 else ''tails''\n\n"],"image":"python:3.11"}' - name: components-comp-core value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--condition-branches-1-oneof-1","producerSubtask":"condition-branches-1"}}}},"tasks":{"condition-branches-1":{"componentRef":{"name":"comp-condition-branches-1"},"dependentTasks":["flip-coin"],"inputs":{"parameters":{"pipelinechannel--flip-coin-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin"}}}},"taskInfo":{"name":"condition-branches-1"}},"flip-coin":{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - name: components-comp-mantle @@ -257,12 +257,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a7bb13e035d75f1941d4a901687d98eb7be27da4abc34edc24ecdb00bb99d745}}' + value: '{{workflow.parameters.components-f8f8a77c7beb5b190f9fb182fe0e0dd40b3a7381615f2c349f0d398819f532c3}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-comp"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"Got heads!"}}}},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-a7bb13e035d75f1941d4a901687d98eb7be27da4abc34edc24ecdb00bb99d745}}' + value: '{{workflow.parameters.implementations-f8f8a77c7beb5b190f9fb182fe0e0dd40b3a7381615f2c349f0d398819f532c3}}' - name: task-name value: core-comp - name: parent-dag-id @@ -290,12 +290,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-991bfbeac7bb7d91fb9d139f44d5e5b5f220227052409ae8a4c3f88b486f84dc}}' + value: '{{workflow.parameters.components-c2e197395954c8d0af830daca22f180b3ccec8198dd5795ab265dec857a88316}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-core-output-comp"},"inputs":{"parameters":{"input":{"runtimeValue":{"constant":"Got tails!"}}}},"taskInfo":{"name":"core-output-comp"}}' - name: container - value: '{{workflow.parameters.implementations-991bfbeac7bb7d91fb9d139f44d5e5b5f220227052409ae8a4c3f88b486f84dc}}' + value: '{{workflow.parameters.implementations-c2e197395954c8d0af830daca22f180b3ccec8198dd5795ab265dec857a88316}}' - name: task-name value: core-output-comp - name: parent-dag-id @@ -476,11 +476,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.components-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin"},"taskInfo":{"name":"flip-coin"}}' - name: container - value: '{{workflow.parameters.implementations-63c60d1934695443fbf91b71abd177ac69246c9328099025670d1d75e5f45f14}}' + value: '{{workflow.parameters.implementations-b7bc282093d4c682a88eee9264eed07197d2e4dc5276c37ed07b353ff7e961aa}}' - name: task-name value: flip-coin - name: parent-dag-id @@ -537,11 +537,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d}}' + value: '{{workflow.parameters.components-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - name: container - value: '{{workflow.parameters.implementations-2f694fdbcf4a9d3446e12aa9fca457c7e2882290abceb728d2f702c4d450df8d}}' + value: '{{workflow.parameters.implementations-e25bc8f7f685ab080a6b49947d416eb8ca7a6937fa27801a940c2bdca54fc786}}' - name: task-name value: crust-comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index 0e9aa4bb3ab..05f09dedbc2 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-934af9f49289ce10e90d26c3cf160534c673f4da28e91bc67331b4a9ae08a463 + - name: components-5983895df78feaa3fc4a33110d0aed33ca16915ec107be6f49c07e9c4c738d16 value: '{"executorLabel":"exec-add-two-numbers","inputDefinitions":{"parameters":{"x":{"parameterType":"LIST"},"y":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-934af9f49289ce10e90d26c3cf160534c673f4da28e91bc67331b4a9ae08a463 + - name: implementations-5983895df78feaa3fc4a33110d0aed33ca16915ec107be6f49c07e9c4c738d16 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_two_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef add_two_numbers(x: List[int], y: List[int]) -\u003e int:\n return sum(x) - + sum(y)\n\n"],"image":"python:3.9"}' - - name: components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + + sum(y)\n\n"],"image":"python:3.11"}' + - name: components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,12 +32,12 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.9"}' + double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-4 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--double-2-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"double-2"}}}},"tasks":{"double-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-3":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--double-2-Output":{"parameterType":"LIST"}}}}' - - name: components-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964 + - name: components-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3 value: '{"executorLabel":"exec-simple-add","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964 + - name: implementations-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","simple_add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -47,13 +47,13 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - simple_add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.9"}' + simple_add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--double-2-Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerSubtask":"for-loop-4"}},"pipelinechannel--double-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"double"}}}},"tasks":{"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[4, 5, 6]"}},"taskInfo":{"name":"for-loop-4"}},"simple-add":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-simple-add"},"dependentTasks":["for-loop-4"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerTask":"for-loop-4"}}}},"taskInfo":{"name":"simple-add"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--double-2-Output":{"parameterType":"LIST"},"pipelinechannel--double-Output":{"parameterType":"LIST"}}}}' - - name: components-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce + - name: components-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4 value: '{"executorLabel":"exec-nested-add-2","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce + - name: implementations-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4 value: '{"args":["--executor_input","{{$}}","--function_to_execute","nested_add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -64,7 +64,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef nested_add(nums: List[List[int]]) -\u003e int:\n import itertools\n return - sum(itertools.chain(*nums))\n\n"],"image":"python:3.9"}' + sum(itertools.chain(*nums))\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-6 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--nested-add-2-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"nested-add-2"}},"pipelinechannel--simple-add-2-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"simple-add-2"}}}},"tasks":{"nested-add-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-add-2"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-2-Output"}}},"taskInfo":{"name":"nested-add-2"}},"simple-add-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-simple-add-2"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-Output"}}},"taskInfo":{"name":"simple-add-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--for-loop-2-pipelinechannel--double-2-Output":{"parameterType":"LIST"},"pipelinechannel--for-loop-2-pipelinechannel--double-Output":{"parameterType":"LIST"},"pipelinechannel--loop-item-param-5":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--nested-add-2-Output":{"parameterType":"LIST"},"pipelinechannel--simple-add-2-Output":{"parameterType":"LIST"}}}}' - name: components-root @@ -251,11 +251,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double-2"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"double-2"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double-2 - name: parent-dag-id @@ -421,11 +421,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double - name: parent-dag-id @@ -451,11 +451,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964}}' + value: '{{workflow.parameters.components-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-simple-add"},"dependentTasks":["for-loop-4"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerTask":"for-loop-4"}}}},"taskInfo":{"name":"simple-add"}}' - name: container - value: '{{workflow.parameters.implementations-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964}}' + value: '{{workflow.parameters.implementations-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3}}' - name: task-name value: simple-add - name: parent-dag-id @@ -484,11 +484,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce}}' + value: '{{workflow.parameters.components-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-add-2"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-2-Output"}}},"taskInfo":{"name":"nested-add-2"}}' - name: container - value: '{{workflow.parameters.implementations-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce}}' + value: '{{workflow.parameters.implementations-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4}}' - name: task-name value: nested-add-2 - name: parent-dag-id @@ -508,11 +508,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964}}' + value: '{{workflow.parameters.components-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-simple-add-2"},"inputs":{"parameters":{"nums":{"componentInputParameter":"pipelinechannel--for-loop-2-pipelinechannel--double-Output"}}},"taskInfo":{"name":"simple-add-2"}}' - name: container - value: '{{workflow.parameters.implementations-843abeeb4ef17bcaea1bd40a0b5348966bd310914d8b2fadc13215c136bf3964}}' + value: '{{workflow.parameters.implementations-8f8f327236d0dee95a8f9e3066a3489084583a3ea55cfbc385dc232fa4205cc3}}' - name: task-name value: simple-add-2 - name: parent-dag-id @@ -662,11 +662,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-934af9f49289ce10e90d26c3cf160534c673f4da28e91bc67331b4a9ae08a463}}' + value: '{{workflow.parameters.components-5983895df78feaa3fc4a33110d0aed33ca16915ec107be6f49c07e9c4c738d16}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-two-numbers"},"dependentTasks":["for-loop-6"],"inputs":{"parameters":{"x":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--simple-add-2-Output","producerTask":"for-loop-6"}},"y":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--nested-add-2-Output","producerTask":"for-loop-6"}}}},"taskInfo":{"name":"add-two-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-934af9f49289ce10e90d26c3cf160534c673f4da28e91bc67331b4a9ae08a463}}' + value: '{{workflow.parameters.implementations-5983895df78feaa3fc4a33110d0aed33ca16915ec107be6f49c07e9c4c738d16}}' - name: task-name value: add-two-numbers - name: parent-dag-id @@ -700,11 +700,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce}}' + value: '{{workflow.parameters.components-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-add"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"nested-add"}}' - name: container - value: '{{workflow.parameters.implementations-305f913c03b5666bf1a6fdd8cb4d3bfa17dde66bcd297bfb8827c23f3ed92dce}}' + value: '{{workflow.parameters.implementations-85ab7701a46287a3152375690acb08e729df2da42b3ddf49938ef1f9be7e8fb4}}' - name: task-name value: nested-add - name: parent-dag-id diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index b5b4d5ce6fc..d3c4a81c22b 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428 + - name: components-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3 value: '{"executorLabel":"exec-add","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428 + - name: implementations-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,15 +18,15 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.9"}' + add(nums: List[int]) -\u003e int:\n return sum(nums)\n\n"],"image":"python:3.11"}' - name: components-a84a92e43cc5061b9721809172326190f017aa27a15f7b894c7be7cb01d2e7e4 value: '{"executorLabel":"exec-add-container","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"sum":{"parameterType":"NUMBER_INTEGER"}}}}' - name: implementations-a84a92e43cc5061b9721809172326190f017aa27a15f7b894c7be7cb01d2e7e4 value: '{"args":["\n set -ex\n mkdir -p $(dirname {{$.outputs.parameters[''sum''].output_file}})\n echo {{$.inputs.parameters[''nums'']}} | jq ''add'' \u003e {{$.outputs.parameters[''sum''].output_file}}\n "],"command":["sh","-c"],"image":"stedolan/jq"}' - - name: components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -36,7 +36,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.9"}' + double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--double-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"double"}}}},"tasks":{"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--double-Output":{"parameterType":"LIST"}}}}' - name: components-root @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double - name: parent-dag-id @@ -392,11 +392,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428}}' + value: '{{workflow.parameters.components-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-0a346d821b1e474f2aea8ab24806f75b1afd4d6f7764be16c00f645cc08d5428}}' + value: '{{workflow.parameters.implementations-db8097682cf93da6b46f7ade69b360e7b202276c350b83e4bbf5da6625c7f8c3}}' - name: task-name value: add - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index 352b954c765..9a8f53e826a 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + - name: components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + - name: implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,13 +18,13 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"notify task failure."}}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--get-run-state-Output":{"parameterType":"STRING"}}}}' - - name: components-2784bcbcbd64b218e0f38de5fe3ce3aecd9c19875e77fb40612978303a5983b8 + - name: components-c7345e051c66d30bc0fcd4558f91a01615e2e208dba34ad8d58d58c265056aa8 value: '{"executorLabel":"exec-get-run-state","inputDefinitions":{"parameters":{"status":{"parameterType":"STRUCT"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-2784bcbcbd64b218e0f38de5fe3ce3aecd9c19875e77fb40612978303a5983b8 + - name: implementations-c7345e051c66d30bc0fcd4558f91a01615e2e208dba34ad8d58d58c265056aa8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","get_run_state"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -35,13 +35,13 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef get_run_state(status: dict) -\u003e str:\n print(''Pipeline status: '', - status)\n return status[''state'']\n\n"],"image":"python:3.9"}' + status)\n return status[''state'']\n\n"],"image":"python:3.11"}' - name: components-comp-conditional-notification value: '{"dag":{"tasks":{"condition-1":{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["get-run-state"],"inputs":{"parameters":{"pipelinechannel--get-run-state-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"get-run-state"}}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--get-run-state-Output''] == ''FAILED''"}},"get-run-state":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-get-run-state"},"inputs":{"parameters":{"status":{"componentInputParameter":"status"}}},"taskInfo":{"name":"get-run-state"}}}},"inputDefinitions":{"parameters":{"status":{"isOptional":true,"parameterType":"TASK_FINAL_STATUS"}}}}' - - name: components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -51,7 +51,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.9"}' + fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.11"}' - name: components-comp-exit-handler-1 value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--message":{"parameterType":"STRING"}}}}' @@ -238,12 +238,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"notify task failure."}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op - name: parent-dag-id @@ -372,11 +372,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2784bcbcbd64b218e0f38de5fe3ce3aecd9c19875e77fb40612978303a5983b8}}' + value: '{{workflow.parameters.components-c7345e051c66d30bc0fcd4558f91a01615e2e208dba34ad8d58d58c265056aa8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-get-run-state"},"inputs":{"parameters":{"status":{"componentInputParameter":"status"}}},"taskInfo":{"name":"get-run-state"}}' - name: container - value: '{{workflow.parameters.implementations-2784bcbcbd64b218e0f38de5fe3ce3aecd9c19875e77fb40612978303a5983b8}}' + value: '{{workflow.parameters.implementations-c7345e051c66d30bc0fcd4558f91a01615e2e208dba34ad8d58d58c265056aa8}}' - name: task-name value: get-run-state - name: parent-dag-id @@ -404,12 +404,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task-name value: fail-op - name: parent-dag-id @@ -429,11 +429,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 10b94b6833f..9102347c308 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"executorLabel":"exec-print-op1-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.9"}' + print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.11"}' - name: components-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348 value: '{"executorLabel":"exec-print-op2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - name: implementations-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348 @@ -207,11 +207,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id @@ -362,11 +362,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"Hello"}}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 5441f3cbf7e..ab28fb406c9 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -14,9 +14,9 @@ spec: value: '{"dag":{"tasks":{"print-op2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"world"}}}},"taskInfo":{"name":"print-op2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--print-op1-Output":{"parameterType":"STRING"}}}}' - name: components-comp-condition-2 value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"Bye!"}}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--print-op1-Output":{"parameterType":"STRING"}}}}' - - name: components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"executorLabel":"exec-print-op1-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -26,7 +26,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.9"}' + print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.11"}' - name: components-comp-inner-pipeline value: '{"dag":{"tasks":{"condition-1":{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] == ''Hello''"}},"condition-2":{"componentRef":{"name":"comp-condition-2"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] @@ -405,11 +405,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id @@ -533,11 +533,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index ef714eca51a..5d9d8a8929b 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"executorLabel":"exec-print-op1-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2 + - name: implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.9"}' + print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.11"}' - name: components-3f2ba2471e2638f8b27a01f4563316e6e3e8c02bae402875e95b5b86e8de1ddc value: '{"executorLabel":"exec-print-op2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: implementations-3f2ba2471e2638f8b27a01f4563316e6e3e8c02bae402875e95b5b86e8de1ddc @@ -28,9 +28,9 @@ spec: value: '{"dag":{"outputs":{"artifacts":{"data":{"artifactSelectors":[{"outputArtifactKey":"data","producerSubtask":"print-op2"}]}},"parameters":{"msg":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"print-op1"}}}},"tasks":{"print-op1":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}},"print-op2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"print-op2"}}}},"inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}},"parameters":{"msg":{"parameterType":"STRING"}}}}' - name: components-comp-pipeline-in-pipeline value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"data","producerSubtask":"inner-pipeline"}]}}},"tasks":{"inner-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-inner-pipeline"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"world"}}}},"taskInfo":{"name":"inner-pipeline"}},"print-op1":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}}},"inputDefinitions":{"parameters":{"msg":{"defaultValue":"Hello","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: components-f310a0bf82b254e5d513a6030f358a14c0199a92aa17fb0534416a5f13d29b31 + - name: components-d86fa51b3af1d2cdca561c7f7cdf7534896739e113579b081fa96e6c40e113c1 value: '{"executorLabel":"exec-print-op1-3","inputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-f310a0bf82b254e5d513a6030f358a14c0199a92aa17fb0534416a5f13d29b31 + - name: implementations-d86fa51b3af1d2cdca561c7f7cdf7534896739e113579b081fa96e6c40e113c1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -40,7 +40,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op1(data: Input[Artifact]):\n with open(data.path, ''r'') as f:\n print(f.read())\n\n"],"image":"python:3.9"}' + print_op1(data: Input[Artifact]):\n with open(data.path, ''r'') as f:\n print(f.read())\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"pipeline-in-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-pipeline-in-pipeline"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"Hello"}}}},"taskInfo":{"name":"pipeline-in-pipeline"}},"print-op1":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-3"},"dependentTasks":["pipeline-in-pipeline"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"pipeline-in-pipeline"}}}},"taskInfo":{"name":"print-op1"}}}}}' entrypoint: entrypoint @@ -223,11 +223,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id @@ -378,11 +378,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.components-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-0136b0f27f214b43cb8217d67d795da9568e61f9dbcd5fbda26cb0f0254cd0d2}}' + value: '{{workflow.parameters.implementations-c4774190f368dac0d3d13c5e2481b325cb0256069205b909a0a2385615a61216}}' - name: task-name value: print-op1 - name: parent-dag-id @@ -431,11 +431,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f310a0bf82b254e5d513a6030f358a14c0199a92aa17fb0534416a5f13d29b31}}' + value: '{{workflow.parameters.components-d86fa51b3af1d2cdca561c7f7cdf7534896739e113579b081fa96e6c40e113c1}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-3"},"dependentTasks":["pipeline-in-pipeline"],"inputs":{"artifacts":{"data":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"pipeline-in-pipeline"}}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-f310a0bf82b254e5d513a6030f358a14c0199a92aa17fb0534416a5f13d29b31}}' + value: '{{workflow.parameters.implementations-d86fa51b3af1d2cdca561c7f7cdf7534896739e113579b081fa96e6c40e113c1}}' - name: task-name value: print-op1 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index 7f1619bf3f9..b282a4691cc 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61 + - name: components-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3 value: '{"executorLabel":"exec-add","inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61 + - name: implementations-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef add(nums: List[List[int]]) -\u003e int:\n import itertools\n return - sum(itertools.chain(*nums))\n\n"],"image":"python:3.9"}' - - name: components-820c6b84550cd793dda8df1e7a9d6e31a46ac14583a0ed36ad9bf2844d5f25b4 + sum(itertools.chain(*nums))\n\n"],"image":"python:3.11"}' + - name: components-5c5f855eb8bb4260342c9f40e736b35407a28d55966e30615d053e6a28f07fed value: '{"executorLabel":"exec-echo-and-return","inputDefinitions":{"parameters":{"string":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-820c6b84550cd793dda8df1e7a9d6e31a46ac14583a0ed36ad9bf2844d5f25b4 + - name: implementations-5c5f855eb8bb4260342c9f40e736b35407a28d55966e30615d053e6a28f07fed value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_and_return"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,15 +32,15 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - echo_and_return(string: str) -\u003e str:\n print(string)\n return string\n\n"],"image":"python:3.9"}' + echo_and_return(string: str) -\u003e str:\n print(string)\n return string\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--echo-and-return-Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"echo-and-return"}}}},"tasks":{"echo-and-return":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-and-return"},"inputs":{"parameters":{"string":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"echo-and-return"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"pipelinechannel--echo-and-return-Output":{"parameterType":"LIST"}}}}' - name: components-comp-add-pipeline value: '{"dag":{"outputs":{"parameters":{"out1":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"add"}},"out2":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--echo-and-return-Output","producerSubtask":"for-loop-2"}}}},"tasks":{"add":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"nums":{"componentInputParameter":"nums"}}},"taskInfo":{"name":"add"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"m\", \"a\", \"t\", \"h\"]"}},"taskInfo":{"name":"for-loop-2"}}}},"inputDefinitions":{"parameters":{"nums":{"parameterType":"LIST"}}},"outputDefinitions":{"parameters":{"out1":{"parameterType":"NUMBER_INTEGER"},"out2":{"parameterType":"LIST"}}}}' - - name: components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"executorLabel":"exec-double","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44 + - name: implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","double"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -50,7 +50,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.9"}' + double(num: int) -\u003e int:\n return 2 * num\n\n"],"image":"python:3.11"}' - name: components-comp-double-pipeline value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"double"}}}},"tasks":{"double":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"num"}}},"taskInfo":{"name":"double"}}}},"inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-comp-for-loop-4 @@ -58,9 +58,9 @@ spec: - name: components-comp-for-loop-2 value: '{"dag":{"outputs":{"parameters":{"pipelinechannel--double-pipeline-Output":{"valueFromParameter":{"outputParameterKey":"pipelinechannel--double-pipeline-Output","producerSubtask":"for-loop-4"}}}},"tasks":{"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"pipelinechannel--double-pipeline-Output":{"parameterType":"LIST"}}}}' - - name: components-4a0d5e75bec2b5be6dfc3d4ef02083b53d09e3e48bbb8b4346d4e919bdc999a2 + - name: components-62336025dcddfbc82e6c78051d9dbd9253225e4f399ded7fb486aabb7c96645a value: '{"executorLabel":"exec-join-and-print","inputDefinitions":{"parameters":{"strings":{"parameterType":"LIST"}}}}' - - name: implementations-4a0d5e75bec2b5be6dfc3d4ef02083b53d09e3e48bbb8b4346d4e919bdc999a2 + - name: implementations-62336025dcddfbc82e6c78051d9dbd9253225e4f399ded7fb486aabb7c96645a value: '{"args":["--executor_input","{{$}}","--function_to_execute","join_and_print"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -70,7 +70,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - join_and_print(strings: List[str]):\n print(''''.join(strings))\n\n"],"image":"python:3.9"}' + join_and_print(strings: List[str]):\n print(''''.join(strings))\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"out1","producerSubtask":"add-pipeline"}}}},"tasks":{"add-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-pipeline"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"nums":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-pipeline-Output","producerTask":"for-loop-2"}}}},"taskInfo":{"name":"add-pipeline"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}},"join-and-print":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-join-and-print"},"dependentTasks":["add-pipeline"],"inputs":{"parameters":{"strings":{"taskOutputParameter":{"outputParameterKey":"out2","producerTask":"add-pipeline"}}}},"taskInfo":{"name":"join-and-print"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' @@ -254,11 +254,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-820c6b84550cd793dda8df1e7a9d6e31a46ac14583a0ed36ad9bf2844d5f25b4}}' + value: '{{workflow.parameters.components-5c5f855eb8bb4260342c9f40e736b35407a28d55966e30615d053e6a28f07fed}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-and-return"},"inputs":{"parameters":{"string":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"echo-and-return"}}' - name: container - value: '{{workflow.parameters.implementations-820c6b84550cd793dda8df1e7a9d6e31a46ac14583a0ed36ad9bf2844d5f25b4}}' + value: '{{workflow.parameters.implementations-5c5f855eb8bb4260342c9f40e736b35407a28d55966e30615d053e6a28f07fed}}' - name: task-name value: echo-and-return - name: parent-dag-id @@ -424,11 +424,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61}}' + value: '{{workflow.parameters.components-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"nums":{"componentInputParameter":"nums"}}},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-278f161477820f25fd603ee488b03203504f1efb0101e60e3fae0c7f63cd9a61}}' + value: '{{workflow.parameters.implementations-1be5f8f8f37d8780d2254662338636e64bb02c2cd31f4099b84c292a06f04cd3}}' - name: task-name value: add - name: parent-dag-id @@ -462,11 +462,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.components-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-double"},"inputs":{"parameters":{"num":{"componentInputParameter":"num"}}},"taskInfo":{"name":"double"}}' - name: container - value: '{{workflow.parameters.implementations-ce925a4ef98c92ae630145dae1155afc9d381ebef3b3f22030ab4257c4910f44}}' + value: '{{workflow.parameters.implementations-f79f58ed4b630aadad448078027429b1d8053e45ed52bed5961317a8e3ebf5e3}}' - name: task-name value: double - name: parent-dag-id @@ -687,11 +687,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4a0d5e75bec2b5be6dfc3d4ef02083b53d09e3e48bbb8b4346d4e919bdc999a2}}' + value: '{{workflow.parameters.components-62336025dcddfbc82e6c78051d9dbd9253225e4f399ded7fb486aabb7c96645a}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-join-and-print"},"dependentTasks":["add-pipeline"],"inputs":{"parameters":{"strings":{"taskOutputParameter":{"outputParameterKey":"out2","producerTask":"add-pipeline"}}}},"taskInfo":{"name":"join-and-print"}}' - name: container - value: '{{workflow.parameters.implementations-4a0d5e75bec2b5be6dfc3d4ef02083b53d09e3e48bbb8b4346d4e919bdc999a2}}' + value: '{{workflow.parameters.implementations-62336025dcddfbc82e6c78051d9dbd9253225e4f399ded7fb486aabb7c96645a}}' - name: task-name value: join-and-print - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 7566416be01..d0c739acf33 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-05e0c7c24c705bcaac86e732ffd5d85e67d9a184ae4703d789ccfe1715a452f2 + - name: components-9881acc01330f69427ffe30acddda484b63239f267d7036beb6070353fda316b value: '{"executorLabel":"exec-download-dataset-and-upload-as-artifact","inputDefinitions":{"parameters":{"dataset_name":{"parameterType":"STRING"},"dataset_repo":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-05e0c7c24c705bcaac86e732ffd5d85e67d9a184ae4703d789ccfe1715a452f2 + - name: implementations-9881acc01330f69427ffe30acddda484b63239f267d7036beb6070353fda316b value: '{"args":["--executor_input","{{$}}","--function_to_execute","download_dataset_and_upload_as_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -23,10 +23,10 @@ spec: output_dataset: Output[Dataset]):\n from datasets import load_dataset\n # Load data set from hugging face\n ds = load_dataset(dataset_repo, dataset_name)\n print(\"Downloaded Hugging Face data\")\n print(f\"Now saving to {output_dataset.path}\")\n ds.save_to_disk(output_dataset.path)\n print(f\"Saved - to {output_dataset.path}\")\n\n"],"image":"python:3.9"}' - - name: components-d0f4b4d95b964a5d78a841aa20e3d529d81189e4c90f9b39a31332bb8c8d437c + to {output_dataset.path}\")\n\n"],"image":"python:3.11"}' + - name: components-c020e8bb66a4c390268500295dd49b3f2dcf34bea30556dd91446c1696154c08 value: '{"executorLabel":"exec-print-dataset-info","inputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-d0f4b4d95b964a5d78a841aa20e3d529d81189e4c90f9b39a31332bb8c8d437c + - name: implementations-c020e8bb66a4c390268500295dd49b3f2dcf34bea30556dd91446c1696154c08 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_dataset_info"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -39,7 +39,7 @@ spec: print_dataset_info(dataset: Dataset):\n print(''Information about the artifact'')\n print(''Name:'', dataset.name)\n print(''URI:'', dataset.uri)\n assert \"download-dataset-and-upload-as-artifact\" in dataset.uri, \"The URI of the downloaded artifact does not match the expected - function''s name that generated it\"\n\n"],"image":"python:3.9"}' + function''s name that generated it\"\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"download-dataset-and-upload-as-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-download-dataset-and-upload-as-artifact"},"inputs":{"parameters":{"dataset_name":{"componentInputParameter":"dataset_name"},"dataset_repo":{"componentInputParameter":"dataset_repo"}}},"taskInfo":{"name":"download-dataset-and-upload-as-artifact"}},"print-dataset-info":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-dataset-info"},"dependentTasks":["download-dataset-and-upload-as-artifact"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset","producerTask":"download-dataset-and-upload-as-artifact"}}}},"taskInfo":{"name":"print-dataset-info"}}}},"inputDefinitions":{"parameters":{"dataset_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"dataset_repo":{"defaultValue":"google/frames-benchmark","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-05e0c7c24c705bcaac86e732ffd5d85e67d9a184ae4703d789ccfe1715a452f2}}' + value: '{{workflow.parameters.components-9881acc01330f69427ffe30acddda484b63239f267d7036beb6070353fda316b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-download-dataset-and-upload-as-artifact"},"inputs":{"parameters":{"dataset_name":{"componentInputParameter":"dataset_name"},"dataset_repo":{"componentInputParameter":"dataset_repo"}}},"taskInfo":{"name":"download-dataset-and-upload-as-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-05e0c7c24c705bcaac86e732ffd5d85e67d9a184ae4703d789ccfe1715a452f2}}' + value: '{{workflow.parameters.implementations-9881acc01330f69427ffe30acddda484b63239f267d7036beb6070353fda316b}}' - name: task-name value: download-dataset-and-upload-as-artifact - name: parent-dag-id @@ -246,11 +246,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-d0f4b4d95b964a5d78a841aa20e3d529d81189e4c90f9b39a31332bb8c8d437c}}' + value: '{{workflow.parameters.components-c020e8bb66a4c390268500295dd49b3f2dcf34bea30556dd91446c1696154c08}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-dataset-info"},"dependentTasks":["download-dataset-and-upload-as-artifact"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"output_dataset","producerTask":"download-dataset-and-upload-as-artifact"}}}},"taskInfo":{"name":"print-dataset-info"}}' - name: container - value: '{{workflow.parameters.implementations-d0f4b4d95b964a5d78a841aa20e3d529d81189e4c90f9b39a31332bb8c8d437c}}' + value: '{{workflow.parameters.implementations-c020e8bb66a4c390268500295dd49b3f2dcf34bea30556dd91446c1696154c08}}' - name: task-name value: print-dataset-info - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index 067cccafd28..7fc3d76bcd4 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"executorLabel":"exec-flip-coin-op-2","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -\u003e str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.9"}' - - name: components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823 + 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.11"}' + - name: components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5 value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823 + - name: implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,7 +33,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.9"}' + print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"flip-coin-op-2":{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin-op-2"},"taskInfo":{"name":"flip-coin-op-2"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["flip-coin-op-2"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-2"}}}},"taskInfo":{"name":"print-op-2"}},"print-op-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--text"}}},"taskInfo":{"name":"print-op-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"},"pipelinechannel--text":{"parameterType":"STRING"}}}}' - name: components-root @@ -220,11 +220,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin-op-2"},"taskInfo":{"name":"flip-coin-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op-2 - name: parent-dag-id @@ -244,11 +244,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["flip-coin-op-2"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-2"}}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op-2 - name: parent-dag-id @@ -269,11 +269,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--text"}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op-3 - name: parent-dag-id @@ -402,11 +402,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-flip-coin-op"},"taskInfo":{"name":"flip-coin-op"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op - name: parent-dag-id @@ -426,11 +426,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["flip-coin-op"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op"}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 4666aaaa2e1..2d8155c37b7 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c + - name: components-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8 value: '{"executorLabel":"exec-accelerator-count","outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c + - name: implementations-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_count"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_count() -\u003e int:\n return 1\n\n"],"image":"python:3.9"}' - - name: components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + accelerator_count() -\u003e int:\n return 1\n\n"],"image":"python:3.11"}' + - name: components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"executorLabel":"exec-accelerator-type","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + - name: implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_type"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.9"}' + accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.11"}' - name: components-5c3db471413bc5e6b0f2f0865964176bec80d4d3e1dca8ac3332029d1bd0e525 value: '{"executorLabel":"exec-custom-training-job","inputDefinitions":{"parameters":{"base_output_directory":{"defaultValue":"","description":"The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. @@ -107,9 +107,9 @@ spec: \"{{$.inputs.parameters[''encryption_spec_key_name'']}}\"}}","--project","{{$.inputs.parameters[''project'']}}","--location","{{$.inputs.parameters[''location'']}}","--gcp_resources","{{$.outputs.parameters[''gcp_resources''].output_file}}"],"command":["python3","-u","-m","google_cloud_pipeline_components.container.v1.custom_job.launcher"],"image":"gcr.io/ml-pipeline/google-cloud-pipeline-components:2.21.0"}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"custom-training-job":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-custom-training-job"},"inputs":{"parameters":{"display_name":{"runtimeValue":{"constant":"add-numbers"}},"encryption_spec_key_name":{"componentInputParameter":"pipelinechannel--encryption_spec_key_name"},"location":{"componentInputParameter":"pipelinechannel--location"},"pipelinechannel--accelerator-count-Output":{"componentInputParameter":"pipelinechannel--accelerator-count-Output"},"pipelinechannel--accelerator-type-Output":{"componentInputParameter":"pipelinechannel--accelerator-type-Output"},"pipelinechannel--machine-type-Output":{"componentInputParameter":"pipelinechannel--machine-type-Output"},"project":{"componentInputParameter":"pipelinechannel--project"},"worker_pool_specs":{"runtimeValue":{"constant":[{"container_spec":{"args":["foo"],"command":["echo"],"image_uri":"gcr.io/ml-pipeline/google-cloud-pipeline-components:2.5.0"},"machine_spec":{"accelerator_count":"{{$.inputs.parameters[''pipelinechannel--accelerator-count-Output'']}}","accelerator_type":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}","machine_type":"{{$.inputs.parameters[''pipelinechannel--machine-type-Output'']}}"},"replica_count":1}]}}}},"taskInfo":{"name":"custom-training-job"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--accelerator-count-Output":{"parameterType":"NUMBER_INTEGER"},"pipelinechannel--accelerator-type-Output":{"parameterType":"STRING"},"pipelinechannel--encryption_spec_key_name":{"parameterType":"STRING"},"pipelinechannel--flip-biased-coin-op-Output":{"parameterType":"STRING"},"pipelinechannel--location":{"parameterType":"STRING"},"pipelinechannel--machine-type-Output":{"parameterType":"STRING"},"pipelinechannel--project":{"parameterType":"STRING"}}}}' - - name: components-51b3f8664f2ff23be29076a65342b8eb091ec17bd73e4295c973baea3dd5f073 + - name: components-0c259210b1fb84dd1577b42a94ae79f0619227d7d05e5f54ffe040c596a5439c value: '{"executorLabel":"exec-flip-biased-coin-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-51b3f8664f2ff23be29076a65342b8eb091ec17bd73e4295c973baea3dd5f073 + - name: implementations-0c259210b1fb84dd1577b42a94ae79f0619227d7d05e5f54ffe040c596a5439c value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_biased_coin_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -120,10 +120,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_biased_coin_op() -\u003e str:\n \"\"\"Flip a coin and output heads.\"\"\"\n return - ''heads''\n\n"],"image":"python:3.9"}' - - name: components-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972 + ''heads''\n\n"],"image":"python:3.11"}' + - name: components-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715 value: '{"executorLabel":"exec-machine-type","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972 + - name: implementations-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715 value: '{"args":["--executor_input","{{$}}","--function_to_execute","machine_type"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -133,7 +133,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - machine_type() -\u003e str:\n return ''n1-standard-4''\n\n"],"image":"python:3.9"}' + machine_type() -\u003e str:\n return ''n1-standard-4''\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"accelerator-count":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-count"},"taskInfo":{"name":"accelerator-count"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"condition-1":{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["accelerator-count","accelerator-type","flip-biased-coin-op","machine-type"],"inputs":{"parameters":{"pipelinechannel--accelerator-count-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-count"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--encryption_spec_key_name":{"componentInputParameter":"encryption_spec_key_name"},"pipelinechannel--flip-biased-coin-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-biased-coin-op"}},"pipelinechannel--location":{"componentInputParameter":"location"},"pipelinechannel--machine-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"machine-type"}},"pipelinechannel--project":{"componentInputParameter":"project"}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--flip-biased-coin-op-Output''] == ''heads''"}},"flip-biased-coin-op":{"cachingOptions":{},"componentRef":{"name":"comp-flip-biased-coin-op"},"taskInfo":{"name":"flip-biased-coin-op"}},"machine-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-machine-type"},"taskInfo":{"name":"machine-type"}}}},"inputDefinitions":{"parameters":{"encryption_spec_key_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"location":{"parameterType":"STRING"},"project":{"parameterType":"STRING"}}}}' @@ -426,11 +426,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c}}' + value: '{{workflow.parameters.components-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-count"},"taskInfo":{"name":"accelerator-count"}}' - name: container - value: '{{workflow.parameters.implementations-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c}}' + value: '{{workflow.parameters.implementations-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8}}' - name: task-name value: accelerator-count - name: parent-dag-id @@ -450,11 +450,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}}' - name: container - value: '{{workflow.parameters.implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task-name value: accelerator-type - name: parent-dag-id @@ -499,11 +499,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-51b3f8664f2ff23be29076a65342b8eb091ec17bd73e4295c973baea3dd5f073}}' + value: '{{workflow.parameters.components-0c259210b1fb84dd1577b42a94ae79f0619227d7d05e5f54ffe040c596a5439c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-flip-biased-coin-op"},"taskInfo":{"name":"flip-biased-coin-op"}}' - name: container - value: '{{workflow.parameters.implementations-51b3f8664f2ff23be29076a65342b8eb091ec17bd73e4295c973baea3dd5f073}}' + value: '{{workflow.parameters.implementations-0c259210b1fb84dd1577b42a94ae79f0619227d7d05e5f54ffe040c596a5439c}}' - name: task-name value: flip-biased-coin-op - name: parent-dag-id @@ -523,11 +523,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972}}' + value: '{{workflow.parameters.components-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-machine-type"},"taskInfo":{"name":"machine-type"}}' - name: container - value: '{{workflow.parameters.implementations-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972}}' + value: '{{workflow.parameters.implementations-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715}}' - name: task-name value: machine-type - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 69a24a751ba..5cf44ee7829 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -14,9 +14,9 @@ spec: value: '{"executorLabel":"exec-importer-2","inputDefinitions":{"parameters":{"metadata":{"parameterType":"STRING"},"metadata-2":{"parameterType":"STRING"},"metadata-3":{"parameterType":"NUMBER_INTEGER"},"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: implementations-comp-importer-2 value: '{"artifactUri":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"},"metadata":{"containerSpec":{"imageUri":"us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-5:latest"},"list-of-data":["{{$.inputs.parameters[''metadata'']}}","{{$.inputs.parameters[''metadata-2'']}}","{{$.inputs.parameters[''metadata-3'']}}"],"name":"prefix-{{$.inputs.parameters[''metadata'']}}","{{$.inputs.parameters[''metadata'']}}":"{{$.inputs.parameters[''metadata'']}}","{{$.inputs.parameters[''metadata-2'']}}":"us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-5:latest"},"typeSchema":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}' - - name: components-958372f93b012956585660b34fdee7e4ee56bc8305edc68a225e8d9fd0e274f2 + - name: components-1f5c6ec51212707117cf959e66eed87794baf9ac92c3f57c43a8e3ce06397f03 value: '{"executorLabel":"exec-make-name","inputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-958372f93b012956585660b34fdee7e4ee56bc8305edc68a225e8d9fd0e274f2 + - name: implementations-1f5c6ec51212707117cf959e66eed87794baf9ac92c3f57c43a8e3ce06397f03 value: '{"args":["--executor_input","{{$}}","--function_to_execute","make_name"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -26,7 +26,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - make_name(name: str) -\u003e str:\n return name\n\n"],"image":"python:3.9"}' + make_name(name: str) -\u003e str:\n return name\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"metadata":{"componentInputParameter":"name"},"metadata-2":{"componentInputParameter":"pipeline_input_image_uri"},"uri":{"componentInputParameter":"pipeline_input_artifact_uri"}}},"taskInfo":{"name":"importer"}},"importer-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer-2"},"dependentTasks":["make-name"],"inputs":{"parameters":{"metadata":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"make-name"}},"metadata-2":{"componentInputParameter":"name"},"metadata-3":{"componentInputParameter":"int_input"},"uri":{"runtimeValue":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"}}}},"taskInfo":{"name":"importer-2"}},"make-name":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-name"},"inputs":{"parameters":{"name":{"runtimeValue":{"constant":"a-different-name"}}}},"taskInfo":{"name":"make-name"}}}},"inputDefinitions":{"parameters":{"int_input":{"defaultValue":1,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"name":{"defaultValue":"default-name","isOptional":true,"parameterType":"STRING"},"pipeline_input_artifact_uri":{"defaultValue":"gs://ml-pipeline-playground/shakespeare1.txt","isOptional":true,"parameterType":"STRING"},"pipeline_input_image_uri":{"defaultValue":"us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-5:latest","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -290,11 +290,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-958372f93b012956585660b34fdee7e4ee56bc8305edc68a225e8d9fd0e274f2}}' + value: '{{workflow.parameters.components-1f5c6ec51212707117cf959e66eed87794baf9ac92c3f57c43a8e3ce06397f03}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-name"},"inputs":{"parameters":{"name":{"runtimeValue":{"constant":"a-different-name"}}}},"taskInfo":{"name":"make-name"}}' - name: container - value: '{{workflow.parameters.implementations-958372f93b012956585660b34fdee7e4ee56bc8305edc68a225e8d9fd0e274f2}}' + value: '{{workflow.parameters.implementations-1f5c6ec51212707117cf959e66eed87794baf9ac92c3f57c43a8e3ce06397f03}}' - name: task-name value: make-name - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index 19cfc0048ef..ed2d187edec 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c + - name: components-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8 value: '{"executorLabel":"exec-accelerator-count","outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c + - name: implementations-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_count"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_count() -\u003e int:\n return 1\n\n"],"image":"python:3.9"}' - - name: components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + accelerator_count() -\u003e int:\n return 1\n\n"],"image":"python:3.11"}' + - name: components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"executorLabel":"exec-accelerator-type","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + - name: implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_type"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.9"}' + accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.11"}' - name: components-52a7e68811aea0ab261ad8816e36b43d20ed63e304bf491e77d3a1ac7b6e5871 value: '{"executorLabel":"exec-custom-training-job","inputDefinitions":{"parameters":{"base_output_directory":{"defaultValue":"","description":"The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. @@ -84,9 +84,9 @@ spec: \"base_output_directory\": {\"output_uri_prefix\": \"{{$.inputs.parameters[''base_output_directory'']}}\"}}, \"labels\": {{$.inputs.parameters[''labels'']}}, \"encryption_spec\": {\"kms_key_name\": \"{{$.inputs.parameters[''encryption_spec_key_name'']}}\"}}","--project","{{$.inputs.parameters[''project'']}}","--location","{{$.inputs.parameters[''location'']}}","--gcp_resources","{{$.outputs.parameters[''gcp_resources''].output_file}}"],"command":["python3","-u","-m","google_cloud_pipeline_components.container.v1.custom_job.launcher"],"image":"gcr.io/ml-pipeline/google-cloud-pipeline-components:2.14.1"}' - - name: components-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972 + - name: components-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715 value: '{"executorLabel":"exec-machine-type","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972 + - name: implementations-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715 value: '{"args":["--executor_input","{{$}}","--function_to_execute","machine_type"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -96,7 +96,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - machine_type() -\u003e str:\n return ''n1-standard-4''\n\n"],"image":"python:3.9"}' + machine_type() -\u003e str:\n return ''n1-standard-4''\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"accelerator-count":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-count"},"taskInfo":{"name":"accelerator-count"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"custom-training-job":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-custom-training-job"},"dependentTasks":["accelerator-count","accelerator-type","machine-type"],"inputs":{"parameters":{"display_name":{"runtimeValue":{"constant":"add-numbers"}},"encryption_spec_key_name":{"componentInputParameter":"encryption_spec_key_name"},"location":{"componentInputParameter":"location"},"pipelinechannel--accelerator-count-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-count"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--machine-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"machine-type"}},"project":{"componentInputParameter":"project"},"worker_pool_specs":{"runtimeValue":{"constant":[{"container_spec":{"args":["foo"],"command":["echo"],"image_uri":"gcr.io/ml-pipeline/google-cloud-pipeline-components:2.5.0"},"machine_spec":{"accelerator_count":"{{$.inputs.parameters[''pipelinechannel--accelerator-count-Output'']}}","accelerator_type":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}","machine_type":"{{$.inputs.parameters[''pipelinechannel--machine-type-Output'']}}"},"replica_count":1}]}}}},"taskInfo":{"name":"custom-training-job"}},"machine-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-machine-type"},"taskInfo":{"name":"machine-type"}}}},"inputDefinitions":{"parameters":{"encryption_spec_key_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"location":{"parameterType":"STRING"},"project":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -279,11 +279,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c}}' + value: '{{workflow.parameters.components-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-count"},"taskInfo":{"name":"accelerator-count"}}' - name: container - value: '{{workflow.parameters.implementations-4ff7c4319829a8830447ba783584f0f40734ff010482a6b9b234ceda17ba214c}}' + value: '{{workflow.parameters.implementations-b644111ca20a6ae3d9e716f9eda564a01b648ac879e153b674558d9c9a9da6e8}}' - name: task-name value: accelerator-count - name: parent-dag-id @@ -303,11 +303,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}}' - name: container - value: '{{workflow.parameters.implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task-name value: accelerator-type - name: parent-dag-id @@ -352,11 +352,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972}}' + value: '{{workflow.parameters.components-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-machine-type"},"taskInfo":{"name":"machine-type"}}' - name: container - value: '{{workflow.parameters.implementations-cc24d59ed416d6169227e7ef954078b09337657b6eb903c1ee2afe6e0d9a5972}}' + value: '{{workflow.parameters.implementations-a8764ade5a7c2d0a4863791e694b80370ee266ca37f2aa429d49e94e203e1715}}' - name: task-name value: machine-type - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index c4546030e8e..0d7f8083e6b 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -11,9 +11,9 @@ spec: - name: implementations-507ec52d0dba96e88845f3ca614b4262d4b5b44fa204a91cd13cb60c8a78d4fe value: '{"command":["sh","-c","set -e -x\necho \"$ENV1\"\necho \"$ENV2\"\necho \"$ENV3\"\n"],"env":[{"name":"ENV1","value":"val0"},{"name":"ENV2","value":"val2"},{"name":"ENV3","value":"val3"}],"image":"alpine"}' - - name: components-62080971c5693dd5ee364f93012c4d84ed64ae7e434d0b5b1e61b3ab408ccc53 + - name: components-dba9846467b474d60f2e0c7f2251f75735fc65a98ecebf7ffcedbd015595be80 value: '{"executorLabel":"exec-print-env-op"}' - - name: implementations-62080971c5693dd5ee364f93012c4d84ed64ae7e434d0b5b1e61b3ab408ccc53 + - name: implementations-dba9846467b474d60f2e0c7f2251f75735fc65a98ecebf7ffcedbd015595be80 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_env_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -24,7 +24,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_env_op():\n import os\n print(''ENV1'', os.environ.get(''ENV1''))\n print(''ENV2'', - os.environ.get(''ENV2''))\n\n"],"env":[{"name":"ENV1","value":"val1"}],"image":"python:3.9"}' + os.environ.get(''ENV2''))\n\n"],"env":[{"name":"ENV1","value":"val1"}],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"print-env":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-env"},"taskInfo":{"name":"print-env"}},"print-env-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-env-op"},"taskInfo":{"name":"print-env-op"}}}}}' entrypoint: entrypoint @@ -231,11 +231,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-62080971c5693dd5ee364f93012c4d84ed64ae7e434d0b5b1e61b3ab408ccc53}}' + value: '{{workflow.parameters.components-dba9846467b474d60f2e0c7f2251f75735fc65a98ecebf7ffcedbd015595be80}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-env-op"},"taskInfo":{"name":"print-env-op"}}' - name: container - value: '{{workflow.parameters.implementations-62080971c5693dd5ee364f93012c4d84ed64ae7e434d0b5b1e61b3ab408ccc53}}' + value: '{{workflow.parameters.implementations-dba9846467b474d60f2e0c7f2251f75735fc65a98ecebf7ffcedbd015595be80}}' - name: task-name value: print-env-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index 752d77c870a..527cb00d03b 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.9"}' - - name: components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.11"}' + - name: components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + - name: implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-comp-exit-handler-1 value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--message":{"parameterType":"STRING"}}}}' @@ -219,12 +219,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task-name value: fail-op - name: parent-dag-id @@ -244,11 +244,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-2 - name: parent-dag-id @@ -276,12 +276,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index fa51453684b..bed4515a363 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -10,9 +10,9 @@ spec: value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"google.VertexDataset","schemaVersion":"0.0.0"}}}}}' - name: implementations-comp-importer value: '{"artifactUri":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"},"metadata":{"key":"value"},"typeSchema":{"schemaTitle":"google.VertexDataset","schemaVersion":"0.0.0"}}' - - name: components-294c7093bd6007beda02527c7a120966fc8e353153d9512485700a047f484f0b + - name: components-d95089443b1478c8651b4c7907aff4ad91082dd878a05b164608a7958a55d36e value: '{"executorLabel":"exec-model-consumer","inputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"google.VertexDataset","schemaVersion":"0.0.0"}},"model":{"artifactType":{"schemaTitle":"google.VertexModel","schemaVersion":"0.0.0"}}}}}' - - name: implementations-294c7093bd6007beda02527c7a120966fc8e353153d9512485700a047f484f0b + - name: implementations-d95089443b1478c8651b4c7907aff4ad91082dd878a05b164608a7958a55d36e value: '{"args":["--executor_input","{{$}}","--function_to_execute","model_consumer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -28,10 +28,10 @@ spec: '', type(model))\n print(''artifact.name: '', model.name)\n print(''artifact.uri: '', model.uri)\n print(''artifact.metadata: '', model.metadata)\n\n print(''Dataset'')\n print(''artifact.type: '', type(dataset))\n print(''artifact.name: '', dataset.name)\n print(''artifact.uri: - '', dataset.uri)\n print(''artifact.metadata: '', dataset.metadata)\n\n"],"image":"python:3.9"}' - - name: components-3d4953dfcdcfe41d72b761df76bfc76edf185ecf8415d539fd17c4c6f8f29200 + '', dataset.uri)\n print(''artifact.metadata: '', dataset.metadata)\n\n"],"image":"python:3.11"}' + - name: components-e9925d1cd495bcfa47d9908820b0f2cbd7304b8cd8e49bc581e7ec0d2e8b0d94 value: '{"executorLabel":"exec-model-producer","outputDefinitions":{"artifacts":{"model":{"artifactType":{"schemaTitle":"google.VertexModel","schemaVersion":"0.0.0"}}}}}' - - name: implementations-3d4953dfcdcfe41d72b761df76bfc76edf185ecf8415d539fd17c4c6f8f29200 + - name: implementations-e9925d1cd495bcfa47d9908820b0f2cbd7304b8cd8e49bc581e7ec0d2e8b0d94 value: '{"args":["--executor_input","{{$}}","--function_to_execute","model_producer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -44,7 +44,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\nimport aiplatform\n\ndef model_producer(model: Output[aiplatform.VertexModel]):\n\n assert isinstance(model, aiplatform.VertexModel), type(model)\n with open(model.path, - ''w'') as f:\n f.write(''my model'')\n\n"],"image":"python:3.9"}' + ''w'') as f:\n f.write(''my model'')\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"}}}},"taskInfo":{"name":"importer"}},"model-consumer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-model-consumer"},"dependentTasks":["importer","model-producer"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}},"model":{"taskOutputArtifact":{"outputArtifactKey":"model","producerTask":"model-producer"}}}},"taskInfo":{"name":"model-consumer"}},"model-producer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-model-producer"},"taskInfo":{"name":"model-producer"}}}}}' entrypoint: entrypoint @@ -296,11 +296,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-294c7093bd6007beda02527c7a120966fc8e353153d9512485700a047f484f0b}}' + value: '{{workflow.parameters.components-d95089443b1478c8651b4c7907aff4ad91082dd878a05b164608a7958a55d36e}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-model-consumer"},"dependentTasks":["importer","model-producer"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}},"model":{"taskOutputArtifact":{"outputArtifactKey":"model","producerTask":"model-producer"}}}},"taskInfo":{"name":"model-consumer"}}' - name: container - value: '{{workflow.parameters.implementations-294c7093bd6007beda02527c7a120966fc8e353153d9512485700a047f484f0b}}' + value: '{{workflow.parameters.implementations-d95089443b1478c8651b4c7907aff4ad91082dd878a05b164608a7958a55d36e}}' - name: task-name value: model-consumer - name: parent-dag-id @@ -321,11 +321,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-3d4953dfcdcfe41d72b761df76bfc76edf185ecf8415d539fd17c4c6f8f29200}}' + value: '{{workflow.parameters.components-e9925d1cd495bcfa47d9908820b0f2cbd7304b8cd8e49bc581e7ec0d2e8b0d94}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-model-producer"},"taskInfo":{"name":"model-producer"}}' - name: container - value: '{{workflow.parameters.implementations-3d4953dfcdcfe41d72b761df76bfc76edf185ecf8415d539fd17c4c6f8f29200}}' + value: '{{workflow.parameters.implementations-e9925d1cd495bcfa47d9908820b0f2cbd7304b8cd8e49bc581e7ec0d2e8b0d94}}' - name: task-name value: model-producer - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index fc7e839429b..4335f8d8827 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -10,9 +10,9 @@ spec: value: '{"executorLabel":"exec-importer-2","inputDefinitions":{"parameters":{"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: implementations-comp-importer-2 value: '{"artifactUri":{"runtimeParameter":"uri"},"reimport":true,"typeSchema":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}' - - name: components-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad + - name: components-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0 value: '{"executorLabel":"exec-train-2","inputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"model":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}},"parameters":{"scalar":{"parameterType":"STRING"}}}}' - - name: implementations-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad + - name: implementations-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","train"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -27,7 +27,7 @@ spec: open(dataset.path) as f:\n data = f.read()\n print(''Dataset:'', data)\n\n scalar = ''123''\n model = f''My model trained using data: {data}''\n\n from collections import namedtuple\n output = namedtuple(''Outputs'', - [''scalar'', ''model''])\n return output(scalar, model)\n\n"],"image":"python:3.9"}' + [''scalar'', ''model''])\n return output(scalar, model)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"importer-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer-2"},"inputs":{"parameters":{"uri":{"componentInputParameter":"pipelinechannel--dataset2"}}},"taskInfo":{"name":"importer-2"}},"train-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-train-2"},"dependentTasks":["importer-2"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer-2"}}}},"taskInfo":{"name":"train-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--dataset2":{"parameterType":"STRING"},"pipelinechannel--train-scalar":{"parameterType":"STRING"}}}}' - name: components-comp-importer @@ -286,11 +286,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad}}' + value: '{{workflow.parameters.components-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-train-2"},"dependentTasks":["importer-2"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer-2"}}}},"taskInfo":{"name":"train-2"}}' - name: container - value: '{{workflow.parameters.implementations-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad}}' + value: '{{workflow.parameters.implementations-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0}}' - name: task-name value: train-2 - name: parent-dag-id @@ -432,11 +432,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad}}' + value: '{{workflow.parameters.components-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-train"},"dependentTasks":["importer"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"train"}}' - name: container - value: '{{workflow.parameters.implementations-af321da91af730e40295efe7bfd4833eb55c7f74f04f43f373927c69fb74e9ad}}' + value: '{{workflow.parameters.implementations-3fe46efe149977fb8647a792ff4ac104e78f0aacef1094c802f8cdaf3f3034b0}}' - name: task-name value: train - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 0096adb03c9..220e1b24679 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f + - name: components-6f22693acb9f2a815b4823044bf0a93f4c912566dd68e1d8ca2a680ac71b67d3 value: '{"executorLabel":"exec-echo-state","inputDefinitions":{"parameters":{"status":{"isOptional":true,"parameterType":"TASK_FINAL_STATUS"}}}}' - - name: implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f + - name: implementations-6f22693acb9f2a815b4823044bf0a93f4c912566dd68e1d8ca2a680ac71b67d3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_state"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,10 +21,10 @@ spec: echo_state(status: dsl.PipelineTaskFinalStatus):\n assert(status.state == ''COMPLETE'')\n assert(''status-state-pipeline'' in status.pipeline_job_resource_name)\n assert(status.pipeline_task_name == ''exit-handler-1'')\n #TODO: Add assert statements to validate status.error_code - and status.error_message values once those fields have been implemented.\n\n"],"image":"python:3.9"}' - - name: components-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566 + and status.error_message values once those fields have been implemented.\n\n"],"image":"python:3.11"}' + - name: components-b7759bec7b7c73abcfcc5fe174f2f6965ece187d7e53beabe3e93fc81431ba91 value: '{"executorLabel":"exec-some-task"}' - - name: implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566 + - name: implementations-b7759bec7b7c73abcfcc5fe174f2f6965ece187d7e53beabe3e93fc81431ba91 value: '{"args":["--executor_input","{{$}}","--function_to_execute","some_task"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,7 +34,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - some_task():\n print(''Executing some_task()...'')\n\n"],"image":"python:3.9"}' + some_task():\n print(''Executing some_task()...'')\n\n"],"image":"python:3.11"}' - name: components-comp-exit-handler-1 value: '{"dag":{"tasks":{"some-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}}}}' - name: components-root @@ -219,11 +219,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' + value: '{{workflow.parameters.components-b7759bec7b7c73abcfcc5fe174f2f6965ece187d7e53beabe3e93fc81431ba91}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}' - name: container - value: '{{workflow.parameters.implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' + value: '{{workflow.parameters.implementations-b7759bec7b7c73abcfcc5fe174f2f6965ece187d7e53beabe3e93fc81431ba91}}' - name: task-name value: some-task - name: parent-dag-id @@ -251,11 +251,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' + value: '{{workflow.parameters.components-6f22693acb9f2a815b4823044bf0a93f4c912566dd68e1d8ca2a680ac71b67d3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' + value: '{{workflow.parameters.implementations-6f22693acb9f2a815b4823044bf0a93f4c912566dd68e1d8ca2a680ac71b67d3}}' - name: task-name value: echo-state - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index d814ea4e574..0d0ac3a1b1a 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-16999c93ab894bbf74840a65c88ede65c740b966f18912c02bc4c89745879c66 + - name: components-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530 value: '{"executorLabel":"exec-args-generator-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-16999c93ab894bbf74840a65c88ede65c740b966f18912c02bc4c89745879c66 + - name: implementations-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530 value: '{"args":["--executor_input","{{$}}","--function_to_execute","args_generator_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -\u003e List[Dict[str, str]]:\n return [{''A_a'': ''1'', - ''B_b'': ''2''}, {''A_a'': ''10'', ''B_b'': ''20''}]\n\n"],"image":"python:3.9"}' - - name: components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8 + ''B_b'': ''2''}, {''A_a'': ''10'', ''B_b'': ''20''}]\n\n"],"image":"python:3.11"}' + - name: components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"executorLabel":"exec-print-text","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8 + - name: implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_text"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,12 +32,12 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_text(msg: str):\n print(msg)\n\n"],"image":"python:3.9"}' + print_text(msg: str):\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"tasks":{"print-text":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"}}}}' - - name: components-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890 + - name: components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f value: '{"executorLabel":"exec-print-struct","inputDefinitions":{"parameters":{"struct":{"parameterType":"STRUCT"}}}}' - - name: implementations-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890 + - name: implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_struct"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -47,7 +47,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_struct(struct: Dict):\n print(struct)\n\n"],"image":"python:3.9"}' + print_struct(struct: Dict):\n print(struct)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"print-struct":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"}}},"taskInfo":{"name":"print-struct"}},"print-text-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-2"}},"print-text-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-Output":{"parameterType":"LIST"},"pipelinechannel--args-generator-op-Output-loop-item":{"parameterType":"STRUCT"}}}}' - name: components-comp-for-loop-4 @@ -235,11 +235,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text - name: parent-dag-id @@ -267,11 +267,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890}}' + value: '{{workflow.parameters.components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"}}},"taskInfo":{"name":"print-struct"}}' - name: container - value: '{{workflow.parameters.implementations-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890}}' + value: '{{workflow.parameters.implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task-name value: print-struct - name: parent-dag-id @@ -291,11 +291,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-2"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-2 - name: parent-dag-id @@ -315,11 +315,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-3"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-3 - name: parent-dag-id @@ -347,11 +347,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890}}' + value: '{{workflow.parameters.components-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct-2"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"print-struct-2"}}' - name: container - value: '{{workflow.parameters.implementations-7872878deb112caffddcb40890e034f7c4e309067293f861e38dce761812e890}}' + value: '{{workflow.parameters.implementations-5d3bbc654be05c6e0386e82c98f120e616a0fc39246372d5e8cd1d973990c50f}}' - name: task-name value: print-struct-2 - name: parent-dag-id @@ -371,11 +371,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-4"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-4"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-4 - name: parent-dag-id @@ -395,11 +395,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-5"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-5"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-5 - name: parent-dag-id @@ -683,11 +683,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-16999c93ab894bbf74840a65c88ede65c740b966f18912c02bc4c89745879c66}}' + value: '{{workflow.parameters.components-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op"},"taskInfo":{"name":"args-generator-op"}}' - name: container - value: '{{workflow.parameters.implementations-16999c93ab894bbf74840a65c88ede65c740b966f18912c02bc4c89745879c66}}' + value: '{{workflow.parameters.implementations-687d86676223e8ed6d5a94779ca21c78d7c45d0c14796f2992c0b78dc949e530}}' - name: task-name value: args-generator-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index f473ec1059b..3c5c55befe6 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873 + - name: components-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247 value: '{"executorLabel":"exec-args-generator-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873 + - name: implementations-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247 value: '{"args":["--executor_input","{{$}}","--function_to_execute","args_generator_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -\u003e list:\n return [\n {\n ''A_a'': ''1'',\n ''B_b'': [''2'', ''20''],\n },\n {\n ''A_a'': - ''10'',\n ''B_b'': [''22'', ''222''],\n },\n ]\n\n"],"image":"python:3.9"}' - - name: components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8 + ''10'',\n ''B_b'': [''22'', ''222''],\n },\n ]\n\n"],"image":"python:3.11"}' + - name: components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52 value: '{"executorLabel":"exec-print-text-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"},"msg2":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8 + - name: implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_text"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,14 +34,14 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f''msg: {msg}, - msg2: {msg2}'')\n\n"],"image":"python:3.9"}' + msg2: {msg2}'')\n\n"],"image":"python:3.11"}' - name: components-comp-condition-3 value: '{"dag":{"tasks":{"print-text-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"parameterType":"STRING"},"pipelinechannel--args-generator-op-Output-loop-item-subvar-A_a":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"}}}}' - name: components-comp-condition-4 value: '{"dag":{"tasks":{"print-text-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"}}}}' - - name: components-ce00364eb1c3a39a23f7aede7572897a04347337501043c04e984fc1c1768cce + - name: components-8853c56a797c7cf679a34e10437594f376f7496f7902f009aa8bb52f84544af7 value: '{"executorLabel":"exec-print-struct","inputDefinitions":{"parameters":{"struct":{"parameterType":"STRUCT"}}}}' - - name: implementations-ce00364eb1c3a39a23f7aede7572897a04347337501043c04e984fc1c1768cce + - name: implementations-8853c56a797c7cf679a34e10437594f376f7496f7902f009aa8bb52f84544af7 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_struct"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -51,7 +51,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_struct(struct: dict):\n print(struct)\n\n"],"image":"python:3.9"}' + print_struct(struct: dict):\n print(struct)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-7 value: '{"dag":{"tasks":{"print-struct":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--loop-item-param-6"}}},"taskInfo":{"name":"print-struct"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"},"pipelinechannel--loop-item-param-6":{"parameterType":"STRUCT"}}}}' - name: components-comp-condition-5 @@ -76,9 +76,9 @@ spec: \"2\"]"}},"taskInfo":{"name":"for-loop-12"}},"for-loop-8":{"componentRef":{"name":"comp-for-loop-8"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"},"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b"}},"taskInfo":{"name":"for-loop-8"}},"for-loop-9":{"componentRef":{"name":"comp-for-loop-9"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-9"}},"print-text":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--msg"}}},"taskInfo":{"name":"print-text"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"parameterType":"LIST"},"pipelinechannel--args-generator-op-Output":{"parameterType":"LIST"},"pipelinechannel--args-generator-op-Output-loop-item":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"},"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--msg":{"parameterType":"STRING"}}}}' - name: components-comp-condition-1 value: '{"dag":{"tasks":{"args-generator-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op-2"},"taskInfo":{"name":"args-generator-op-2"}},"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op-2"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op-2"}},"pipelinechannel--args-generator-op-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--msg":{"componentInputParameter":"pipelinechannel--msg"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--args-generator-op-Output":{"parameterType":"LIST"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"},"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--msg":{"parameterType":"STRING"}}}}' - - name: components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"executorLabel":"exec-flip-coin-op","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -90,7 +90,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -\u003e str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.9"}' + 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-16 value: '{"dag":{"tasks":{"print-text-9":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-9"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item"}}},"taskInfo":{"name":"print-text-9"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"},"pipelinechannel--loop_parameter-loop-item-subvar-B_b":{"parameterType":"STRING"},"pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item":{"parameterType":"STRING"}}}}' - name: components-comp-condition-15 @@ -281,11 +281,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-2"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-2 - name: parent-dag-id @@ -313,11 +313,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-3"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-3 - name: parent-dag-id @@ -345,11 +345,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ce00364eb1c3a39a23f7aede7572897a04347337501043c04e984fc1c1768cce}}' + value: '{{workflow.parameters.components-8853c56a797c7cf679a34e10437594f376f7496f7902f009aa8bb52f84544af7}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-struct"},"inputs":{"parameters":{"struct":{"componentInputParameter":"pipelinechannel--loop-item-param-6"}}},"taskInfo":{"name":"print-struct"}}' - name: container - value: '{{workflow.parameters.implementations-ce00364eb1c3a39a23f7aede7572897a04347337501043c04e984fc1c1768cce}}' + value: '{{workflow.parameters.implementations-8853c56a797c7cf679a34e10437594f376f7496f7902f009aa8bb52f84544af7}}' - name: task-name value: print-struct - name: parent-dag-id @@ -529,11 +529,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-8"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"1"}}}},"taskInfo":{"name":"print-text-8"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-8 - name: parent-dag-id @@ -584,11 +584,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-7"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-11"}}},"taskInfo":{"name":"print-text-7"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-7 - name: parent-dag-id @@ -616,11 +616,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-4"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item"}}},"taskInfo":{"name":"print-text-4"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-4 - name: parent-dag-id @@ -648,11 +648,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-6"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"msg2":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-6"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-6 - name: parent-dag-id @@ -745,11 +745,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-5"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text-5"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-5 - name: parent-dag-id @@ -1043,11 +1043,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--msg"}}},"taskInfo":{"name":"print-text"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text - name: parent-dag-id @@ -1134,11 +1134,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873}}' + value: '{{workflow.parameters.components-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op-2"},"taskInfo":{"name":"args-generator-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873}}' + value: '{{workflow.parameters.implementations-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247}}' - name: task-name value: args-generator-op-2 - name: parent-dag-id @@ -1173,11 +1173,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.components-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-9"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item"}}},"taskInfo":{"name":"print-text-9"}}' - name: container - value: '{{workflow.parameters.implementations-5cc139171a458f30e91631e3d693f08f057723d3c0c51040446f26fb0b5fd6c8}}' + value: '{{workflow.parameters.implementations-32f06a188ecaaebfd19502d5193eca7e8ceab20f38ca2c202c113df604e90a52}}' - name: task-name value: print-text-9 - name: parent-dag-id @@ -1368,11 +1368,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873}}' + value: '{{workflow.parameters.components-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-args-generator-op"},"taskInfo":{"name":"args-generator-op"}}' - name: container - value: '{{workflow.parameters.implementations-efceadb4fda65c2cf2180d4b55f8552bb47943172d073cf46fae6237e4d5c873}}' + value: '{{workflow.parameters.implementations-ac68354549bad0bc7c4df15cdbcb5ecaacdd64a74d077c031e7c4c8d91aab247}}' - name: task-name value: args-generator-op - name: parent-dag-id @@ -1416,11 +1416,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op"},"taskInfo":{"name":"flip-coin-op"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index 6b7f4ec34db..736fcb608e7 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -6,13 +6,13 @@ metadata: spec: arguments: parameters: - - name: components-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f + - name: components-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff value: '{"executorLabel":"exec-dataset-joiner","inputDefinitions":{"artifacts":{"dataset_a":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"First dataset."},"dataset_b":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"Second dataset."}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"The concatenated dataset."}},"parameters":{"Output":{"description":"The concatenated string.","parameterType":"STRING"}}}}' - - name: implementations-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f + - name: implementations-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff value: '{"args":["--executor_input","{{$}}","--function_to_execute","dataset_joiner"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -29,12 +29,12 @@ spec: The concatenated string.\n \"\"\"\n with open(dataset_a.path) as f:\n content_a = f.read()\n\n with open(dataset_b.path) as f:\n content_b = f.read()\n\n concatenated_string = content_a + content_b\n with open(out_dataset.path, ''w'') as f:\n f.write(concatenated_string)\n\n return - concatenated_string\n\n"],"image":"python:3.9"}' - - name: components-cf015d965d4dbce23b2fa6520b6fd9409bc8c0e0de1513abaf9d5e1c03d3835d + concatenated_string\n\n"],"image":"python:3.11"}' + - name: components-0067337b015ccea33fdb5b6603365552f4cca4f9eae4b1497bd033c4b3f843b9 value: '{"executorLabel":"exec-str-to-dataset","inputDefinitions":{"parameters":{"string":{"description":"The string.","parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"The dataset."}}}}' - - name: implementations-cf015d965d4dbce23b2fa6520b6fd9409bc8c0e0de1513abaf9d5e1c03d3835d + - name: implementations-0067337b015ccea33fdb5b6603365552f4cca4f9eae4b1497bd033c4b3f843b9 value: '{"args":["--executor_input","{{$}}","--function_to_execute","str_to_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -46,7 +46,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef str_to_dataset(string: str, dataset: Output[Dataset]):\n \"\"\"Convert string to dataset.\n\n Args:\n string: The string.\n\n Returns:\n dataset: - The dataset.\n \"\"\"\n with open(dataset.path, ''w'') as f:\n f.write(string)\n\n"],"image":"python:3.9"}' + The dataset.\n \"\"\"\n with open(dataset.path, ''w'') as f:\n f.write(string)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"out_dataset","producerSubtask":"dataset-joiner"}]}}},"tasks":{"dataset-joiner":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-joiner"},"dependentTasks":["str-to-dataset"],"inputs":{"artifacts":{"dataset_a":{"taskOutputArtifact":{"outputArtifactKey":"dataset","producerTask":"str-to-dataset"}},"dataset_b":{"componentInputArtifact":"in_dataset"}}},"taskInfo":{"name":"dataset-joiner"}},"str-to-dataset":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-str-to-dataset"},"inputs":{"parameters":{"string":{"componentInputParameter":"string"}}},"taskInfo":{"name":"str-to-dataset"}}}},"inputDefinitions":{"artifacts":{"in_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"description":"Dataset to which to concatenate string."}},"parameters":{"string":{"description":"String @@ -232,11 +232,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f}}' + value: '{{workflow.parameters.components-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-joiner"},"dependentTasks":["str-to-dataset"],"inputs":{"artifacts":{"dataset_a":{"taskOutputArtifact":{"outputArtifactKey":"dataset","producerTask":"str-to-dataset"}},"dataset_b":{"componentInputArtifact":"in_dataset"}}},"taskInfo":{"name":"dataset-joiner"}}' - name: container - value: '{{workflow.parameters.implementations-3e4e63ec82d130c2d86a0de3eb781d6a47ff58129bf5f5aa0aa8891ef448dc7f}}' + value: '{{workflow.parameters.implementations-e64230eee42e89b7bed44c01e0e5aadf92c2b123bf1ae03372c1e07eeadee7ff}}' - name: task-name value: dataset-joiner - name: parent-dag-id @@ -257,11 +257,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-cf015d965d4dbce23b2fa6520b6fd9409bc8c0e0de1513abaf9d5e1c03d3835d}}' + value: '{{workflow.parameters.components-0067337b015ccea33fdb5b6603365552f4cca4f9eae4b1497bd033c4b3f843b9}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-str-to-dataset"},"inputs":{"parameters":{"string":{"componentInputParameter":"string"}}},"taskInfo":{"name":"str-to-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-cf015d965d4dbce23b2fa6520b6fd9409bc8c0e0de1513abaf9d5e1c03d3835d}}' + value: '{{workflow.parameters.implementations-0067337b015ccea33fdb5b6603365552f4cca4f9eae4b1497bd033c4b3f843b9}}' - name: task-name value: str-to-dataset - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index d5fb251888f..ea3242719c8 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4 + - name: components-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97 value: '{"executorLabel":"exec-output-metrics-2","outputDefinitions":{"artifacts":{"metrics":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}}}}}' - - name: implementations-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4 + - name: implementations-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97 value: '{"args":["--executor_input","{{$}}","--function_to_execute","output_metrics"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,7 +20,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef output_metrics(metrics: Output[Metrics]):\n \"\"\"Dummy component that outputs metrics with a random accuracy.\"\"\"\n import random\n result - = random.randint(0, 100)\n metrics.log_metric(''accuracy'', result)\n\n"],"image":"python:3.9"}' + = random.randint(0, 100)\n metrics.log_metric(''accuracy'', result)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"output-metrics-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-metrics-2"},"taskInfo":{"name":"output-metrics-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"NUMBER_INTEGER"}}}}' - name: components-root @@ -206,11 +206,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.components-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-metrics-2"},"taskInfo":{"name":"output-metrics-2"}}' - name: container - value: '{{workflow.parameters.implementations-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.implementations-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task-name value: output-metrics-2 - name: parent-dag-id @@ -382,11 +382,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.components-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-output-metrics"},"taskInfo":{"name":"output-metrics"}}' - name: container - value: '{{workflow.parameters.implementations-33c6ad9ab035611571f521208ada9a5231b3729ee087168ce0eeb8765d17fcc4}}' + value: '{{workflow.parameters.implementations-ab5b53a52a65c78da1d8dcfca19f7568d6844ad951c971baa75d318f6528cd97}}' - name: task-name value: output-metrics - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index 645510cfac7..e28c8a67071 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf + - name: implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.9"}' - - name: components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.11"}' + - name: components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"executorLabel":"exec-print-op-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d + - name: implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,7 +31,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-comp-exit-handler-1 value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}},"print-op-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--message":{"parameterType":"STRING"}}}}' @@ -225,12 +225,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.components-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-de343ed68bcc74a4b31b2b5a8230f35dfb47ca8d90eabe51d37dc3849d7acecf}}' + value: '{{workflow.parameters.implementations-0e20296d1c94dd957c12f688261536d608c0a9430b71b2c67c664244a11161e8}}' - name: task-name value: fail-op - name: parent-dag-id @@ -250,11 +250,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-2 - name: parent-dag-id @@ -282,11 +282,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-4"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-4 - name: parent-dag-id @@ -314,11 +314,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-6"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-6"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-6 - name: parent-dag-id @@ -346,12 +346,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"First exit handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op - name: parent-dag-id @@ -379,12 +379,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"dependentTasks":["exit-handler-2"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Second exit handler has worked!"}}}},"taskInfo":{"name":"print-op-3"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-3 - name: parent-dag-id @@ -412,12 +412,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.components-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-5"},"dependentTasks":["exit-handler-3"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Third exit handler has worked!"}}}},"taskInfo":{"name":"print-op-5"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-49c335a1a327c8628eed2e87d0e06fd7a9721bf1e50c93c2a624bd17d2277d8d}}' + value: '{{workflow.parameters.implementations-af17df745243f885d9854f1931988534c66b26d6c60c5279e354d6b3480dceb8}}' - name: task-name value: print-op-5 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 7824bcd6dbc..016fc3dccd5 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"executorLabel":"exec-flip-coin-op-4","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339 + - name: implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98 value: '{"args":["--executor_input","{{$}}","--function_to_execute","flip_coin_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -\u003e str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = ''heads'' if random.randint(0, - 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.9"}' - - name: components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823 + 1) == 0 else ''tails''\n return result\n\n"],"image":"python:3.11"}' + - name: components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5 value: '{"executorLabel":"exec-print-op-4","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823 + - name: implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -33,7 +33,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.9"}' + print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-condition-2 value: '{"dag":{"tasks":{"flip-coin-op-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op-4"},"taskInfo":{"name":"flip-coin-op-4"}},"print-op-4":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"dependentTasks":["flip-coin-op-4"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-4"}}}},"taskInfo":{"name":"print-op-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--flip-coin-op-2-Output":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-3-Output":{"parameterType":"STRING"},"pipelinechannel--flip-coin-op-Output":{"parameterType":"STRING"}}}}' - name: components-comp-condition-1 @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op-4"},"taskInfo":{"name":"flip-coin-op-4"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op-4 - name: parent-dag-id @@ -246,11 +246,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"dependentTasks":["flip-coin-op-4"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-4"}}}},"taskInfo":{"name":"print-op-4"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op-4 - name: parent-dag-id @@ -380,11 +380,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op-3"},"taskInfo":{"name":"flip-coin-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op-3 - name: parent-dag-id @@ -404,11 +404,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"dependentTasks":["flip-coin-op-3"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-3"}}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op-3 - name: parent-dag-id @@ -461,11 +461,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op"},"taskInfo":{"name":"flip-coin-op"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op - name: parent-dag-id @@ -485,11 +485,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.components-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-flip-coin-op-2"},"taskInfo":{"name":"flip-coin-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-74c752241e4386e530a02be0534451962c07a3c96d9c8f0a3b3433cad25b0339}}' + value: '{{workflow.parameters.implementations-663f234c274873c6b1c76e2f91561ee25169ff505cfe45310e51e4b0265a2d98}}' - name: task-name value: flip-coin-op-2 - name: parent-dag-id @@ -509,11 +509,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["flip-coin-op"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op"}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op - name: parent-dag-id @@ -534,11 +534,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.components-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["flip-coin-op-2"],"inputs":{"parameters":{"msg":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"flip-coin-op-2"}}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-ea4c10718a561edf4521c83916c18a873e35e86dce27f6050d1677d0da06b823}}' + value: '{{workflow.parameters.implementations-0a97e2e381526d8572c8a0a776550e66305543ba3a2032eb60de0ef3e42504b5}}' - name: task-name value: print-op-2 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index 69403f3771d..2e06dc3d1dd 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f + - name: components-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77 value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"},"msg2":{"isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f + - name: implementations-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,7 +19,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n print(f''msg: {msg}, - msg2: {msg2}'')\n\n"],"image":"python:3.9"}' + msg2: {msg2}'')\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"q_a\"]"}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"},"pipelinechannel--loop_parameter-loop-item-subvar-p_a":{"parameterType":"STRING"},"pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item":{"parameterType":"STRING"}}}}' - name: components-comp-for-loop-1 @@ -212,11 +212,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.components-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"q_a\"]"}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.implementations-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task-name value: print-op - name: parent-dag-id @@ -394,11 +394,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.components-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3"},"msg2":{"componentInputParameter":"pipelinechannel--loop-item-param-5"}}},"taskInfo":{"name":"print-op-3"}}' - name: container - value: '{{workflow.parameters.implementations-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.implementations-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task-name value: print-op-3 - name: parent-dag-id @@ -493,11 +493,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.components-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-7205c5edce36aade15ea9a24eba3c9a21dcee0e50400f096ce31ecb24d439c9f}}' + value: '{{workflow.parameters.implementations-8fce140f5724af7f5973bdf372e2b23f2cbf1b9e99ac8244a5ec5fa13a3bce77}}' - name: task-name value: print-op-2 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index 648427e4d0b..9fc10b45a82 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb + - name: components-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba value: '{"executorLabel":"exec-print-op1-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb + - name: implementations-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.9"}' + print_op1(msg: str) -\u003e str:\n print(msg)\n return msg\n\n"],"image":"python:3.11"}' - name: components-3f2ba2471e2638f8b27a01f4563316e6e3e8c02bae402875e95b5b86e8de1ddc value: '{"executorLabel":"exec-print-op2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"data":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - name: implementations-3f2ba2471e2638f8b27a01f4563316e6e3e8c02bae402875e95b5b86e8de1ddc @@ -208,11 +208,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb}}' + value: '{{workflow.parameters.components-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb}}' + value: '{{workflow.parameters.implementations-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba}}' - name: task-name value: print-op1 - name: parent-dag-id @@ -363,11 +363,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb}}' + value: '{{workflow.parameters.components-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - name: container - value: '{{workflow.parameters.implementations-c83060295ab94fddb3f516b217b28974809d6b60bd538d1a56c520c6ed9b7ecb}}' + value: '{{workflow.parameters.implementations-50751e65c3e6d87ebc6013c4afbd5fca5a399580ffb9de27fe21ca0af60f37ba}}' - name: task-name value: print-op1 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index 37b26cd977b..8ac5f05fb46 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8 + - name: components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"executorLabel":"exec-print-text-2","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8 + - name: implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_text"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,14 +18,14 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_text(msg: str):\n print(msg)\n\n"],"image":"python:3.9"}' + print_text(msg: str):\n print(msg)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"print-text-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"}}}}' - name: components-comp-for-loop-1 value: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}},"print-text":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop_parameter":{"parameterType":"LIST"},"pipelinechannel--loop_parameter-loop-item":{"parameterType":"STRING"}}}}' - - name: components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a + - name: components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8 value: '{"executorLabel":"exec-print-int-3","inputDefinitions":{"parameters":{"x":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a + - name: implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_int"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -35,7 +35,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_int(x: int):\n print(x)\n\n"],"image":"python:3.9"}' + print_int(x: int):\n print(x)\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-10 value: '{"dag":{"tasks":{"print-int-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-3"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-1-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--list-dict-maker-1-Output":{"parameterType":"LIST"},"pipelinechannel--list-dict-maker-1-Output-loop-item":{"parameterType":"STRUCT"}}}}' - name: components-comp-for-loop-11 @@ -53,9 +53,9 @@ spec: value: '{"dag":{"tasks":{"print-int":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--loop-item-param-7","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-7":{"parameterType":"STRUCT"}}}}' - name: components-comp-for-loop-9 value: '{"dag":{"tasks":{"print-int-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-2"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-0-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--list-dict-maker-0-Output":{"parameterType":"LIST"},"pipelinechannel--list-dict-maker-0-Output-loop-item":{"parameterType":"STRUCT"}}}}' - - name: components-836a4c37a90c1e5ed46b6d5da20665c829d3deeca0e86399d01477c361d28d9b + - name: components-06195db0e7bb31c20391b65c5986a0f0166ba2c57e589e9f8ec5ff0c8ab2bf6b value: '{"executorLabel":"exec-list-dict-maker-0","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-836a4c37a90c1e5ed46b6d5da20665c829d3deeca0e86399d01477c361d28d9b + - name: implementations-06195db0e7bb31c20391b65c5986a0f0166ba2c57e589e9f8ec5ff0c8ab2bf6b value: '{"args":["--executor_input","{{$}}","--function_to_execute","list_dict_maker_0"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -68,10 +68,10 @@ spec: list_dict_maker_0() -\u003e List[Dict[str, int]]:\n \"\"\"Enforces strict type checking - returns a list of dictionaries \n where keys are strings and values are integers. For testing type \n handling during compilation.\"\"\"\n return - [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.9"}' - - name: components-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df + [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.11"}' + - name: components-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4 value: '{"executorLabel":"exec-list-dict-maker-1","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df + - name: implementations-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4 value: '{"args":["--executor_input","{{$}}","--function_to_execute","list_dict_maker_1"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -84,10 +84,10 @@ spec: list_dict_maker_1() -\u003e List[Dict]:\n \"\"\"Utilizes generic dictionary typing (no enforcement of specific key or\n value types).\n\n Tests flexibility in type handling.\n \"\"\"\n return [{''a'': 1, ''b'': 2}, - {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.9"}' - - name: components-8d525622f8b91622073787310058be85ff563a019f2e268d47bffa49d10b0342 + {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.11"}' + - name: components-cccaadf427920aa88170927a2a48949cff1715df09e55aedcdc2a5713766b51c value: '{"executorLabel":"exec-list-dict-maker-2","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-8d525622f8b91622073787310058be85ff563a019f2e268d47bffa49d10b0342 + - name: implementations-cccaadf427920aa88170927a2a48949cff1715df09e55aedcdc2a5713766b51c value: '{"args":["--executor_input","{{$}}","--function_to_execute","list_dict_maker_2"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -99,10 +99,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef list_dict_maker_2() -\u003e List[dict]:\n \"\"\"Returns a list of dictionaries without type enforcement.\n\n Tests flexibility in type handling.\n \"\"\"\n return - [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.9"}' - - name: components-c48bb447bbb57c3886a493ebda03842c1ef51f6d4dd651635e917f508d1175dd + [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.11"}' + - name: components-7fcc431058b62e0fd2bab27c674e9083faadc5d7b7d9a6fccd78345c0fd58286 value: '{"executorLabel":"exec-list-dict-maker-3","outputDefinitions":{"parameters":{"Output":{"parameterType":"LIST"}}}}' - - name: implementations-c48bb447bbb57c3886a493ebda03842c1ef51f6d4dd651635e917f508d1175dd + - name: implementations-7fcc431058b62e0fd2bab27c674e9083faadc5d7b7d9a6fccd78345c0fd58286 value: '{"args":["--executor_input","{{$}}","--function_to_execute","list_dict_maker_3"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -114,7 +114,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef list_dict_maker_3() -\u003e List:\n \"\"\"Returns a basic list (no typing or structure guarantees).\n\n Tests the limits of compiler type handling.\n \"\"\"\n return - [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.9"}' + [{''a'': 1, ''b'': 2}, {''a'': 2, ''b'': 3}, {''a'': 3, ''b'': 4}]\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}},"for-loop-10":{"componentRef":{"name":"comp-for-loop-10"},"dependentTasks":["list-dict-maker-1"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-Output"}},"taskInfo":{"name":"for-loop-10"}},"for-loop-11":{"componentRef":{"name":"comp-for-loop-11"},"dependentTasks":["list-dict-maker-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-2-Output"}},"taskInfo":{"name":"for-loop-11"}},"for-loop-12":{"componentRef":{"name":"comp-for-loop-12"},"dependentTasks":["list-dict-maker-3"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-3-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-3"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-3-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-3-Output"}},"taskInfo":{"name":"for-loop-12"}},"for-loop-13":{"componentRef":{"name":"comp-for-loop-13"},"dependentTasks":["list-dict-maker-1-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-2-Output"}},"taskInfo":{"name":"for-loop-13"}},"for-loop-4":{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}},"for-loop-8":{"componentRef":{"name":"comp-for-loop-8"},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-7","items":{"raw":"[{\"a\": @@ -299,11 +299,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text-2"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-2 - name: parent-dag-id @@ -473,11 +473,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"taskInfo":{"name":"print-text"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text - name: parent-dag-id @@ -505,11 +505,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-3"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-1-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-3"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int-3 - name: parent-dag-id @@ -537,11 +537,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-4"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-2-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-4"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int-4 - name: parent-dag-id @@ -569,11 +569,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-5"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-3-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-5"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int-5 - name: parent-dag-id @@ -601,11 +601,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-6"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-1-2-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-6"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int-6 - name: parent-dag-id @@ -633,11 +633,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-5"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-5","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-5"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-5 - name: parent-dag-id @@ -657,11 +657,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-6"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-5","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-6"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-6 - name: parent-dag-id @@ -757,11 +757,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-3"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-text-3"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-3 - name: parent-dag-id @@ -781,11 +781,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.components-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-text-4"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-text-4"}}' - name: container - value: '{{workflow.parameters.implementations-c70825d1fe54b54a70f9bc27f8c81631450492bb07eeab3259d332992c5e48d8}}' + value: '{{workflow.parameters.implementations-a37ae37968ac30a4c2376035e115521a351c56fae22ef37e91f7df600a415f7d}}' - name: task-name value: print-text-4 - name: parent-dag-id @@ -813,11 +813,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--loop-item-param-7","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int - name: parent-dag-id @@ -845,11 +845,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.components-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-int-2"},"inputs":{"parameters":{"x":{"componentInputParameter":"pipelinechannel--list-dict-maker-0-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"a\"]"}}},"taskInfo":{"name":"print-int-2"}}' - name: container - value: '{{workflow.parameters.implementations-7c6688357d7777b7dadbdb7acda6f1774e7c9fc71f6d002466243931bf300c2a}}' + value: '{{workflow.parameters.implementations-7898b23e6d22eaa693de615bf8e0d54ba7a75b5eacd5bc1ba5fa2a79afa961f8}}' - name: task-name value: print-int-2 - name: parent-dag-id @@ -1408,11 +1408,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-836a4c37a90c1e5ed46b6d5da20665c829d3deeca0e86399d01477c361d28d9b}}' + value: '{{workflow.parameters.components-06195db0e7bb31c20391b65c5986a0f0166ba2c57e589e9f8ec5ff0c8ab2bf6b}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-list-dict-maker-0"},"taskInfo":{"name":"list-dict-maker-0"}}' - name: container - value: '{{workflow.parameters.implementations-836a4c37a90c1e5ed46b6d5da20665c829d3deeca0e86399d01477c361d28d9b}}' + value: '{{workflow.parameters.implementations-06195db0e7bb31c20391b65c5986a0f0166ba2c57e589e9f8ec5ff0c8ab2bf6b}}' - name: task-name value: list-dict-maker-0 - name: parent-dag-id @@ -1432,11 +1432,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df}}' + value: '{{workflow.parameters.components-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-list-dict-maker-1"},"taskInfo":{"name":"list-dict-maker-1"}}' - name: container - value: '{{workflow.parameters.implementations-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df}}' + value: '{{workflow.parameters.implementations-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4}}' - name: task-name value: list-dict-maker-1 - name: parent-dag-id @@ -1456,11 +1456,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df}}' + value: '{{workflow.parameters.components-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-list-dict-maker-1-2"},"taskInfo":{"name":"list-dict-maker-1-2"}}' - name: container - value: '{{workflow.parameters.implementations-1ae8be565131b8c7ec5ffc8e03ac7af15afb6dcd95ef8d33087aca373a4b70df}}' + value: '{{workflow.parameters.implementations-01c5f47f9250430c22049f8c5a56a816a7b031058d432946165fb45bb014e6c4}}' - name: task-name value: list-dict-maker-1-2 - name: parent-dag-id @@ -1480,11 +1480,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8d525622f8b91622073787310058be85ff563a019f2e268d47bffa49d10b0342}}' + value: '{{workflow.parameters.components-cccaadf427920aa88170927a2a48949cff1715df09e55aedcdc2a5713766b51c}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-list-dict-maker-2"},"taskInfo":{"name":"list-dict-maker-2"}}' - name: container - value: '{{workflow.parameters.implementations-8d525622f8b91622073787310058be85ff563a019f2e268d47bffa49d10b0342}}' + value: '{{workflow.parameters.implementations-cccaadf427920aa88170927a2a48949cff1715df09e55aedcdc2a5713766b51c}}' - name: task-name value: list-dict-maker-2 - name: parent-dag-id @@ -1504,11 +1504,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c48bb447bbb57c3886a493ebda03842c1ef51f6d4dd651635e917f508d1175dd}}' + value: '{{workflow.parameters.components-7fcc431058b62e0fd2bab27c674e9083faadc5d7b7d9a6fccd78345c0fd58286}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-list-dict-maker-3"},"taskInfo":{"name":"list-dict-maker-3"}}' - name: container - value: '{{workflow.parameters.implementations-c48bb447bbb57c3886a493ebda03842c1ef51f6d4dd651635e917f508d1175dd}}' + value: '{{workflow.parameters.implementations-7fcc431058b62e0fd2bab27c674e9083faadc5d7b7d9a6fccd78345c0fd58286}}' - name: task-name value: list-dict-maker-3 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 243e842caaf..1c3edcf3e50 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-bdc598acad507af0648cefd0773f42b1236c6c71afe115dd49ee06605c3e7a6e + - name: components-eceb448d63a8e48362b7a15bc3e511d1455ffdbb43c514e0a03d7c39df3c06be value: '{"executorLabel":"exec-print-op2","inputDefinitions":{"parameters":{"text1":{"parameterType":"STRING"},"text2":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-bdc598acad507af0648cefd0773f42b1236c6c71afe115dd49ee06605c3e7a6e + - name: implementations-eceb448d63a8e48362b7a15bc3e511d1455ffdbb43c514e0a03d7c39df3c06be value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op2"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,13 +19,13 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_op2(text1: str, text2: str) -\u003e str:\n print(text1 + text2)\n return - text1 + text2\n\n"],"image":"python:3.9"}' + text1 + text2\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-2 value: '{"dag":{"tasks":{"print-op2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"pipelinechannel--name"},"text1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"},"text2":{"runtimeValue":{"constant":" and {{$.inputs.parameters[''pipelinechannel--name'']}}."}}}},"taskInfo":{"name":"print-op2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--loop-item-param-1":{"parameterType":"STRING"},"pipelinechannel--name":{"parameterType":"STRING"}}}}' - - name: components-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f + - name: components-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88 value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f + - name: implementations-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -35,7 +35,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.9"}' + print_op(text: str) -\u003e str:\n print(text)\n return text\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"for-loop-2":{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"name"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-2"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"name"},"text":{"runtimeValue":{"constant":"Hello @@ -221,12 +221,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-bdc598acad507af0648cefd0773f42b1236c6c71afe115dd49ee06605c3e7a6e}}' + value: '{{workflow.parameters.components-eceb448d63a8e48362b7a15bc3e511d1455ffdbb43c514e0a03d7c39df3c06be}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"pipelinechannel--name"},"text1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"},"text2":{"runtimeValue":{"constant":" and {{$.inputs.parameters[''pipelinechannel--name'']}}."}}}},"taskInfo":{"name":"print-op2"}}' - name: container - value: '{{workflow.parameters.implementations-bdc598acad507af0648cefd0773f42b1236c6c71afe115dd49ee06605c3e7a6e}}' + value: '{{workflow.parameters.implementations-eceb448d63a8e48362b7a15bc3e511d1455ffdbb43c514e0a03d7c39df3c06be}}' - name: task-name value: print-op2 - name: parent-dag-id @@ -398,12 +398,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f}}' + value: '{{workflow.parameters.components-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"name"},"text":{"runtimeValue":{"constant":"Hello {{$.inputs.parameters[''pipelinechannel--name'']}}"}}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f}}' + value: '{{workflow.parameters.implementations-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88}}' - name: task-name value: print-op - name: parent-dag-id @@ -423,12 +423,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f}}' + value: '{{workflow.parameters.components-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"dependentTasks":["print-op"],"inputs":{"parameters":{"pipelinechannel--print-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op"}},"text":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--print-op-Output'']}}, again."}}}},"taskInfo":{"name":"print-op-2"}}' - name: container - value: '{{workflow.parameters.implementations-984f43d1ff549827aa35bc616c4b5b6479714c835e683349f065c2da19055b4f}}' + value: '{{workflow.parameters.implementations-321fe99287ca488cce9f936c3f6cfc0f77e2412bb1dbc00af4495c6701e41a88}}' - name: task-name value: print-op-2 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index bf780c438c9..e5f87de8121 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-1c2d9ee5193f6bab323886b46d1c14a22e2f07a25c79fd4e264b5f797275a7cb + - name: components-252e4a170354e881b8fd2729c1e3ee58684e5eee1443c6782901dd4330265f5e value: '{"executorLabel":"exec-print-all-placeholders","inputDefinitions":{"parameters":{"job_id":{"parameterType":"STRING"},"job_name":{"parameterType":"STRING"},"job_resource_name":{"parameterType":"STRING"},"task_id":{"parameterType":"STRING"},"task_name":{"parameterType":"STRING"}}}}' - - name: implementations-1c2d9ee5193f6bab323886b46d1c14a22e2f07a25c79fd4e264b5f797275a7cb + - name: implementations-252e4a170354e881b8fd2729c1e3ee58684e5eee1443c6782901dd4330265f5e value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_all_placeholders"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -25,7 +25,7 @@ spec: \"\":\n raise RuntimeError(\n \"Expected the placeholder to be replaced with a value: \" + placeholder\n )\n\n assert task_name == \"print-all-placeholders\"\n assert job_resource_name.startswith(\"pipeline-with-placeholders-\")\n\n output - = \", \".join(allPlaceholders)\n print(output)\n\n"],"image":"python:3.9"}' + = \", \".join(allPlaceholders)\n print(output)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"print-all-placeholders":{"cachingOptions":{},"componentRef":{"name":"comp-print-all-placeholders"},"inputs":{"parameters":{"job_id":{"runtimeValue":{"constant":"{{$.pipeline_job_uuid}}"}},"job_name":{"runtimeValue":{"constant":"{{$.pipeline_job_name}}"}},"job_resource_name":{"runtimeValue":{"constant":"{{$.pipeline_job_resource_name}}"}},"task_id":{"runtimeValue":{"constant":"{{$.pipeline_task_uuid}}"}},"task_name":{"runtimeValue":{"constant":"{{$.pipeline_task_name}}"}}}},"taskInfo":{"name":"print-all-placeholders"}}}}}' entrypoint: entrypoint @@ -208,11 +208,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-1c2d9ee5193f6bab323886b46d1c14a22e2f07a25c79fd4e264b5f797275a7cb}}' + value: '{{workflow.parameters.components-252e4a170354e881b8fd2729c1e3ee58684e5eee1443c6782901dd4330265f5e}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-print-all-placeholders"},"inputs":{"parameters":{"job_id":{"runtimeValue":{"constant":"{{$.pipeline_job_uuid}}"}},"job_name":{"runtimeValue":{"constant":"{{$.pipeline_job_name}}"}},"job_resource_name":{"runtimeValue":{"constant":"{{$.pipeline_job_resource_name}}"}},"task_id":{"runtimeValue":{"constant":"{{$.pipeline_task_uuid}}"}},"task_name":{"runtimeValue":{"constant":"{{$.pipeline_task_name}}"}}}},"taskInfo":{"name":"print-all-placeholders"}}' - name: container - value: '{{workflow.parameters.implementations-1c2d9ee5193f6bab323886b46d1c14a22e2f07a25c79fd4e264b5f797275a7cb}}' + value: '{{workflow.parameters.implementations-252e4a170354e881b8fd2729c1e3ee58684e5eee1443c6782901dd4330265f5e}}' - name: task-name value: print-all-placeholders - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index e562d5cbb12..8605d61e14b 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-validate-no-pod-metadata value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation'']","name":"POD_TASK_ANNOTATION"},{"fieldPath":"metadata.annotations[''task-annotation-1'']","name":"POD_TASK_ANNOTATION_1"},{"fieldPath":"metadata.labels[''task-label'']","name":"POD_TASK_LABEL"},{"fieldPath":"metadata.labels[''task-label-1'']","name":"POD_TASK_LABEL_1"}]}' - - name: components-3d278faa61c986d801a417f587e50005053b72d5108dc720e6ce32593c050c24 + - name: components-3e1061118c8caaf7fb2580deca4c26c68777ea648895af667b433198f8d95a35 value: '{"executorLabel":"exec-validate-no-pod-metadata","outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-3d278faa61c986d801a417f587e50005053b72d5108dc720e6ce32593c050c24 + - name: implementations-3e1061118c8caaf7fb2580deca4c26c68777ea648895af667b433198f8d95a35 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_no_pod_metadata"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -26,12 +26,12 @@ spec: annotation is {annotation} but is supposed to be None.''\n )\n label = os.getenv(''POD_TASK_LABEL'')\n label_2 = os.getenv(''POD_TASK_LABEL_1'')\n if label != '''' or label_2 != '''':\n raise ValueError(\n f''Pod - label is {label} but is supposed to be None.''\n )\n return True\n\n"],"image":"python:3.9"}' + label is {label} but is supposed to be None.''\n )\n return True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata-task-a value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation'']","name":"POD_TASK_ANNOTATION"},{"fieldPath":"metadata.labels[''task-label-1'']","name":"POD_TASK_LABEL_1"},{"fieldPath":"metadata.labels[''task-label-2'']","name":"POD_TASK_LABEL_2"}],"podMetadata":{"annotations":{"task-annotation":"annotation"},"labels":{"task-label-1":"label-1","task-label-2":"label-2"}}}' - - name: components-a58a8ba91674b53e9cfd2a56b122fa32c32733648c4337cfd4d489147baa0351 + - name: components-36b4f5715a9e847c587a42d8970df700b9d7977c0097b0b0a12b7be3bc7cb4f8 value: '{"executorLabel":"exec-validate-pod-metadata-task-a","inputDefinitions":{"parameters":{"annotation_exp_val":{"parameterType":"STRING"},"annotation_path":{"parameterType":"STRING"},"label_exp_val_1":{"parameterType":"STRING"},"label_exp_val_2":{"parameterType":"STRING"},"label_path_1":{"parameterType":"STRING"},"label_path_2":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-a58a8ba91674b53e9cfd2a56b122fa32c32733648c4337cfd4d489147baa0351 + - name: implementations-36b4f5715a9e847c587a42d8970df700b9d7977c0097b0b0a12b7be3bc7cb4f8 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_pod_metadata_task_a"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -52,12 +52,12 @@ spec: be {label_exp_val_1}.''\n )\n\n label_val_2 = os.getenv(label_path_2)\n if label_val_2 != label_exp_val_2:\n raise ValueError(\n f''Pod label is {label_val_2} but is supposed to be {label_exp_val_2}.''\n )\n return - True\n\n"],"image":"python:3.9"}' + True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata-task-b value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation-1'']","name":"POD_TASK_ANNOTATION_1"},{"fieldPath":"metadata.annotations[''task-annotation-2'']","name":"POD_TASK_ANNOTATION_2"},{"fieldPath":"metadata.annotations[''task-annotation-3'']","name":"POD_TASK_ANNOTATION_3"},{"fieldPath":"metadata.annotations[''task-annotation-4'']","name":"POD_TASK_ANNOTATION_4"},{"fieldPath":"metadata.labels[''task-label-1'']","name":"POD_TASK_LABEL_1"},{"fieldPath":"metadata.labels[''task-label-2'']","name":"POD_TASK_LABEL_2"},{"fieldPath":"metadata.labels[''task-label-3'']","name":"POD_TASK_LABEL_3"}],"podMetadata":{"annotations":{"task-annotation-1":"annotation-1","task-annotation-2":"annotation-2","task-annotation-3":"annotation-3","task-annotation-4":"annotation-4"},"labels":{"task-label-1":"label-1","task-label-2":"label-2","task-label-3":"label-3"}}}' - - name: components-ff4adfd20fbb1ccd7f6e9330882576b7e34ffba16bc568acc10956b8652c28d9 + - name: components-e07ac8a27d331c96ac2d5559c43f483a79d4e8956b1941d0e306aee450ad83f1 value: '{"executorLabel":"exec-validate-pod-metadata-task-b","inputDefinitions":{"parameters":{"annotation_exp_val_1":{"parameterType":"STRING"},"annotation_exp_val_2":{"parameterType":"STRING"},"annotation_exp_val_3":{"parameterType":"STRING"},"annotation_exp_val_4":{"parameterType":"STRING"},"annotation_path_1":{"parameterType":"STRING"},"annotation_path_2":{"parameterType":"STRING"},"annotation_path_3":{"parameterType":"STRING"},"annotation_path_4":{"parameterType":"STRING"},"label_exp_val_1":{"parameterType":"STRING"},"label_exp_val_2":{"parameterType":"STRING"},"label_exp_val_3":{"parameterType":"STRING"},"label_path_1":{"parameterType":"STRING"},"label_path_2":{"parameterType":"STRING"},"label_path_3":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-ff4adfd20fbb1ccd7f6e9330882576b7e34ffba16bc568acc10956b8652c28d9 + - name: implementations-e07ac8a27d331c96ac2d5559c43f483a79d4e8956b1941d0e306aee450ad83f1 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_pod_metadata_task_b"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -92,12 +92,12 @@ spec: be {label_exp_val_2}.''\n )\n label_val_3 = os.getenv(label_path_3)\n if label_val_3 != label_exp_val_3:\n raise ValueError(\n f''Pod label is {label_val_3} but is supposed to be {label_exp_val_3}.''\n )\n return - True\n\n"],"image":"python:3.9"}' + True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata-task-c value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.annotations[''task-annotation-1'']","name":"POD_TASK_ANNOTATION_1"},{"fieldPath":"metadata.annotations[''task-annotation-2'']","name":"POD_TASK_ANNOTATION_2"}],"podMetadata":{"annotations":{"task-annotation-1":"annotation-1","task-annotation-2":"annotation-2"}}}' - - name: components-0fe8b44e6cf954e8e447976b0b620a7737c5a0371bfd14ca4329c665328fa1d6 + - name: components-08b02a2c7b00c133f7d053207ab1e549fe9374102122ca84dcd6f5a15683fe01 value: '{"executorLabel":"exec-validate-pod-metadata-task-c","inputDefinitions":{"parameters":{"annotation_exp_val_1":{"parameterType":"STRING"},"annotation_exp_val_2":{"parameterType":"STRING"},"annotation_path_1":{"parameterType":"STRING"},"annotation_path_2":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-0fe8b44e6cf954e8e447976b0b620a7737c5a0371bfd14ca4329c665328fa1d6 + - name: implementations-08b02a2c7b00c133f7d053207ab1e549fe9374102122ca84dcd6f5a15683fe01 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_pod_metadata_task_c"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -114,12 +114,12 @@ spec: annotation is {annotation_val_1} but is supposed to be {annotation_exp_val_1}.''\n )\n\n annotation_val_2 = os.getenv(annotation_path_2)\n if annotation_val_2 != annotation_exp_val_2:\n raise ValueError(\n f''Pod annotation is {annotation_val_2} but is supposed - to be {annotation_exp_val_2}.''\n )\n return True\n\n"],"image":"python:3.9"}' + to be {annotation_exp_val_2}.''\n )\n return True\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-validate-pod-metadata-task-d value: '{"fieldPathAsEnv":[{"fieldPath":"metadata.labels[''task-label-1'']","name":"POD_TASK_LABEL_1"},{"fieldPath":"metadata.labels[''task-label-2'']","name":"POD_TASK_LABEL_2"},{"fieldPath":"metadata.labels[''task-label-3'']","name":"POD_TASK_LABEL_3"}],"podMetadata":{"labels":{"task-label-1":"label-1","task-label-2":"label-2","task-label-3":"label-3"}}}' - - name: components-9ed49be6257bd027ea2d0882c606eddc7d5d2e83c82c63774ddd209b876bc316 + - name: components-d56a2097f634e26b0381658dc361347f393e19552ad2c43e9b6b0b6606678502 value: '{"executorLabel":"exec-validate-pod-metadata-task-d","inputDefinitions":{"parameters":{"label_exp_val_1":{"parameterType":"STRING"},"label_exp_val_2":{"parameterType":"STRING"},"label_exp_val_3":{"parameterType":"STRING"},"label_path_1":{"parameterType":"STRING"},"label_path_2":{"parameterType":"STRING"},"label_path_3":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}' - - name: implementations-9ed49be6257bd027ea2d0882c606eddc7d5d2e83c82c63774ddd209b876bc316 + - name: implementations-d56a2097f634e26b0381658dc361347f393e19552ad2c43e9b6b0b6606678502 value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_pod_metadata_task_d"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -140,7 +140,7 @@ spec: be {label_exp_val_2}.''\n )\n label_val_3 = os.getenv(label_path_3)\n if label_val_3 != label_exp_val_3:\n raise ValueError(\n f''Pod label is {label_val_3} but is supposed to be {label_exp_val_3}.''\n )\n return - True\n\n"],"image":"python:3.9"}' + True\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"validate-no-pod-metadata":{"cachingOptions":{},"componentRef":{"name":"comp-validate-no-pod-metadata"},"taskInfo":{"name":"validate-no-pod-metadata"}},"validate-pod-metadata-task-a":{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-a"},"inputs":{"parameters":{"annotation_exp_val":{"runtimeValue":{"constant":"annotation"}},"annotation_path":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION"}},"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}}}},"taskInfo":{"name":"validate-pod-metadata-task-a"}},"validate-pod-metadata-task-b":{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-b"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation-1"}},"annotation_exp_val_2":{"runtimeValue":{"constant":"annotation-2"}},"annotation_exp_val_3":{"runtimeValue":{"constant":"annotation-3"}},"annotation_exp_val_4":{"runtimeValue":{"constant":"annotation-4"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_1"}},"annotation_path_2":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_2"}},"annotation_path_3":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_3"}},"annotation_path_4":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_4"}},"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_exp_val_3":{"runtimeValue":{"constant":"label-3"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}},"label_path_3":{"runtimeValue":{"constant":"POD_TASK_LABEL_3"}}}},"taskInfo":{"name":"validate-pod-metadata-task-b"}},"validate-pod-metadata-task-c":{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-c"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation-1"}},"annotation_exp_val_2":{"runtimeValue":{"constant":"annotation-2"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_1"}},"annotation_path_2":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_2"}}}},"taskInfo":{"name":"validate-pod-metadata-task-c"}},"validate-pod-metadata-task-d":{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-d"},"inputs":{"parameters":{"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_exp_val_3":{"runtimeValue":{"constant":"label-3"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}},"label_path_3":{"runtimeValue":{"constant":"POD_TASK_LABEL_3"}}}},"taskInfo":{"name":"validate-pod-metadata-task-d"}}}}}' entrypoint: entrypoint @@ -824,11 +824,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-3d278faa61c986d801a417f587e50005053b72d5108dc720e6ce32593c050c24}}' + value: '{{workflow.parameters.components-3e1061118c8caaf7fb2580deca4c26c68777ea648895af667b433198f8d95a35}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-no-pod-metadata"},"taskInfo":{"name":"validate-no-pod-metadata"}}' - name: container - value: '{{workflow.parameters.implementations-3d278faa61c986d801a417f587e50005053b72d5108dc720e6ce32593c050c24}}' + value: '{{workflow.parameters.implementations-3e1061118c8caaf7fb2580deca4c26c68777ea648895af667b433198f8d95a35}}' - name: task-name value: validate-no-pod-metadata - name: parent-dag-id @@ -850,11 +850,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a58a8ba91674b53e9cfd2a56b122fa32c32733648c4337cfd4d489147baa0351}}' + value: '{{workflow.parameters.components-36b4f5715a9e847c587a42d8970df700b9d7977c0097b0b0a12b7be3bc7cb4f8}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-a"},"inputs":{"parameters":{"annotation_exp_val":{"runtimeValue":{"constant":"annotation"}},"annotation_path":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION"}},"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}}}},"taskInfo":{"name":"validate-pod-metadata-task-a"}}' - name: container - value: '{{workflow.parameters.implementations-a58a8ba91674b53e9cfd2a56b122fa32c32733648c4337cfd4d489147baa0351}}' + value: '{{workflow.parameters.implementations-36b4f5715a9e847c587a42d8970df700b9d7977c0097b0b0a12b7be3bc7cb4f8}}' - name: task-name value: validate-pod-metadata-task-a - name: parent-dag-id @@ -888,11 +888,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ff4adfd20fbb1ccd7f6e9330882576b7e34ffba16bc568acc10956b8652c28d9}}' + value: '{{workflow.parameters.components-e07ac8a27d331c96ac2d5559c43f483a79d4e8956b1941d0e306aee450ad83f1}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-b"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation-1"}},"annotation_exp_val_2":{"runtimeValue":{"constant":"annotation-2"}},"annotation_exp_val_3":{"runtimeValue":{"constant":"annotation-3"}},"annotation_exp_val_4":{"runtimeValue":{"constant":"annotation-4"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_1"}},"annotation_path_2":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_2"}},"annotation_path_3":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_3"}},"annotation_path_4":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_4"}},"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_exp_val_3":{"runtimeValue":{"constant":"label-3"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}},"label_path_3":{"runtimeValue":{"constant":"POD_TASK_LABEL_3"}}}},"taskInfo":{"name":"validate-pod-metadata-task-b"}}' - name: container - value: '{{workflow.parameters.implementations-ff4adfd20fbb1ccd7f6e9330882576b7e34ffba16bc568acc10956b8652c28d9}}' + value: '{{workflow.parameters.implementations-e07ac8a27d331c96ac2d5559c43f483a79d4e8956b1941d0e306aee450ad83f1}}' - name: task-name value: validate-pod-metadata-task-b - name: parent-dag-id @@ -942,11 +942,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0fe8b44e6cf954e8e447976b0b620a7737c5a0371bfd14ca4329c665328fa1d6}}' + value: '{{workflow.parameters.components-08b02a2c7b00c133f7d053207ab1e549fe9374102122ca84dcd6f5a15683fe01}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-c"},"inputs":{"parameters":{"annotation_exp_val_1":{"runtimeValue":{"constant":"annotation-1"}},"annotation_exp_val_2":{"runtimeValue":{"constant":"annotation-2"}},"annotation_path_1":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_1"}},"annotation_path_2":{"runtimeValue":{"constant":"POD_TASK_ANNOTATION_2"}}}},"taskInfo":{"name":"validate-pod-metadata-task-c"}}' - name: container - value: '{{workflow.parameters.implementations-0fe8b44e6cf954e8e447976b0b620a7737c5a0371bfd14ca4329c665328fa1d6}}' + value: '{{workflow.parameters.implementations-08b02a2c7b00c133f7d053207ab1e549fe9374102122ca84dcd6f5a15683fe01}}' - name: task-name value: validate-pod-metadata-task-c - name: parent-dag-id @@ -976,11 +976,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-9ed49be6257bd027ea2d0882c606eddc7d5d2e83c82c63774ddd209b876bc316}}' + value: '{{workflow.parameters.components-d56a2097f634e26b0381658dc361347f393e19552ad2c43e9b6b0b6606678502}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-pod-metadata-task-d"},"inputs":{"parameters":{"label_exp_val_1":{"runtimeValue":{"constant":"label-1"}},"label_exp_val_2":{"runtimeValue":{"constant":"label-2"}},"label_exp_val_3":{"runtimeValue":{"constant":"label-3"}},"label_path_1":{"runtimeValue":{"constant":"POD_TASK_LABEL_1"}},"label_path_2":{"runtimeValue":{"constant":"POD_TASK_LABEL_2"}},"label_path_3":{"runtimeValue":{"constant":"POD_TASK_LABEL_3"}}}},"taskInfo":{"name":"validate-pod-metadata-task-d"}}' - name: container - value: '{{workflow.parameters.implementations-9ed49be6257bd027ea2d0882c606eddc7d5d2e83c82c63774ddd209b876bc316}}' + value: '{{workflow.parameters.implementations-d56a2097f634e26b0381658dc361347f393e19552ad2c43e9b6b0b6606678502}}' - name: task-name value: validate-pod-metadata-task-d - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 5d1680190f6..d8fe0a9855f 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-134a4a180bb4bfeacb7a318916bac3b02f8ba93e7aca4257373412cf9e37ca3b + - name: components-2815fa01c723249ff7034b9449ad45f7da5d129882c5d188191c2e30f2847773 value: '{"executorLabel":"exec-add","inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_DOUBLE"},"b":{"parameterType":"NUMBER_DOUBLE"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_DOUBLE"}}}}' - - name: implementations-134a4a180bb4bfeacb7a318916bac3b02f8ba93e7aca4257373412cf9e37ca3b + - name: implementations-2815fa01c723249ff7034b9449ad45f7da5d129882c5d188191c2e30f2847773 value: '{"args":["--executor_input","{{$}}","--function_to_execute","add"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add(a: float, b: float) -\u003e float:\n return a + b\n\n"],"image":"python:3.9"}' + add(a: float, b: float) -\u003e float:\n return a + b\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"add":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"componentInputParameter":"b"}}},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":3},"taskInfo":{"name":"add"}}}},"inputDefinitions":{"parameters":{"a":{"defaultValue":1,"isOptional":true,"parameterType":"NUMBER_DOUBLE"},"b":{"defaultValue":7,"isOptional":true,"parameterType":"NUMBER_DOUBLE"}}}}' entrypoint: entrypoint @@ -227,11 +227,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-134a4a180bb4bfeacb7a318916bac3b02f8ba93e7aca4257373412cf9e37ca3b}}' + value: '{{workflow.parameters.components-2815fa01c723249ff7034b9449ad45f7da5d129882c5d188191c2e30f2847773}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"componentInputParameter":"b"}}},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":3},"taskInfo":{"name":"add"}}' - name: container - value: '{{workflow.parameters.implementations-134a4a180bb4bfeacb7a318916bac3b02f8ba93e7aca4257373412cf9e37ca3b}}' + value: '{{workflow.parameters.implementations-2815fa01c723249ff7034b9449ad45f7da5d129882c5d188191c2e30f2847773}}' - name: task-name value: add - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 8efdd9f5fb5..1c9d0e640a2 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496 + - name: components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e value: '{"executorLabel":"exec-add-numbers","inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_INTEGER"},"b":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496 + - name: implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e value: '{"args":["--executor_input","{{$}}","--function_to_execute","add_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - add_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.9"}' + add_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"add-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"runtimeValue":{"constant":3}}}},"taskInfo":{"name":"add-numbers"}},"add-numbers-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers-2"},"dependentTasks":["add-numbers"],"inputs":{"parameters":{"a":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers"}},"b":{"componentInputParameter":"b"}}},"taskInfo":{"name":"add-numbers-2"}},"add-numbers-3":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers-3"},"dependentTasks":["add-numbers-2"],"inputs":{"parameters":{"a":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers-2"}},"b":{"runtimeValue":{"constant":7}}}},"taskInfo":{"name":"add-numbers-3"}}}},"inputDefinitions":{"parameters":{"a":{"defaultValue":2,"isOptional":true,"parameterType":"NUMBER_INTEGER"},"b":{"defaultValue":5,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers"},"inputs":{"parameters":{"a":{"componentInputParameter":"a"},"b":{"runtimeValue":{"constant":3}}}},"taskInfo":{"name":"add-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task-name value: add-numbers - name: parent-dag-id @@ -225,11 +225,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers-2"},"dependentTasks":["add-numbers"],"inputs":{"parameters":{"a":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers"}},"b":{"componentInputParameter":"b"}}},"taskInfo":{"name":"add-numbers-2"}}' - name: container - value: '{{workflow.parameters.implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task-name value: add-numbers-2 - name: parent-dag-id @@ -250,11 +250,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.components-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-add-numbers-3"},"dependentTasks":["add-numbers-2"],"inputs":{"parameters":{"a":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"add-numbers-2"}},"b":{"runtimeValue":{"constant":7}}}},"taskInfo":{"name":"add-numbers-3"}}' - name: container - value: '{{workflow.parameters.implementations-97747bd2a90b59d576dc436e2ecaa2956d4e20b1642cea8581b61483a4f73496}}' + value: '{{workflow.parameters.implementations-a06e49d55601ed1e78432721afac56c959e05f0346dc650f749ccab33c0e425e}}' - name: task-name value: add-numbers-3 - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index f8bb71ead52..97ff856c299 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"secretAsEnv":[{"keyToEnv":[{"envVar":"USER_NAME","secretKey":"username"}],"optional":false,"secretNameParameter":{"componentInputParameter":"secret_parm"}},{"keyToEnv":[{"envVar":"PASSWORD_VAR1","secretKey":"password"}],"optional":false,"secretName":"test-secret-2","secretNameParameter":{"runtimeValue":{"constant":"test-secret-2"}}},{"keyToEnv":[{"envVar":"PASSWORD_VAR2","secretKey":"password"}],"optional":false,"secretNameParameter":{"taskOutputParameter":{"outputParameterKey":"some_output","producerTask":"generate-secret-name"}}}]}' - - name: components-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab + - name: components-f5f5569b7b865a5b2950a75dfc385075059e8dc453c48f01c473f3b01f0a4027 value: '{"executorLabel":"exec-comp"}' - - name: implementations-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab + - name: implementations-f5f5569b7b865a5b2950a75dfc385075059e8dc453c48f01c473f3b01f0a4027 value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -23,10 +23,10 @@ spec: comp():\n import os\n username = os.getenv(\"USER_NAME\", \"\")\n psw1 = os.getenv(\"PASSWORD_VAR1\", \"\")\n psw2 = os.getenv(\"PASSWORD_VAR2\", \"\")\n assert username == \"user1\"\n assert psw1 == \"psw1\"\n assert - psw2 == \"psw2\"\n\n"],"image":"python:3.9"}' - - name: components-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c + psw2 == \"psw2\"\n\n"],"image":"python:3.11"}' + - name: components-a6f0a699a84a6a4255ba12971522f2e31ce71807004b6bd0c8b832f750351283 value: '{"executorLabel":"exec-generate-secret-name","outputDefinitions":{"parameters":{"some_output":{"parameterType":"STRING"}}}}' - - name: implementations-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c + - name: implementations-a6f0a699a84a6a4255ba12971522f2e31ce71807004b6bd0c8b832f750351283 value: '{"args":["--executor_input","{{$}}","--function_to_execute","generate_secret_name"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -37,7 +37,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef generate_secret_name(some_output: OutputPath(str)):\n secret_name = \"test-secret-3\"\n with - open(some_output, ''w'') as f:\n f.write(secret_name)\n\n"],"image":"python:3.9"}' + open(some_output, ''w'') as f:\n f.write(secret_name)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["generate-secret-name"],"taskInfo":{"name":"comp"}},"generate-secret-name":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-generate-secret-name"},"taskInfo":{"name":"generate-secret-name"}}}},"inputDefinitions":{"parameters":{"secret_parm":{"defaultValue":"test-secret-1","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -220,11 +220,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' + value: '{{workflow.parameters.components-f5f5569b7b865a5b2950a75dfc385075059e8dc453c48f01c473f3b01f0a4027}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["generate-secret-name"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' + value: '{{workflow.parameters.implementations-f5f5569b7b865a5b2950a75dfc385075059e8dc453c48f01c473f3b01f0a4027}}' - name: task-name value: comp - name: parent-dag-id @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' + value: '{{workflow.parameters.components-a6f0a699a84a6a4255ba12971522f2e31ce71807004b6bd0c8b832f750351283}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-generate-secret-name"},"taskInfo":{"name":"generate-secret-name"}}' - name: container - value: '{{workflow.parameters.implementations-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' + value: '{{workflow.parameters.implementations-a6f0a699a84a6a4255ba12971522f2e31ce71807004b6bd0c8b832f750351283}}' - name: task-name value: generate-secret-name - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index 0f25aa648bb..60930b69d2d 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"secretAsVolume":[{"mountPath":"/mnt/my_vol","optional":false,"secretNameParameter":{"componentInputParameter":"secret_param"}}]}' - - name: components-cc440df32ad452fe9f1028b109b0e39809977643b2f7a2e744913ac9b1714074 + - name: components-552dd41fa2418575aa5ae289a30234ba1e45c44408832759059a616eeb2327ba value: '{"executorLabel":"exec-comp"}' - - name: implementations-cc440df32ad452fe9f1028b109b0e39809977643b2f7a2e744913ac9b1714074 + - name: implementations-552dd41fa2418575aa5ae289a30234ba1e45c44408832759059a616eeb2327ba value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -25,7 +25,7 @@ spec: Exception(''Secret not found'')\n\n # Open the secret\n with open(username_path, ''rb'') as secret_file:\n username = secret_file.read()\n\n # Decode the secret\n username = username.decode(''utf-8'')\n\n # Print the secret\n print(f\"username: - {username}\")\n assert username == \"user1\"\n\n"],"image":"python:3.9"}' + {username}\")\n assert username == \"user1\"\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}}},"inputDefinitions":{"parameters":{"secret_param":{"defaultValue":"test-secret-1","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -208,11 +208,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-cc440df32ad452fe9f1028b109b0e39809977643b2f7a2e744913ac9b1714074}}' + value: '{{workflow.parameters.components-552dd41fa2418575aa5ae289a30234ba1e45c44408832759059a616eeb2327ba}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-cc440df32ad452fe9f1028b109b0e39809977643b2f7a2e744913ac9b1714074}}' + value: '{{workflow.parameters.implementations-552dd41fa2418575aa5ae289a30234ba1e45c44408832759059a616eeb2327ba}}' - name: task-name value: comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_semaphore.yaml b/test_data/compiled-workflows/pipeline_with_semaphore.yaml index ebd7b82fa78..a71ba7ab29d 100644 --- a/test_data/compiled-workflows/pipeline_with_semaphore.yaml +++ b/test_data/compiled-workflows/pipeline_with_semaphore.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-539df5f72ca5a96ea4048c58d4da2d2582e1354734df9db466b2772e959f4b59 + - name: components-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2 value: '{"executorLabel":"exec-comp"}' - - name: implementations-539df5f72ca5a96ea4048c58d4da2d2582e1354734df9db466b2772e959f4b59 + - name: implementations-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - comp():\n pass\n\n"],"image":"python:3.9"}' + comp():\n pass\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-539df5f72ca5a96ea4048c58d4da2d2582e1354734df9db466b2772e959f4b59}}' + value: '{{workflow.parameters.components-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-539df5f72ca5a96ea4048c58d4da2d2582e1354734df9db466b2772e959f4b59}}' + value: '{{workflow.parameters.implementations-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2}}' - name: task-name value: comp - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 7f3f07d4d76..cd416d5126c 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-5c76ebcd4e091481fc4c07f0360039dad12c412bc1b099337a4f415a0888be3d + - name: components-ac7285064d7d99b0f19ca3e34669996b38e3e05eda2a6b78e0b3463b1a2a6186 value: '{"executorLabel":"exec-sum-numbers","inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_INTEGER"},"b":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-5c76ebcd4e091481fc4c07f0360039dad12c412bc1b099337a4f415a0888be3d + - name: implementations-ac7285064d7d99b0f19ca3e34669996b38e3e05eda2a6b78e0b3463b1a2a6186 value: '{"args":["--executor_input","{{$}}","--function_to_execute","sum_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,7 +18,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.9","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}"}}' + sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}"}}' - name: components-root value: '{"dag":{"tasks":{"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}"}},"pipelinechannel--accelerator_limit":{"componentInputParameter":"accelerator_limit"},"pipelinechannel--accelerator_type":{"componentInputParameter":"accelerator_type"},"pipelinechannel--cpu_limit":{"componentInputParameter":"cpu_limit"},"pipelinechannel--memory_limit":{"componentInputParameter":"memory_limit"}}},"taskInfo":{"name":"sum-numbers"}}}},"inputDefinitions":{"parameters":{"accelerator_limit":{"defaultValue":"1","isOptional":true,"parameterType":"STRING"},"accelerator_type":{"defaultValue":"NVIDIA_TESLA_P4","isOptional":true,"parameterType":"STRING"},"cpu_limit":{"defaultValue":"4000m","isOptional":true,"parameterType":"STRING"},"memory_limit":{"defaultValue":"15G","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -201,11 +201,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5c76ebcd4e091481fc4c07f0360039dad12c412bc1b099337a4f415a0888be3d}}' + value: '{{workflow.parameters.components-ac7285064d7d99b0f19ca3e34669996b38e3e05eda2a6b78e0b3463b1a2a6186}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator_type'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu_limit'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory_limit'']}}"}},"pipelinechannel--accelerator_limit":{"componentInputParameter":"accelerator_limit"},"pipelinechannel--accelerator_type":{"componentInputParameter":"accelerator_type"},"pipelinechannel--cpu_limit":{"componentInputParameter":"cpu_limit"},"pipelinechannel--memory_limit":{"componentInputParameter":"memory_limit"}}},"taskInfo":{"name":"sum-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-5c76ebcd4e091481fc4c07f0360039dad12c412bc1b099337a4f415a0888be3d}}' + value: '{{workflow.parameters.implementations-ac7285064d7d99b0f19ca3e34669996b38e3e05eda2a6b78e0b3463b1a2a6186}}' - name: task-name value: sum-numbers - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 0da72091594..5efc8f18da4 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-5ce77d07cd95a5f71e354d597878e3e0bc89527d3d81450cd270ed6183594740 + - name: components-12837483d1ac7aafcef7d75cb8553ad734a85089a7511ff009cbace2b8f18312 value: '{"executorLabel":"exec-accelerator-limit","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-5ce77d07cd95a5f71e354d597878e3e0bc89527d3d81450cd270ed6183594740 + - name: implementations-12837483d1ac7aafcef7d75cb8553ad734a85089a7511ff009cbace2b8f18312 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_limit"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_limit() -\u003e str:\n return ''1''\n\n"],"image":"python:3.9"}' - - name: components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + accelerator_limit() -\u003e str:\n return ''1''\n\n"],"image":"python:3.11"}' + - name: components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"executorLabel":"exec-accelerator-type","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b + - name: implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236 value: '{"args":["--executor_input","{{$}}","--function_to_execute","accelerator_type"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,10 +31,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.9"}' - - name: components-a78005c8cf24945a5b595201f36dc4905feff000fc7a59dc0aabaa7c0dc1413b + accelerator_type() -\u003e str:\n return ''NVIDIA_TESLA_P4''\n\n"],"image":"python:3.11"}' + - name: components-3e42152114e50a3fa222dc0a74cb1a91431ac56c168a1fc6a6c12749d1c24939 value: '{"executorLabel":"exec-cpu-limit","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-a78005c8cf24945a5b595201f36dc4905feff000fc7a59dc0aabaa7c0dc1413b + - name: implementations-3e42152114e50a3fa222dc0a74cb1a91431ac56c168a1fc6a6c12749d1c24939 value: '{"args":["--executor_input","{{$}}","--function_to_execute","cpu_limit"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -44,10 +44,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - cpu_limit() -\u003e str:\n return ''4000m''\n\n"],"image":"python:3.9"}' - - name: components-a3d64b3ea9d79d1b1dc471faada5215bf21142914244bbfd26fa4aec93a232d7 + cpu_limit() -\u003e str:\n return ''4000m''\n\n"],"image":"python:3.11"}' + - name: components-85f92f571187f62f60b0032a62bbab38678ebb5c899ba184021054c62fdb8a1c value: '{"executorLabel":"exec-memory-limit","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-a3d64b3ea9d79d1b1dc471faada5215bf21142914244bbfd26fa4aec93a232d7 + - name: implementations-85f92f571187f62f60b0032a62bbab38678ebb5c899ba184021054c62fdb8a1c value: '{"args":["--executor_input","{{$}}","--function_to_execute","memory_limit"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -57,10 +57,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - memory_limit() -\u003e str:\n return ''15G''\n\n"],"image":"python:3.9"}' - - name: components-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309 + memory_limit() -\u003e str:\n return ''15G''\n\n"],"image":"python:3.11"}' + - name: components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b value: '{"executorLabel":"exec-sum-numbers","inputDefinitions":{"parameters":{"a":{"parameterType":"NUMBER_INTEGER"},"b":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"NUMBER_INTEGER"}}}}' - - name: implementations-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309 + - name: implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b value: '{"args":["--executor_input","{{$}}","--function_to_execute","sum_numbers"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -70,9 +70,9 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.9","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' + sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-limit","accelerator-type","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -253,11 +253,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5ce77d07cd95a5f71e354d597878e3e0bc89527d3d81450cd270ed6183594740}}' + value: '{{workflow.parameters.components-12837483d1ac7aafcef7d75cb8553ad734a85089a7511ff009cbace2b8f18312}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}}' - name: container - value: '{{workflow.parameters.implementations-5ce77d07cd95a5f71e354d597878e3e0bc89527d3d81450cd270ed6183594740}}' + value: '{{workflow.parameters.implementations-12837483d1ac7aafcef7d75cb8553ad734a85089a7511ff009cbace2b8f18312}}' - name: task-name value: accelerator-limit - name: parent-dag-id @@ -277,11 +277,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.components-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}}' - name: container - value: '{{workflow.parameters.implementations-2212d778fdcf0682a268c7b100e165b811b818a3b68c0bfd48dd5681e999936b}}' + value: '{{workflow.parameters.implementations-f9a25b0b0eb8f4240e030d4007d8ba04dd922f7ac266816c8751d15ad42c5236}}' - name: task-name value: accelerator-type - name: parent-dag-id @@ -301,11 +301,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a78005c8cf24945a5b595201f36dc4905feff000fc7a59dc0aabaa7c0dc1413b}}' + value: '{{workflow.parameters.components-3e42152114e50a3fa222dc0a74cb1a91431ac56c168a1fc6a6c12749d1c24939}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}}' - name: container - value: '{{workflow.parameters.implementations-a78005c8cf24945a5b595201f36dc4905feff000fc7a59dc0aabaa7c0dc1413b}}' + value: '{{workflow.parameters.implementations-3e42152114e50a3fa222dc0a74cb1a91431ac56c168a1fc6a6c12749d1c24939}}' - name: task-name value: cpu-limit - name: parent-dag-id @@ -325,11 +325,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-a3d64b3ea9d79d1b1dc471faada5215bf21142914244bbfd26fa4aec93a232d7}}' + value: '{{workflow.parameters.components-85f92f571187f62f60b0032a62bbab38678ebb5c899ba184021054c62fdb8a1c}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}}' - name: container - value: '{{workflow.parameters.implementations-a3d64b3ea9d79d1b1dc471faada5215bf21142914244bbfd26fa4aec93a232d7}}' + value: '{{workflow.parameters.implementations-85f92f571187f62f60b0032a62bbab38678ebb5c899ba184021054c62fdb8a1c}}' - name: task-name value: memory-limit - name: parent-dag-id @@ -349,17 +349,17 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309}}' + value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-limit","accelerator-type","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container - value: '{{workflow.parameters.implementations-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309}}' + value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: memory-limit.Succeeded && accelerator-limit.Succeeded && accelerator-type.Succeeded - && cpu-limit.Succeeded + depends: accelerator-limit.Succeeded && cpu-limit.Succeeded && accelerator-type.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index d8289664744..9d7cde234ba 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-ee01610b5099d6bfa9c7d87fa1c59efff070025700504b54cf163c39ef70648b + - name: components-f7d7acf968eee71dd35f342b2defffecd0b666d98ba3f84067666f01761ce844 value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-ee01610b5099d6bfa9c7d87fa1c59efff070025700504b54cf163c39ef70648b + - name: implementations-f7d7acf968eee71dd35f342b2defffecd0b666d98ba3f84067666f01761ce844 value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -18,10 +18,10 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.9"}' - - name: components-e867d2e58d704ce4a73fa5b50e0068c7b54a23f058c93b8dc7d73a854b7786a2 + fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n\n"],"image":"python:3.11"}' + - name: components-092cff624d6cd885e152ba4746a8171a94bbfad01e209a7d23d4499a65520f8d value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-e867d2e58d704ce4a73fa5b50e0068c7b54a23f058c93b8dc7d73a854b7786a2 + - name: implementations-092cff624d6cd885e152ba4746a8171a94bbfad01e209a7d23d4499a65520f8d value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -31,13 +31,13 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-comp-exit-handler-1 value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--message":{"parameterType":"STRING"}}}}' - - name: components-0920327c9e00f705cfd5d7191535f63964abc5d4ca6fb04e69052c264d8ed9a4 + - name: components-8c1d358f7eb020b42803091b690668b1d8107ef638cd1bb1ff074c364f18639f value: '{"executorLabel":"exec-exit-op","inputDefinitions":{"parameters":{"status":{"isOptional":true,"parameterType":"TASK_FINAL_STATUS"},"user_input":{"parameterType":"STRING"}}}}' - - name: implementations-0920327c9e00f705cfd5d7191535f63964abc5d4ca6fb04e69052c264d8ed9a4 + - name: implementations-8c1d358f7eb020b42803091b690668b1d8107ef638cd1bb1ff074c364f18639f value: '{"args":["--executor_input","{{$}}","--function_to_execute","exit_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -51,7 +51,7 @@ spec: pipeline run status.\"\"\"\n print(''Pipeline status: '', status.state)\n print(''Job resource name: '', status.pipeline_job_resource_name)\n print(''Pipeline task name: '', status.pipeline_task_name)\n print(''Error code: '', status.error_code)\n print(''Error - message: '', status.error_message)\n\n"],"image":"python:3.9"}' + message: '', status.error_message)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"my-pipeline"}},"exit-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-exit-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}},"user_input":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}}},"inputDefinitions":{"parameters":{"message":{"defaultValue":"Hello World!","isOptional":true,"parameterType":"STRING"}}}}' @@ -235,12 +235,12 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ee01610b5099d6bfa9c7d87fa1c59efff070025700504b54cf163c39ef70648b}}' + value: '{{workflow.parameters.components-f7d7acf968eee71dd35f342b2defffecd0b666d98ba3f84067666f01761ce844}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-ee01610b5099d6bfa9c7d87fa1c59efff070025700504b54cf163c39ef70648b}}' + value: '{{workflow.parameters.implementations-f7d7acf968eee71dd35f342b2defffecd0b666d98ba3f84067666f01761ce844}}' - name: task-name value: fail-op - name: parent-dag-id @@ -260,11 +260,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e867d2e58d704ce4a73fa5b50e0068c7b54a23f058c93b8dc7d73a854b7786a2}}' + value: '{{workflow.parameters.components-092cff624d6cd885e152ba4746a8171a94bbfad01e209a7d23d4499a65520f8d}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op"}}' - name: container - value: '{{workflow.parameters.implementations-e867d2e58d704ce4a73fa5b50e0068c7b54a23f058c93b8dc7d73a854b7786a2}}' + value: '{{workflow.parameters.implementations-092cff624d6cd885e152ba4746a8171a94bbfad01e209a7d23d4499a65520f8d}}' - name: task-name value: print-op - name: parent-dag-id @@ -292,11 +292,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-0920327c9e00f705cfd5d7191535f63964abc5d4ca6fb04e69052c264d8ed9a4}}' + value: '{{workflow.parameters.components-8c1d358f7eb020b42803091b690668b1d8107ef638cd1bb1ff074c364f18639f}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-exit-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}},"user_input":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-0920327c9e00f705cfd5d7191535f63964abc5d4ca6fb04e69052c264d8ed9a4}}' + value: '{{workflow.parameters.implementations-8c1d358f7eb020b42803091b690668b1d8107ef638cd1bb1ff074c364f18639f}}' - name: task-name value: exit-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index d6b8a7c939a..f9a6c0307f5 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-59806fd7437e0169671c5386035d1c9e16abc59cf47bf73a4a88c9c72f9b89c5 + - name: components-efcc763bebb8f65a468ef9c3dd844fdcc92d0e17c005c84793244e2775cc61fe value: '{"executorLabel":"exec-fail-op","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-59806fd7437e0169671c5386035d1c9e16abc59cf47bf73a4a88c9c72f9b89c5 + - name: implementations-efcc763bebb8f65a468ef9c3dd844fdcc92d0e17c005c84793244e2775cc61fe value: '{"args":["--executor_input","{{$}}","--function_to_execute","fail_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -19,10 +19,10 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef fail_op(message: str) -\u003e str:\n \"\"\"Fails.\"\"\"\n import sys\n print(message)\n sys.exit(1)\n return - message\n\n"],"image":"python:3.9"}' - - name: components-b206be17c679a17aaa7a27bfbdc4c9282eb4a6dae575e3758a3af57a18424138 + message\n\n"],"image":"python:3.11"}' + - name: components-54651fb1cfb6bbf8377ab247bf3477ceefbfe476ff3687f7116d051f7f526f93 value: '{"executorLabel":"exec-print-op","inputDefinitions":{"parameters":{"message":{"defaultValue":"default","isOptional":true,"parameterType":"STRING"}}}}' - - name: implementations-b206be17c679a17aaa7a27bfbdc4c9282eb4a6dae575e3758a3af57a18424138 + - name: implementations-54651fb1cfb6bbf8377ab247bf3477ceefbfe476ff3687f7116d051f7f526f93 value: '{"args":["--executor_input","{{$}}","--function_to_execute","print_op"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -32,7 +32,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - print_op(message: str = ''default''):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + print_op(message: str = ''default''):\n \"\"\"Prints a message.\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"fail-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"sample_input"}}},"taskInfo":{"name":"fail-op"}},"print-op":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["fail-op"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"fail-op"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}}},"inputDefinitions":{"parameters":{"sample_input":{"defaultValue":"message","isOptional":true,"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -215,11 +215,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-b206be17c679a17aaa7a27bfbdc4c9282eb4a6dae575e3758a3af57a18424138}}' + value: '{{workflow.parameters.components-54651fb1cfb6bbf8377ab247bf3477ceefbfe476ff3687f7116d051f7f526f93}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["fail-op"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"fail-op"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - name: container - value: '{{workflow.parameters.implementations-b206be17c679a17aaa7a27bfbdc4c9282eb4a6dae575e3758a3af57a18424138}}' + value: '{{workflow.parameters.implementations-54651fb1cfb6bbf8377ab247bf3477ceefbfe476ff3687f7116d051f7f526f93}}' - name: task-name value: print-op - name: parent-dag-id @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-59806fd7437e0169671c5386035d1c9e16abc59cf47bf73a4a88c9c72f9b89c5}}' + value: '{{workflow.parameters.components-efcc763bebb8f65a468ef9c3dd844fdcc92d0e17c005c84793244e2775cc61fe}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"componentInputParameter":"sample_input"}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-59806fd7437e0169671c5386035d1c9e16abc59cf47bf73a4a88c9c72f9b89c5}}' + value: '{{workflow.parameters.implementations-efcc763bebb8f65a468ef9c3dd844fdcc92d0e17c005c84793244e2775cc61fe}}' - name: task-name value: fail-op - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index 38b6e0804f3..85a0ae926f3 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-257c3d2b996441cda9547359972f80c521a24efee58aff87d48282b55f7782bd + - name: components-f3dee78c44ac18676c4c12c69e34f993a82f0c5af39391208a5dcd57c54c24fc value: '{"executorLabel":"exec-echo","inputDefinitions":{"parameters":{"msg":{"parameterType":"STRING"}}}}' - - name: implementations-257c3d2b996441cda9547359972f80c521a24efee58aff87d48282b55f7782bd + - name: implementations-f3dee78c44ac18676c4c12c69e34f993a82f0c5af39391208a5dcd57c54c24fc value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,7 +21,7 @@ spec: util_func(msg: str) -\u003e str:\n return f\"{msg} from util_func\"\n\n\ndef util_func2(msg: str) -\u003e str:\n return f\"{msg} from util_func2\"\n\n\ndef echo(msg: str):\n assert util_func(msg) == f\"{msg} from util_func\"\n assert - util_func2(msg) == f\"{msg} from util_func2\"\n\n"],"image":"python:3.9"}' + util_func2(msg) == f\"{msg} from util_func2\"\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"echo"}}}},"inputDefinitions":{"parameters":{"msg":{"defaultValue":"Hello, World!","isOptional":true,"parameterType":"STRING"}}}}' @@ -205,11 +205,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-257c3d2b996441cda9547359972f80c521a24efee58aff87d48282b55f7782bd}}' + value: '{{workflow.parameters.components-f3dee78c44ac18676c4c12c69e34f993a82f0c5af39391208a5dcd57c54c24fc}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"echo"}}' - name: container - value: '{{workflow.parameters.implementations-257c3d2b996441cda9547359972f80c521a24efee58aff87d48282b55f7782bd}}' + value: '{{workflow.parameters.implementations-f3dee78c44ac18676c4c12c69e34f993a82f0c5af39391208a5dcd57c54c24fc}}' - name: task-name value: echo - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index 5b045ffe880..360d128735d 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-consumer value: '{"pvcMount":[{"mountPath":"/data","pvcNameParameter":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}},"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d + - name: components-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede value: '{"executorLabel":"exec-consumer","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d + - name: implementations-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede value: '{"args":["--executor_input","{{$}}","--function_to_execute","consumer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,7 +21,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef consumer() -\u003e str:\n with open(''/data/file.txt'', ''r'') as file:\n content - = file.read()\n print(content)\n return content\n\n"],"image":"python:3.9"}' + = file.read()\n print(content)\n return content\n\n"],"image":"python:3.11"}' - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"description":"AccessModes to request for the provisioned PVC. May\nbe one or more of ``''ReadWriteOnce''``, @@ -52,9 +52,9 @@ spec: value: '{"image":"argostub/deletepvc"}' - name: kubernetes-comp-producer value: '{"pvcMount":[{"mountPath":"/data","pvcNameParameter":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}},"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d + - name: components-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9 value: '{"executorLabel":"exec-producer","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d + - name: implementations-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9 value: '{"args":["--executor_input","{{$}}","--function_to_execute","producer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -66,7 +66,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef producer() -\u003e str:\n with open(''/data/file.txt'', ''w'') as file:\n file.write(''Hello world'')\n with open(''/data/file.txt'', ''r'') as file:\n content - = file.read()\n print(content)\n return content\n\n"],"image":"python:3.9"}' + = file.read()\n print(content)\n return content\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"consumer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["createpvc","producer"],"taskInfo":{"name":"consumer"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Mi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["consumer","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}},"producer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"dependentTasks":["createpvc"],"taskInfo":{"name":"producer"}}}}}' entrypoint: entrypoint @@ -249,11 +249,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d}}' + value: '{{workflow.parameters.components-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["createpvc","producer"],"taskInfo":{"name":"consumer"}}' - name: container - value: '{{workflow.parameters.implementations-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d}}' + value: '{{workflow.parameters.implementations-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede}}' - name: task-name value: consumer - name: parent-dag-id @@ -305,11 +305,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d}}' + value: '{{workflow.parameters.components-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"dependentTasks":["createpvc"],"taskInfo":{"name":"producer"}}' - name: container - value: '{{workflow.parameters.implementations-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d}}' + value: '{{workflow.parameters.implementations-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9}}' - name: task-name value: producer - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index faaea9d5add..fc2d74a1863 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-consumer value: '{"pvcMount":[{"mountPath":"/data","pvcNameParameter":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}},"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d + - name: components-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede value: '{"executorLabel":"exec-consumer","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d + - name: implementations-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede value: '{"args":["--executor_input","{{$}}","--function_to_execute","consumer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,7 +21,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef consumer() -\u003e str:\n with open(''/data/file.txt'', ''r'') as file:\n content - = file.read()\n print(content)\n return content\n\n"],"image":"python:3.9"}' + = file.read()\n print(content)\n return content\n\n"],"image":"python:3.11"}' - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"description":"AccessModes to request for the provisioned PVC. May\nbe one or more of ``''ReadWriteOnce''``, @@ -52,9 +52,9 @@ spec: value: '{"image":"argostub/deletepvc"}' - name: kubernetes-comp-producer value: '{"pvcMount":[{"mountPath":"/data","pvcNameParameter":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}},"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d + - name: components-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9 value: '{"executorLabel":"exec-producer","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d + - name: implementations-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9 value: '{"args":["--executor_input","{{$}}","--function_to_execute","producer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -66,7 +66,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef producer() -\u003e str:\n with open(''/data/file.txt'', ''w'') as file:\n file.write(''Hello world'')\n with open(''/data/file.txt'', ''r'') as file:\n content - = file.read()\n print(content)\n return content\n\n"],"image":"python:3.9"}' + = file.read()\n print(content)\n return content\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"consumer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["createpvc","producer"],"taskInfo":{"name":"consumer"}},"createpvc":{"cachingOptions":{},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Mi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["consumer","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}},"producer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"dependentTasks":["createpvc"],"taskInfo":{"name":"producer"}}}}}' entrypoint: entrypoint @@ -249,11 +249,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d}}' + value: '{{workflow.parameters.components-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["createpvc","producer"],"taskInfo":{"name":"consumer"}}' - name: container - value: '{{workflow.parameters.implementations-522286ff8480712f50be32c25dedca8517bb73693bea29170288afdd11255e1d}}' + value: '{{workflow.parameters.implementations-d7fb91ca6b78edad062c1f44d01d414280f5740b5260436244e8b7638538eede}}' - name: task-name value: consumer - name: parent-dag-id @@ -305,11 +305,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d}}' + value: '{{workflow.parameters.components-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"dependentTasks":["createpvc"],"taskInfo":{"name":"producer"}}' - name: container - value: '{{workflow.parameters.implementations-c83a4f299993efdf1e9a883b7d05cb035219013b529f03866c0be2efd4da651d}}' + value: '{{workflow.parameters.implementations-35323c00ea64cb236ca34272f55bdb938f396af10887a294134939d9819557b9}}' - name: task-name value: producer - name: parent-dag-id diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 289d5ff32d1..88260ac3b87 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-72f73bfd1c06ca202f77c1bc36f3e0e5fba88ef0bca17fb76a71700051b8ac63 + - name: components-099322c2301dec986b734828a313953d589b6759b84e2204d3be55050a87abf3 value: '{"executorLabel":"exec-read-from-workspace","inputDefinitions":{"parameters":{"file_path":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-72f73bfd1c06ca202f77c1bc36f3e0e5fba88ef0bca17fb76a71700051b8ac63 + - name: implementations-099322c2301dec986b734828a313953d589b6759b84e2204d3be55050a87abf3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","read_from_workspace"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -24,10 +24,10 @@ spec: = f.read()\n print(f\"Read content from: {file_path}\")\n print(f\"Content: {content}\")\n assert content == \"Hello from workspace!\"\n return content\n else:\n print(f\"File not found at: {file_path}\")\n return - \"File not found\"\n\n"],"image":"python:3.9"}' - - name: components-64aebdd242571626248d5281bdedd9ce0fa8953ccf40f22c7d37ba51574f7e23 + \"File not found\"\n\n"],"image":"python:3.11"}' + - name: components-d94cb6155685654cc8749b6be9ebf3fb71fe13a85b91fd244acdd09265f42081 value: '{"executorLabel":"exec-write-to-workspace","inputDefinitions":{"parameters":{"workspace_path":{"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-64aebdd242571626248d5281bdedd9ce0fa8953ccf40f22c7d37ba51574f7e23 + - name: implementations-d94cb6155685654cc8749b6be9ebf3fb71fe13a85b91fd244acdd09265f42081 value: '{"args":["--executor_input","{{$}}","--function_to_execute","write_to_workspace"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -42,7 +42,7 @@ spec: = os.path.join(workspace_path, \"data\", \"test_file.txt\")\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n\n with open(file_path, \"w\") as f:\n f.write(\"Hello from workspace!\")\n\n print(f\"Wrote file to: {file_path}\")\n return - file_path\n\n"],"image":"python:3.9"}' + file_path\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"parameters":{"Output":{"valueFromParameter":{"outputParameterKey":"Output","producerSubtask":"read-from-workspace"}}}},"tasks":{"read-from-workspace":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-read-from-workspace"},"dependentTasks":["write-to-workspace"],"inputs":{"parameters":{"file_path":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"write-to-workspace"}}}},"taskInfo":{"name":"read-from-workspace"}},"write-to-workspace":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-write-to-workspace"},"inputs":{"parameters":{"workspace_path":{"runtimeValue":{"constant":"{{$.workspace_path}}"}}}},"taskInfo":{"name":"write-to-workspace"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -225,11 +225,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-72f73bfd1c06ca202f77c1bc36f3e0e5fba88ef0bca17fb76a71700051b8ac63}}' + value: '{{workflow.parameters.components-099322c2301dec986b734828a313953d589b6759b84e2204d3be55050a87abf3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-read-from-workspace"},"dependentTasks":["write-to-workspace"],"inputs":{"parameters":{"file_path":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"write-to-workspace"}}}},"taskInfo":{"name":"read-from-workspace"}}' - name: container - value: '{{workflow.parameters.implementations-72f73bfd1c06ca202f77c1bc36f3e0e5fba88ef0bca17fb76a71700051b8ac63}}' + value: '{{workflow.parameters.implementations-099322c2301dec986b734828a313953d589b6759b84e2204d3be55050a87abf3}}' - name: task-name value: read-from-workspace - name: parent-dag-id @@ -250,11 +250,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-64aebdd242571626248d5281bdedd9ce0fa8953ccf40f22c7d37ba51574f7e23}}' + value: '{{workflow.parameters.components-d94cb6155685654cc8749b6be9ebf3fb71fe13a85b91fd244acdd09265f42081}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-write-to-workspace"},"inputs":{"parameters":{"workspace_path":{"runtimeValue":{"constant":"{{$.workspace_path}}"}}}},"taskInfo":{"name":"write-to-workspace"}}' - name: container - value: '{{workflow.parameters.implementations-64aebdd242571626248d5281bdedd9ce0fa8953ccf40f22c7d37ba51574f7e23}}' + value: '{{workflow.parameters.implementations-d94cb6155685654cc8749b6be9ebf3fb71fe13a85b91fd244acdd09265f42081}}' - name: task-name value: write-to-workspace - name: parent-dag-id diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 250fba6d9d4..e6be2d4037b 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f + - name: components-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4 value: '{"executorLabel":"exec-preprocess","inputDefinitions":{"parameters":{"input_dict_parameter":{"parameterType":"STRUCT"},"input_list_parameter":{"parameterType":"LIST"},"message":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_dataset_one":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"output_dataset_two_path":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_bool_parameter_path":{"parameterType":"BOOLEAN"},"output_dict_parameter_path":{"parameterType":"STRUCT"},"output_list_parameter_path":{"parameterType":"LIST"},"output_parameter_path":{"parameterType":"STRING"}}}}' - - name: implementations-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f + - name: implementations-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4 value: '{"args":["--executor_input","{{$}}","--function_to_execute","preprocess"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -40,7 +40,7 @@ spec: open(output_bool_parameter_path, ''w'') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, ''w'') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with - open(output_list_parameter_path, ''w'') as f:\n f.write(json.dumps(input_list_parameter))\n\n"],"image":"python:3.9"}' + open(output_list_parameter_path, ''w'') as f:\n f.write(json.dumps(input_list_parameter))\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"output_dataset_one":{"artifactSelectors":[{"outputArtifactKey":"output_dataset_one","producerSubtask":"preprocess"}]},"output_dataset_two_path":{"artifactSelectors":[{"outputArtifactKey":"output_dataset_two_path","producerSubtask":"preprocess"}]}},"parameters":{"output_bool_parameter_path":{"valueFromParameter":{"outputParameterKey":"output_bool_parameter_path","producerSubtask":"preprocess"}},"output_dict_parameter_path":{"valueFromParameter":{"outputParameterKey":"output_dict_parameter_path","producerSubtask":"preprocess"}},"output_list_parameter_path":{"valueFromParameter":{"outputParameterKey":"output_list_parameter_path","producerSubtask":"preprocess"}},"output_parameter_path":{"valueFromParameter":{"outputParameterKey":"output_parameter_path","producerSubtask":"preprocess"}}}},"tasks":{"preprocess":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"input_dict_parameter":{"componentInputParameter":"input_dict_parameter"},"input_list_parameter":{"componentInputParameter":"input_list_parameter"},"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"preprocess"}}}},"inputDefinitions":{"parameters":{"input_dict_parameter":{"parameterType":"STRUCT"},"input_list_parameter":{"parameterType":"LIST"},"message":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_dataset_one":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"output_dataset_two_path":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"output_bool_parameter_path":{"parameterType":"BOOLEAN"},"output_dict_parameter_path":{"parameterType":"STRUCT"},"output_list_parameter_path":{"parameterType":"LIST"},"output_parameter_path":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -223,11 +223,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f}}' + value: '{{workflow.parameters.components-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-preprocess"},"inputs":{"parameters":{"input_dict_parameter":{"componentInputParameter":"input_dict_parameter"},"input_list_parameter":{"componentInputParameter":"input_list_parameter"},"message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"preprocess"}}' - name: container - value: '{{workflow.parameters.implementations-58eb44fee654093cb441c465d8ce6209048c161178ed5ecd93354b97de8df48f}}' + value: '{{workflow.parameters.implementations-ea150e85c8a1d73e8fa5cc90cd210a9ae6d20007a7071b293f971fafdecbe6f4}}' - name: task-name value: preprocess - name: parent-dag-id diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index ed8ed76f2bb..6f27cae6091 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-consumer value: '{"pvcMount":[{"componentInputParameter":"pvc_name","mountPath":"/data","pvcNameParameter":{"componentInputParameter":"pvc_name"}}]}' - - name: components-dc9efe4dbc12a5d21a9be59b86f7103d95eaa794480a8f8d10d5199a0badc9d4 + - name: components-ce3e01089ee23e8efa89e03cf85a7ac0e7d23e8c90eb9480c0ba530606857c31 value: '{"executorLabel":"exec-consumer"}' - - name: implementations-dc9efe4dbc12a5d21a9be59b86f7103d95eaa794480a8f8d10d5199a0badc9d4 + - name: implementations-ce3e01089ee23e8efa89e03cf85a7ac0e7d23e8c90eb9480c0ba530606857c31 value: '{"args":["--executor_input","{{$}}","--function_to_execute","consumer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,12 +20,12 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - consumer() -\u003e None:\n with open(''/data/file.txt'', ''r'') as f:\n print(f.read())\n\n"],"image":"python:3.9"}' + consumer() -\u003e None:\n with open(''/data/file.txt'', ''r'') as f:\n print(f.read())\n\n"],"image":"python:3.11"}' - name: kubernetes-comp-producer value: '{"pvcMount":[{"componentInputParameter":"pvc_name","mountPath":"/data","pvcNameParameter":{"componentInputParameter":"pvc_name"}}]}' - - name: components-ad6bd24309f0be281c426daa392d4491a6d223e3a5a56f10dfb7ae7c162f9f26 + - name: components-e666b1854737f2a7cd7fb811fc0fca9cd637782450adc694b816d1980f0b2bd2 value: '{"executorLabel":"exec-producer","outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-ad6bd24309f0be281c426daa392d4491a6d223e3a5a56f10dfb7ae7c162f9f26 + - name: implementations-e666b1854737f2a7cd7fb811fc0fca9cd637782450adc694b816d1980f0b2bd2 value: '{"args":["--executor_input","{{$}}","--function_to_execute","producer"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -36,7 +36,7 @@ spec: -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef producer() -\u003e str:\n with open(''/data/file.txt'', ''w'') as f:\n f.write(''hello'')\n with - open(''/data/file.txt'', ''r'') as f:\n return f.read()\n\n"],"image":"python:3.9"}' + open(''/data/file.txt'', ''r'') as f:\n return f.read()\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"consumer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["producer"],"taskInfo":{"name":"consumer"}},"producer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"taskInfo":{"name":"producer"}}}},"inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -219,11 +219,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-dc9efe4dbc12a5d21a9be59b86f7103d95eaa794480a8f8d10d5199a0badc9d4}}' + value: '{{workflow.parameters.components-ce3e01089ee23e8efa89e03cf85a7ac0e7d23e8c90eb9480c0ba530606857c31}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-consumer"},"dependentTasks":["producer"],"taskInfo":{"name":"consumer"}}' - name: container - value: '{{workflow.parameters.implementations-dc9efe4dbc12a5d21a9be59b86f7103d95eaa794480a8f8d10d5199a0badc9d4}}' + value: '{{workflow.parameters.implementations-ce3e01089ee23e8efa89e03cf85a7ac0e7d23e8c90eb9480c0ba530606857c31}}' - name: task-name value: consumer - name: parent-dag-id @@ -246,11 +246,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-ad6bd24309f0be281c426daa392d4491a6d223e3a5a56f10dfb7ae7c162f9f26}}' + value: '{{workflow.parameters.components-e666b1854737f2a7cd7fb811fc0fca9cd637782450adc694b816d1980f0b2bd2}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-producer"},"taskInfo":{"name":"producer"}}' - name: container - value: '{{workflow.parameters.implementations-ad6bd24309f0be281c426daa392d4491a6d223e3a5a56f10dfb7ae7c162f9f26}}' + value: '{{workflow.parameters.implementations-e666b1854737f2a7cd7fb811fc0fca9cd637782450adc694b816d1980f0b2bd2}}' - name: task-name value: producer - name: parent-dag-id diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index b2285a5ada7..4434697a31a 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -10,9 +10,9 @@ spec: value: '{"executorLabel":"exec-importer","inputDefinitions":{"parameters":{"uri":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: implementations-comp-importer value: '{"artifactUri":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"},"metadata":{"key":"value"},"typeSchema":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}' - - name: components-c8e04e29003554b439081c46cb32438ef1f27bc6030910c9c95c7df619a4084b + - name: components-4351a2bfc0420795b356dad17db637309898c44eae821502eab5e2c753a1cddb value: '{"executorLabel":"exec-make-language-model","inputDefinitions":{"artifacts":{"text_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-c8e04e29003554b439081c46cb32438ef1f27bc6030910c9c95c7df619a4084b + - name: implementations-4351a2bfc0420795b356dad17db637309898c44eae821502eab5e2c753a1cddb value: '{"args":["--executor_input","{{$}}","--function_to_execute","make_language_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -29,7 +29,7 @@ spec: on text here #\n\n def dummy_model(x: str) -\u003e str:\n return x\n\n model = Model(\n uri=dsl.get_uri(suffix=''model''),\n metadata={''data'': text_dataset.name},\n )\n\n with open(model.path, ''wb'') as f:\n dill.dump(dummy_model, - f)\n\n return model\n\n"],"image":"python:3.9"}' + f)\n\n return model\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"Output","producerSubtask":"make-language-model"}]}}},"tasks":{"importer":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"}}}},"taskInfo":{"name":"importer"}},"make-language-model":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-language-model"},"dependentTasks":["importer"],"inputs":{"artifacts":{"text_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"make-language-model"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' entrypoint: entrypoint @@ -281,11 +281,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c8e04e29003554b439081c46cb32438ef1f27bc6030910c9c95c7df619a4084b}}' + value: '{{workflow.parameters.components-4351a2bfc0420795b356dad17db637309898c44eae821502eab5e2c753a1cddb}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-language-model"},"dependentTasks":["importer"],"inputs":{"artifacts":{"text_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"make-language-model"}}' - name: container - value: '{{workflow.parameters.implementations-c8e04e29003554b439081c46cb32438ef1f27bc6030910c9c95c7df619a4084b}}' + value: '{{workflow.parameters.implementations-4351a2bfc0420795b356dad17db637309898c44eae821502eab5e2c753a1cddb}}' - name: task-name value: make-language-model - name: parent-dag-id diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index b81d4e1df15..b61a359e629 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-f03144e0835f90a75b071337bbe5e305fd22a078d3fe93a595dab07e9c256d93 + - name: components-c192368c8461c3cdeb1a1c1ce2c7130e644b68af5a8e9b05afbaa3aac70143cc value: '{"executorLabel":"exec-gen-data","outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-f03144e0835f90a75b071337bbe5e305fd22a078d3fe93a595dab07e9c256d93 + - name: implementations-c192368c8461c3cdeb1a1c1ce2c7130e644b68af5a8e9b05afbaa3aac70143cc value: '{"args":["--executor_input","{{$}}","--function_to_execute","gen_data"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef gen_data() -\u003e Dataset:\n dataset = Dataset(uri=dsl.get_uri())\n with open(dataset.path, \"w\") as f:\n f.write(\"some data\")\n\n dataset.metadata[\"length\"] - = len(\"some data\")\n return dataset\n\n"],"image":"python:3.9"}' - - name: components-4271229df0e3741657d9f339d415cee57460bed8e7b47f53f0b87b0c71b90ea8 + = len(\"some data\")\n return dataset\n\n"],"image":"python:3.11"}' + - name: components-d051e68c28d78cde70e680b1fdf49912c1eed8b335b05d1d4518af362cc097f5 value: '{"executorLabel":"exec-train-model","inputDefinitions":{"artifacts":{"dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Model","schemaVersion":"0.0.1"}}}}}' - - name: implementations-4271229df0e3741657d9f339d415cee57460bed8e7b47f53f0b87b0c71b90ea8 + - name: implementations-d051e68c28d78cde70e680b1fdf49912c1eed8b335b05d1d4518af362cc097f5 value: '{"args":["--executor_input","{{$}}","--function_to_execute","train_model"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -37,7 +37,7 @@ spec: as f:\n lines = f.read()\n\n assert lines == \"some data\"\n assert dataset.metadata[\"length\"] == len(\"some data\")\n\n model_artifact = Model(uri=dsl.get_uri(\"model\"))\n with open(model_artifact.path, \"w\") - as f:\n f.write(\"model trained\")\n\n return model_artifact\n\n"],"image":"python:3.9"}' + as f:\n f.write(\"model trained\")\n\n return model_artifact\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"gen-data":{"cachingOptions":{},"componentRef":{"name":"comp-gen-data"},"taskInfo":{"name":"gen-data"}},"train-model":{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["gen-data"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"gen-data"}}}},"taskInfo":{"name":"train-model"}}}}}' entrypoint: entrypoint @@ -220,11 +220,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-f03144e0835f90a75b071337bbe5e305fd22a078d3fe93a595dab07e9c256d93}}' + value: '{{workflow.parameters.components-c192368c8461c3cdeb1a1c1ce2c7130e644b68af5a8e9b05afbaa3aac70143cc}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-gen-data"},"taskInfo":{"name":"gen-data"}}' - name: container - value: '{{workflow.parameters.implementations-f03144e0835f90a75b071337bbe5e305fd22a078d3fe93a595dab07e9c256d93}}' + value: '{{workflow.parameters.implementations-c192368c8461c3cdeb1a1c1ce2c7130e644b68af5a8e9b05afbaa3aac70143cc}}' - name: task-name value: gen-data - name: parent-dag-id @@ -244,11 +244,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-4271229df0e3741657d9f339d415cee57460bed8e7b47f53f0b87b0c71b90ea8}}' + value: '{{workflow.parameters.components-d051e68c28d78cde70e680b1fdf49912c1eed8b335b05d1d4518af362cc097f5}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-train-model"},"dependentTasks":["gen-data"],"inputs":{"artifacts":{"dataset":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"gen-data"}}}},"taskInfo":{"name":"train-model"}}' - name: container - value: '{{workflow.parameters.implementations-4271229df0e3741657d9f339d415cee57460bed8e7b47f53f0b87b0c71b90ea8}}' + value: '{{workflow.parameters.implementations-d051e68c28d78cde70e680b1fdf49912c1eed8b335b05d1d4518af362cc097f5}}' - name: task-name value: train-model - name: parent-dag-id diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 0c0e71b1b1f..f75f2c462c1 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-78078b3cb8d4867a5748fe50ca82fd5a6078e1d7d26a34603c77a1d8bc21c3cb + - name: components-00785a58f9cf0df219dc62ecbba2450770628d2c6d43751435fa5c1b9a479637 value: '{"executorLabel":"exec-make-dataset","inputDefinitions":{"parameters":{"text":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-78078b3cb8d4867a5748fe50ca82fd5a6078e1d7d26a34603c77a1d8bc21c3cb + - name: implementations-00785a58f9cf0df219dc62ecbba2450770628d2c6d43751435fa5c1b9a479637 value: '{"args":["--executor_input","{{$}}","--function_to_execute","make_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,12 +20,12 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef make_dataset(text: str) -\u003e Dataset:\n dataset = Dataset(uri=dsl.get_uri(), metadata={''length'': len(text)})\n with open(dataset.path, ''w'') as f:\n f.write(text)\n return - dataset\n\n"],"image":"python:3.9"}' + dataset\n\n"],"image":"python:3.11"}' - name: components-comp-for-loop-1 value: '{"dag":{"outputs":{"artifacts":{"pipelinechannel--make-dataset-Output":{"artifactSelectors":[{"outputArtifactKey":"Output","producerSubtask":"make-dataset"}]}}},"tasks":{"make-dataset":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-dataset"},"inputs":{"parameters":{"text":{"componentInputParameter":"pipelinechannel--texts-loop-item"}}},"taskInfo":{"name":"make-dataset"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--texts":{"parameterType":"LIST"},"pipelinechannel--texts-loop-item":{"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"pipelinechannel--make-dataset-Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}}}' - - name: components-326818c936509d860f5df651a290f7146d2c51d6c6ecf522e82815b71bc56419 + - name: components-c10c6ce95f2c2e5236bcfc27f308556c66fb19eed301c24c3b93be8a1998b8d3 value: '{"executorLabel":"exec-join-datasets","inputDefinitions":{"artifacts":{"datasets":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"},"isArtifactList":true}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-326818c936509d860f5df651a290f7146d2c51d6c6ecf522e82815b71bc56419 + - name: implementations-c10c6ce95f2c2e5236bcfc27f308556c66fb19eed301c24c3b93be8a1998b8d3 value: '{"args":["--executor_input","{{$}}","--function_to_execute","join_datasets"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -37,7 +37,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef join_datasets(datasets: List[Dataset]) -\u003e Dataset:\n texts = []\n for dataset in datasets:\n with open(dataset.path, ''r'') as f:\n texts.append(f.read())\n\n return - ''''.join(texts)\n\n"],"image":"python:3.9"}' + ''''.join(texts)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"outputs":{"artifacts":{"Output":{"artifactSelectors":[{"outputArtifactKey":"Output","producerSubtask":"join-datasets"}]}}},"tasks":{"for-loop-1":{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--texts":{"componentInputParameter":"texts"}}},"parameterIterator":{"itemInput":"pipelinechannel--texts-loop-item","items":{"inputParameter":"pipelinechannel--texts"}},"taskInfo":{"name":"for-loop-1"}},"join-datasets":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-join-datasets"},"dependentTasks":["for-loop-1"],"inputs":{"artifacts":{"datasets":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--make-dataset-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"join-datasets"}}}},"inputDefinitions":{"parameters":{"texts":{"defaultValue":["Hello",","," ","world!"],"isOptional":true,"parameterType":"LIST"}}},"outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' @@ -221,11 +221,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-78078b3cb8d4867a5748fe50ca82fd5a6078e1d7d26a34603c77a1d8bc21c3cb}}' + value: '{{workflow.parameters.components-00785a58f9cf0df219dc62ecbba2450770628d2c6d43751435fa5c1b9a479637}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-dataset"},"inputs":{"parameters":{"text":{"componentInputParameter":"pipelinechannel--texts-loop-item"}}},"taskInfo":{"name":"make-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-78078b3cb8d4867a5748fe50ca82fd5a6078e1d7d26a34603c77a1d8bc21c3cb}}' + value: '{{workflow.parameters.implementations-00785a58f9cf0df219dc62ecbba2450770628d2c6d43751435fa5c1b9a479637}}' - name: task-name value: make-dataset - name: parent-dag-id @@ -395,11 +395,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-326818c936509d860f5df651a290f7146d2c51d6c6ecf522e82815b71bc56419}}' + value: '{{workflow.parameters.components-c10c6ce95f2c2e5236bcfc27f308556c66fb19eed301c24c3b93be8a1998b8d3}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-join-datasets"},"dependentTasks":["for-loop-1"],"inputs":{"artifacts":{"datasets":{"taskOutputArtifact":{"outputArtifactKey":"pipelinechannel--make-dataset-Output","producerTask":"for-loop-1"}}}},"taskInfo":{"name":"join-datasets"}}' - name: container - value: '{{workflow.parameters.implementations-326818c936509d860f5df651a290f7146d2c51d6c6ecf522e82815b71bc56419}}' + value: '{{workflow.parameters.implementations-c10c6ce95f2c2e5236bcfc27f308556c66fb19eed301c24c3b93be8a1998b8d3}}' - name: task-name value: join-datasets - name: parent-dag-id diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index 98831e41636..323901b6e43 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-86b07190b395429f683afe3cd56a6974f473d3ea998451027e7bfa62aa1be472 + - name: components-8985a8fb7d17ffd865e926bd80a135f8c5e0ff2fcc20f64013705bc6175ad440 value: '{"executorLabel":"exec-make-dataset","outputDefinitions":{"artifacts":{"Output":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-86b07190b395429f683afe3cd56a6974f473d3ea998451027e7bfa62aa1be472 + - name: implementations-8985a8fb7d17ffd865e926bd80a135f8c5e0ff2fcc20f64013705bc6175ad440 value: '{"args":["--executor_input","{{$}}","--function_to_execute","make_dataset"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -20,10 +20,10 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef make_dataset() -\u003e Artifact:\n artifact = Artifact(uri=dsl.get_uri(''dataset''))\n with open(artifact.path, ''w'') as f:\n f.write(''Hello, world'')\n return - artifact\n\n"],"image":"python:3.9"}' - - name: components-efab22a295fc4d308178d42f46828ee08d7a03921b278b3061e33b418da4a0d6 + artifact\n\n"],"image":"python:3.11"}' + - name: components-a1cab77dea9e6edee97745b998070da20716dcf0eee94102ec9bfac02152a267 value: '{"executorLabel":"exec-dataset-splitter","inputDefinitions":{"artifacts":{"in_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"dataset1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"dataset2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - - name: implementations-efab22a295fc4d308178d42f46828ee08d7a03921b278b3061e33b418da4a0d6 + - name: implementations-a1cab77dea9e6edee97745b998070da20716dcf0eee94102ec9bfac02152a267 value: '{"args":["--executor_input","{{$}}","--function_to_execute","dataset_splitter"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -41,7 +41,7 @@ spec: = Dataset(\n uri=dsl.get_uri(suffix=''dataset2''),\n metadata={''original_data'': in_dataset.name},\n )\n with open(dataset2.path, ''w'') as f:\n f.write(out_data2)\n\n outputs = NamedTuple(\n ''outputs'',\n dataset1=Dataset,\n dataset2=Dataset,\n )\n return - outputs(dataset1=dataset1, dataset2=dataset2)\n\n"],"image":"python:3.9"}' + outputs(dataset1=dataset1, dataset2=dataset2)\n\n"],"image":"python:3.11"}' - name: components-comp-splitter-pipeline value: '{"dag":{"outputs":{"artifacts":{"dataset1":{"artifactSelectors":[{"outputArtifactKey":"dataset1","producerSubtask":"dataset-splitter"}]},"dataset2":{"artifactSelectors":[{"outputArtifactKey":"dataset1","producerSubtask":"dataset-splitter"}]}}},"tasks":{"dataset-splitter":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-splitter"},"inputs":{"artifacts":{"in_dataset":{"componentInputArtifact":"in_dataset"}}},"taskInfo":{"name":"dataset-splitter"}}}},"inputDefinitions":{"artifacts":{"in_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"dataset1":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"dataset2":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}' - name: components-root @@ -226,11 +226,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-efab22a295fc4d308178d42f46828ee08d7a03921b278b3061e33b418da4a0d6}}' + value: '{{workflow.parameters.components-a1cab77dea9e6edee97745b998070da20716dcf0eee94102ec9bfac02152a267}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dataset-splitter"},"inputs":{"artifacts":{"in_dataset":{"componentInputArtifact":"in_dataset"}}},"taskInfo":{"name":"dataset-splitter"}}' - name: container - value: '{{workflow.parameters.implementations-efab22a295fc4d308178d42f46828ee08d7a03921b278b3061e33b418da4a0d6}}' + value: '{{workflow.parameters.implementations-a1cab77dea9e6edee97745b998070da20716dcf0eee94102ec9bfac02152a267}}' - name: task-name value: dataset-splitter - name: parent-dag-id @@ -335,11 +335,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-86b07190b395429f683afe3cd56a6974f473d3ea998451027e7bfa62aa1be472}}' + value: '{{workflow.parameters.components-8985a8fb7d17ffd865e926bd80a135f8c5e0ff2fcc20f64013705bc6175ad440}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-dataset"},"taskInfo":{"name":"make-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-86b07190b395429f683afe3cd56a6974f473d3ea998451027e7bfa62aa1be472}}' + value: '{{workflow.parameters.implementations-8985a8fb7d17ffd865e926bd80a135f8c5e0ff2fcc20f64013705bc6175ad440}}' - name: task-name value: make-dataset - name: parent-dag-id diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 8bcab3fea98..427b7aa5e96 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b + - name: components-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c value: '{"executorLabel":"exec-take-nap","inputDefinitions":{"parameters":{"naptime_secs":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b + - name: implementations-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c value: '{"args":["--executor_input","{{$}}","--function_to_execute","take_nap"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,10 +21,10 @@ spec: take_nap(naptime_secs: int) -\u003e str:\n \"\"\"Sleeps for secs\"\"\"\n from time import sleep # noqa: PLC0415\n\n print(f\"Sleeping for {naptime_secs} seconds: Zzzzzz ...\")\n sleep(naptime_secs)\n return \"I''m awake now. - Did I snore?\"\n\n"],"image":"python:3.9"}' - - name: components-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224 + Did I snore?\"\n\n"],"image":"python:3.11"}' + - name: components-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775 value: '{"executorLabel":"exec-wake-up","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224 + - name: implementations-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775 value: '{"args":["--executor_input","{{$}}","--function_to_execute","wake_up"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,7 +34,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - wake_up(message: str):\n \"\"\"Wakes up from nap printing a message\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + wake_up(message: str):\n \"\"\"Wakes up from nap printing a message\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"take-nap":{"cachingOptions":{},"componentRef":{"name":"comp-take-nap"},"inputs":{"parameters":{"naptime_secs":{"componentInputParameter":"naptime_secs"}}},"taskInfo":{"name":"take-nap"}},"wake-up":{"cachingOptions":{},"componentRef":{"name":"comp-wake-up"},"dependentTasks":["take-nap"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"take-nap"}}}},"taskInfo":{"name":"wake-up"}}}},"inputDefinitions":{"parameters":{"naptime_secs":{"defaultValue":900,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' entrypoint: entrypoint @@ -217,11 +217,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b}}' + value: '{{workflow.parameters.components-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-take-nap"},"inputs":{"parameters":{"naptime_secs":{"componentInputParameter":"naptime_secs"}}},"taskInfo":{"name":"take-nap"}}' - name: container - value: '{{workflow.parameters.implementations-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b}}' + value: '{{workflow.parameters.implementations-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c}}' - name: task-name value: take-nap - name: parent-dag-id @@ -241,11 +241,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224}}' + value: '{{workflow.parameters.components-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-wake-up"},"dependentTasks":["take-nap"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"take-nap"}}}},"taskInfo":{"name":"wake-up"}}' - name: container - value: '{{workflow.parameters.implementations-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224}}' + value: '{{workflow.parameters.implementations-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775}}' - name: task-name value: wake-up - name: parent-dag-id diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 8bcab3fea98..427b7aa5e96 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b + - name: components-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c value: '{"executorLabel":"exec-take-nap","inputDefinitions":{"parameters":{"naptime_secs":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"STRING"}}}}' - - name: implementations-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b + - name: implementations-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c value: '{"args":["--executor_input","{{$}}","--function_to_execute","take_nap"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,10 +21,10 @@ spec: take_nap(naptime_secs: int) -\u003e str:\n \"\"\"Sleeps for secs\"\"\"\n from time import sleep # noqa: PLC0415\n\n print(f\"Sleeping for {naptime_secs} seconds: Zzzzzz ...\")\n sleep(naptime_secs)\n return \"I''m awake now. - Did I snore?\"\n\n"],"image":"python:3.9"}' - - name: components-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224 + Did I snore?\"\n\n"],"image":"python:3.11"}' + - name: components-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775 value: '{"executorLabel":"exec-wake-up","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224 + - name: implementations-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775 value: '{"args":["--executor_input","{{$}}","--function_to_execute","wake_up"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -34,7 +34,7 @@ spec: \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - wake_up(message: str):\n \"\"\"Wakes up from nap printing a message\"\"\"\n print(message)\n\n"],"image":"python:3.9"}' + wake_up(message: str):\n \"\"\"Wakes up from nap printing a message\"\"\"\n print(message)\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"take-nap":{"cachingOptions":{},"componentRef":{"name":"comp-take-nap"},"inputs":{"parameters":{"naptime_secs":{"componentInputParameter":"naptime_secs"}}},"taskInfo":{"name":"take-nap"}},"wake-up":{"cachingOptions":{},"componentRef":{"name":"comp-wake-up"},"dependentTasks":["take-nap"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"take-nap"}}}},"taskInfo":{"name":"wake-up"}}}},"inputDefinitions":{"parameters":{"naptime_secs":{"defaultValue":900,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' entrypoint: entrypoint @@ -217,11 +217,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b}}' + value: '{{workflow.parameters.components-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-take-nap"},"inputs":{"parameters":{"naptime_secs":{"componentInputParameter":"naptime_secs"}}},"taskInfo":{"name":"take-nap"}}' - name: container - value: '{{workflow.parameters.implementations-8cf797f23380487bd1f1fb96644cbde025b64d18dae83a7b814083646a23230b}}' + value: '{{workflow.parameters.implementations-548fb7a269f053e3060d5a7b3eee8d609d93eccb623ab5a9b4913b8e62a3d71c}}' - name: task-name value: take-nap - name: parent-dag-id @@ -241,11 +241,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224}}' + value: '{{workflow.parameters.components-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-wake-up"},"dependentTasks":["take-nap"],"inputs":{"parameters":{"message":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"take-nap"}}}},"taskInfo":{"name":"wake-up"}}' - name: container - value: '{{workflow.parameters.implementations-5d8466fde111ff505f28c4c8a093c971b044637d9e786d6aa26ff7e24346b224}}' + value: '{{workflow.parameters.implementations-578a199df014da25449b2880c39c343872ba3c2c790e062fd52bbfd871d14775}}' - name: task-name value: wake-up - name: parent-dag-id diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 9bc9af3d702..230ca4cf796 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-e5d34ee36841e876565d23eaf4d49fb55df421558117e42fb4f0f56dfb8eb7cb + - name: components-c33fa3b99654c0449406568c8f79055a726bdfef6f12ef6bdd2fff0e1cc32bd0 value: '{"executorLabel":"exec-receive-file","inputDefinitions":{"artifacts":{"incomingfile":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"artifacts":{"saveartifact":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-e5d34ee36841e876565d23eaf4d49fb55df421558117e42fb4f0f56dfb8eb7cb + - name: implementations-c33fa3b99654c0449406568c8f79055a726bdfef6f12ef6bdd2fff0e1cc32bd0 value: '{"args":["--executor_input","{{$}}","--function_to_execute","receive_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -23,10 +23,10 @@ spec: %s, size is %s\" % (incomingfile, os.path.getsize(incomingfile)))\n\n with open(incomingfile, \"rb\") as f:\n b = f.read(1)\n print(\"read byte: %s\" % b)\n f.close()\n\n print(\"copying in %s to out %s\" - % (incomingfile, saveartifact))\n shutil.copyfile(incomingfile, saveartifact)\n\n"],"image":"python:3.9"}' - - name: components-2baa105dc921c5395c01625982411e9ae576e4eeaed37130715ad12ac007288f + % (incomingfile, saveartifact))\n shutil.copyfile(incomingfile, saveartifact)\n\n"],"image":"python:3.11"}' + - name: components-21623f5d1d15354c4b58c256e42c5e4fcabb7503930b4a6f4ace6876d023a660 value: '{"executorLabel":"exec-send-file","inputDefinitions":{"parameters":{"file_size_bytes":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"outgoingfile":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}}}}' - - name: implementations-2baa105dc921c5395c01625982411e9ae576e4eeaed37130715ad12ac007288f + - name: implementations-21623f5d1d15354c4b58c256e42c5e4fcabb7503930b4a6f4ace6876d023a660 value: '{"args":["--executor_input","{{$}}","--function_to_execute","send_file"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -43,10 +43,10 @@ spec: \"w\", compression=zipfile.ZIP_DEFLATED) as zipf:\n zipf.write(input_file_path, os.path.basename(input_file_path))\n\n print(\"starting creating the file...\")\n file_path = \"/tmp/large_file.txt\"\n create_large_file(file_path, file_size_bytes)\n zip_file(file_path, - outgoingfile)\n print(f\"saved: {outgoingfile}\")\n\n"],"image":"python:3.9"}' - - name: components-55196ea4ba5ecbd374c5ad86dbe244930e8558effdeb5a02e2a7f98d71dbbdc6 + outgoingfile)\n print(f\"saved: {outgoingfile}\")\n\n"],"image":"python:3.11"}' + - name: components-c222529d08b231cbea2debe39f47b0c9f12c39e63429d94017fceeb53226b68c value: '{"executorLabel":"exec-test-uploaded-artifact","inputDefinitions":{"artifacts":{"previous_step":{"artifactType":{"schemaTitle":"system.Artifact","schemaVersion":"0.0.1"}}},"parameters":{"bucket_name":{"parameterType":"STRING"},"file_size_bytes":{"parameterType":"NUMBER_INTEGER"},"mlpipeline_minio_artifact_secret":{"parameterType":"STRING"}}}}' - - name: implementations-55196ea4ba5ecbd374c5ad86dbe244930e8558effdeb5a02e2a7f98d71dbbdc6 + - name: implementations-c222529d08b231cbea2debe39f47b0c9f12c39e63429d94017fceeb53226b68c value: '{"args":["--executor_input","{{$}}","--function_to_execute","test_uploaded_artifact"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -73,7 +73,7 @@ spec: \"wb\") as file_data:\n for d in data.stream(32 * 1024):\n file_data.write(d)\n bytes_written = file_data.tell()\n\n print(file_size_bytes, bytes_written)\n diff = round((bytes_written / file_size_bytes) - 1, 3)\n print(diff)\n # - if not matching, the test will fail\n assert diff == 0\n\n"],"image":"python:3.9"}' + if not matching, the test will fail\n assert diff == 0\n\n"],"image":"python:3.11"}' - name: components-root value: '{"dag":{"tasks":{"receive-file":{"cachingOptions":{},"componentRef":{"name":"comp-receive-file"},"dependentTasks":["send-file"],"inputs":{"artifacts":{"incomingfile":{"taskOutputArtifact":{"outputArtifactKey":"outgoingfile","producerTask":"send-file"}}}},"taskInfo":{"name":"receive-file"}},"send-file":{"cachingOptions":{},"componentRef":{"name":"comp-send-file"},"inputs":{"parameters":{"file_size_bytes":{"runtimeValue":{"constant":20971520}}}},"taskInfo":{"name":"send-file"}},"test-uploaded-artifact":{"cachingOptions":{},"componentRef":{"name":"comp-test-uploaded-artifact"},"dependentTasks":["receive-file"],"inputs":{"artifacts":{"previous_step":{"taskOutputArtifact":{"outputArtifactKey":"saveartifact","producerTask":"receive-file"}}},"parameters":{"bucket_name":{"componentInputParameter":"bucket_name"},"file_size_bytes":{"runtimeValue":{"constant":20971520}},"mlpipeline_minio_artifact_secret":{"componentInputParameter":"mlpipeline_minio_artifact_secret"}}},"taskInfo":{"name":"test-uploaded-artifact"}}}},"inputDefinitions":{"parameters":{"bucket_name":{"parameterType":"STRING"},"mlpipeline_minio_artifact_secret":{"parameterType":"STRING"}}}}' entrypoint: entrypoint @@ -256,11 +256,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-e5d34ee36841e876565d23eaf4d49fb55df421558117e42fb4f0f56dfb8eb7cb}}' + value: '{{workflow.parameters.components-c33fa3b99654c0449406568c8f79055a726bdfef6f12ef6bdd2fff0e1cc32bd0}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-receive-file"},"dependentTasks":["send-file"],"inputs":{"artifacts":{"incomingfile":{"taskOutputArtifact":{"outputArtifactKey":"outgoingfile","producerTask":"send-file"}}}},"taskInfo":{"name":"receive-file"}}' - name: container - value: '{{workflow.parameters.implementations-e5d34ee36841e876565d23eaf4d49fb55df421558117e42fb4f0f56dfb8eb7cb}}' + value: '{{workflow.parameters.implementations-c33fa3b99654c0449406568c8f79055a726bdfef6f12ef6bdd2fff0e1cc32bd0}}' - name: task-name value: receive-file - name: parent-dag-id @@ -281,11 +281,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-2baa105dc921c5395c01625982411e9ae576e4eeaed37130715ad12ac007288f}}' + value: '{{workflow.parameters.components-21623f5d1d15354c4b58c256e42c5e4fcabb7503930b4a6f4ace6876d023a660}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-send-file"},"inputs":{"parameters":{"file_size_bytes":{"runtimeValue":{"constant":20971520}}}},"taskInfo":{"name":"send-file"}}' - name: container - value: '{{workflow.parameters.implementations-2baa105dc921c5395c01625982411e9ae576e4eeaed37130715ad12ac007288f}}' + value: '{{workflow.parameters.implementations-21623f5d1d15354c4b58c256e42c5e4fcabb7503930b4a6f4ace6876d023a660}}' - name: task-name value: send-file - name: parent-dag-id @@ -305,11 +305,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-55196ea4ba5ecbd374c5ad86dbe244930e8558effdeb5a02e2a7f98d71dbbdc6}}' + value: '{{workflow.parameters.components-c222529d08b231cbea2debe39f47b0c9f12c39e63429d94017fceeb53226b68c}}' - name: task value: '{"cachingOptions":{},"componentRef":{"name":"comp-test-uploaded-artifact"},"dependentTasks":["receive-file"],"inputs":{"artifacts":{"previous_step":{"taskOutputArtifact":{"outputArtifactKey":"saveartifact","producerTask":"receive-file"}}},"parameters":{"bucket_name":{"componentInputParameter":"bucket_name"},"file_size_bytes":{"runtimeValue":{"constant":20971520}},"mlpipeline_minio_artifact_secret":{"componentInputParameter":"mlpipeline_minio_artifact_secret"}}},"taskInfo":{"name":"test-uploaded-artifact"}}' - name: container - value: '{{workflow.parameters.implementations-55196ea4ba5ecbd374c5ad86dbe244930e8558effdeb5a02e2a7f98d71dbbdc6}}' + value: '{{workflow.parameters.implementations-c222529d08b231cbea2debe39f47b0c9f12c39e63429d94017fceeb53226b68c}}' - name: task-name value: test-uploaded-artifact - name: parent-dag-id diff --git a/test_data/sdk_compiled_pipelines/valid/artifacts_complex.yaml b/test_data/sdk_compiled_pipelines/valid/artifacts_complex.yaml index cd17aefa59c..7ee38a0659f 100644 --- a/test_data/sdk_compiled_pipelines/valid/artifacts_complex.yaml +++ b/test_data/sdk_compiled_pipelines/valid/artifacts_complex.yaml @@ -304,7 +304,7 @@ deploymentSpec: \ with open(dataset.path) as f:\n nums.append(int(f.read()))\n\ \ with open(out_dataset.path, 'w') as f:\n f.write(str(sum(nums)))\n\ \n" - image: python:3.9 + image: python:3.11 exec-add-2: container: args: @@ -336,7 +336,7 @@ deploymentSpec: \ with open(dataset.path) as f:\n nums.append(int(f.read()))\n\ \ with open(out_dataset.path, 'w') as f:\n f.write(str(sum(nums)))\n\ \n" - image: python:3.9 + image: python:3.11 exec-add-two-ints: container: args: @@ -368,7 +368,7 @@ deploymentSpec: \ as f:\n in_dataset1 = int(f.read())\n\n with open(in_dataset2.path)\ \ as f:\n in_dataset2 = int(f.read())\n\n with open(out_dataset.path,\ \ 'w') as f:\n f.write(str(in_dataset1 + in_dataset2))\n\n" - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -398,7 +398,7 @@ deploymentSpec: \ *\n\ndef double(\n num: int,\n out_dataset: Output[Dataset],\n):\n\ \ with open(out_dataset.path, 'w') as f:\n f.write(str(2 * num))\n\ \n" - image: python:3.9 + image: python:3.11 exec-double-2: container: args: @@ -428,7 +428,7 @@ deploymentSpec: \ *\n\ndef double(\n num: int,\n out_dataset: Output[Dataset],\n):\n\ \ with open(out_dataset.path, 'w') as f:\n f.write(str(2 * num))\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/artifacts_simple.py b/test_data/sdk_compiled_pipelines/valid/artifacts_simple.py index 04910311de0..a3541152344 100644 --- a/test_data/sdk_compiled_pipelines/valid/artifacts_simple.py +++ b/test_data/sdk_compiled_pipelines/valid/artifacts_simple.py @@ -28,7 +28,7 @@ def add(in_datasets: Input[List[Dataset]], out_dataset: Output[Dataset]): def add_container(in_datasets: Input[List[Dataset]], out_dataset: Output[Dataset]): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['python', '-c'], args=[ textwrap.dedent(""" diff --git a/test_data/sdk_compiled_pipelines/valid/artifacts_simple.yaml b/test_data/sdk_compiled_pipelines/valid/artifacts_simple.yaml index b4ac1e08207..2b159dd0f80 100644 --- a/test_data/sdk_compiled_pipelines/valid/artifacts_simple.yaml +++ b/test_data/sdk_compiled_pipelines/valid/artifacts_simple.yaml @@ -108,7 +108,7 @@ deploymentSpec: \ nums = []\n for dataset in in_datasets:\n with open(dataset.path)\ \ as f:\n nums.append(int(f.read()))\n with open(out_dataset.path,\ \ 'w') as f:\n f.write(str(sum(nums)))\n\n" - image: python:3.9 + image: python:3.11 exec-add-container: container: args: @@ -125,7 +125,7 @@ deploymentSpec: command: - python - -c - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -154,7 +154,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int, out_dataset: Output[Dataset]):\n with open(out_dataset.path,\ \ 'w') as f:\n f.write(str(2 * num))\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/collected_artifacts.yaml b/test_data/sdk_compiled_pipelines/valid/collected_artifacts.yaml index c8293e73bc0..b420e6699b9 100644 --- a/test_data/sdk_compiled_pipelines/valid/collected_artifacts.yaml +++ b/test_data/sdk_compiled_pipelines/valid/collected_artifacts.yaml @@ -392,7 +392,7 @@ deploymentSpec: \ *\n\ndef create_dataset(data: Output[Dataset], content: str):\n print(f'Creating\ \ file with content: {content}')\n with open(data.path, 'w') as f:\n\ \ f.write(content)\n\n" - image: python:3.9 + image: python:3.11 exec-create-file: container: args: @@ -422,7 +422,7 @@ deploymentSpec: \ *\n\ndef create_file(file: Output[Artifact], content: str):\n print(f'Creating\ \ file with content: {content}')\n with open(file.path, 'w') as f:\n\ \ f.write(content)\n\n" - image: python:3.9 + image: python:3.11 exec-read-datasets: container: args: @@ -453,7 +453,7 @@ deploymentSpec: \ print(f'Reading dataset {d.name} file: {d.path}')\n with\ \ open(d.path, 'r') as f:\n print(f.read())\n\n return 'files\ \ read'\n\n" - image: python:3.9 + image: python:3.11 exec-read-models: container: args: @@ -485,7 +485,7 @@ deploymentSpec: \ 'r') as f:\n info = f.read()\n print(f\"Model raw\ \ data: {info}\")\n print(f\"Model metadata: {m.metadata}\")\n\ \ return 'models read'\n\n" - image: python:3.9 + image: python:3.11 exec-read-models-2: container: args: @@ -517,7 +517,7 @@ deploymentSpec: \ 'r') as f:\n info = f.read()\n print(f\"Model raw\ \ data: {info}\")\n print(f\"Model metadata: {m.metadata}\")\n\ \ return 'models read'\n\n" - image: python:3.9 + image: python:3.11 exec-read-single-dataset-generate-model: container: args: @@ -550,7 +550,7 @@ deploymentSpec: \ f2:\n f2.write(f\"{info}-{id}\")\n results.metadata['model']\ \ = info\n results.metadata['model_name'] = f\"model-artifact-inner-iteration-{info}-{id}\"\ \n\n" - image: python:3.9 + image: python:3.11 exec-read-single-file: container: args: @@ -580,7 +580,7 @@ deploymentSpec: \ *\n\ndef read_single_file(file: Artifact) -> str:\n print(f'Reading\ \ file: {file.path}')\n with open(file.path, 'r') as f:\n print(f.read())\n\ \n return file.uri\n\n" - image: python:3.9 + image: python:3.11 exec-split-chars: container: args: @@ -609,7 +609,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef split_chars(model_ids: str) -> list:\n return model_ids.split(',')\n\ \n" - image: python:3.9 + image: python:3.11 exec-split-ids: container: args: @@ -638,7 +638,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef split_ids(model_ids: str) -> list:\n return model_ids.split(',')\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: collected-artifact-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/component_with_metadata_fields.yaml b/test_data/sdk_compiled_pipelines/valid/component_with_metadata_fields.yaml index 6fc7319f7bc..712079871e3 100644 --- a/test_data/sdk_compiled_pipelines/valid/component_with_metadata_fields.yaml +++ b/test_data/sdk_compiled_pipelines/valid/component_with_metadata_fields.yaml @@ -72,7 +72,7 @@ deploymentSpec: \ as f:\n content_b = f.read()\n\n concatenated_string = content_a\ \ + content_b\n with open(out_dataset.path, 'w') as f:\n f.write(concatenated_string)\n\ \n return concatenated_string\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: dataset-joiner root: diff --git a/test_data/sdk_compiled_pipelines/valid/component_with_task_final_status_GH-12033.yaml b/test_data/sdk_compiled_pipelines/valid/component_with_task_final_status_GH-12033.yaml index b0e830bea66..72944f2288d 100644 --- a/test_data/sdk_compiled_pipelines/valid/component_with_task_final_status_GH-12033.yaml +++ b/test_data/sdk_compiled_pipelines/valid/component_with_task_final_status_GH-12033.yaml @@ -40,7 +40,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef exit_comp(status: dsl.PipelineTaskFinalStatus):\n print(status)\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: exit-comp root: diff --git a/test_data/sdk_compiled_pipelines/valid/components_with_optional_artifacts.yaml b/test_data/sdk_compiled_pipelines/valid/components_with_optional_artifacts.yaml index 23213d69662..6b14795dc7f 100644 --- a/test_data/sdk_compiled_pipelines/valid/components_with_optional_artifacts.yaml +++ b/test_data/sdk_compiled_pipelines/valid/components_with_optional_artifacts.yaml @@ -144,7 +144,7 @@ deploymentSpec: \ None):\n if artifact is not None:\n print(artifact.name)\n \ \ print(artifact.uri)\n print(artifact.metadata)\n else:\n\ \ print('No artifact provided!')\n\n" - image: python:3.9 + image: python:3.11 exec-python-artifact-printer-2: container: args: @@ -175,7 +175,7 @@ deploymentSpec: \ None):\n if artifact is not None:\n print(artifact.name)\n \ \ print(artifact.uri)\n print(artifact.metadata)\n else:\n\ \ print('No artifact provided!')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: optional-artifact-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/conditional_producer_and_consumers.yaml b/test_data/sdk_compiled_pipelines/valid/conditional_producer_and_consumers.yaml index 693f163dd5b..975669e5fbb 100644 --- a/test_data/sdk_compiled_pipelines/valid/conditional_producer_and_consumers.yaml +++ b/test_data/sdk_compiled_pipelines/valid/conditional_producer_and_consumers.yaml @@ -147,7 +147,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add(nums: List[int]) -> int:\n return sum(nums)\n\n" - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -175,7 +175,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/container_io.py b/test_data/sdk_compiled_pipelines/valid/container_io.py index 294de386d7d..61ad94d05a2 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_io.py +++ b/test_data/sdk_compiled_pipelines/valid/container_io.py @@ -19,7 +19,7 @@ @container_component def container_io(text: str, output_path: OutputPath(str)): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=['my_program', text], args=['--output_path', output_path]) diff --git a/test_data/sdk_compiled_pipelines/valid/container_io.yaml b/test_data/sdk_compiled_pipelines/valid/container_io.yaml index 2f0e6a2cd6c..9ceb1796642 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_io.yaml +++ b/test_data/sdk_compiled_pipelines/valid/container_io.yaml @@ -25,7 +25,7 @@ deploymentSpec: command: - my_program - '{{$.inputs.parameters[''text'']}}' - image: python:3.9 + image: python:3.11 pipelineInfo: name: container-io root: diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.py b/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.py index e3da70a3851..ca1ad3ae7a9 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.py +++ b/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.py @@ -23,7 +23,7 @@ def container_with_concat_placeholder(text1: str, text2: Output[Dataset], output_path: OutputPath(str)): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', ConcatPlaceholder(['prefix-', text1, text2.uri]) diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.yaml b/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.yaml index 3e3258479de..3f1b83bfc37 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.yaml +++ b/test_data/sdk_compiled_pipelines/valid/container_with_concat_placeholder.yaml @@ -31,7 +31,7 @@ deploymentSpec: command: - my_program - '{"Concat": ["prefix-", "{{$.inputs.parameters[''text1'']}}", "{{$.outputs.artifacts[''text2''].uri}}"]}' - image: python:3.9 + image: python:3.11 pipelineInfo: name: container-with-concat-placeholder root: diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.py b/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.py index 7b21edc3e47..d0f84c11cfd 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.py +++ b/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.py @@ -24,7 +24,7 @@ def container_with_if_placeholder(output_path: OutputPath(str), dataset: Output[Dataset], optional_input: str = 'default'): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', IfPresentPlaceholder( diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.yaml b/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.yaml index 5523d0ff473..8bde17cf46f 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.yaml +++ b/test_data/sdk_compiled_pipelines/valid/container_with_if_placeholder.yaml @@ -37,7 +37,7 @@ deploymentSpec: - --dataset - '{"IfPresent": {"InputName": "optional_input", "Then": ["{{$.outputs.artifacts[''dataset''].uri}}"], "Else": ["bye"]}}' - image: python:3.9 + image: python:3.11 pipelineInfo: name: container-with-if-placeholder root: diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.py b/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.py index e5bc2f5cb0b..fc48bcd7f9f 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.py +++ b/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.py @@ -24,7 +24,7 @@ def container_with_placeholder_in_fstring( text1: str = 'text!', ): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=[ 'my_program', f'prefix-{text1}', diff --git a/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.yaml b/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.yaml index ac17854c778..cb113c7730c 100644 --- a/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.yaml +++ b/test_data/sdk_compiled_pipelines/valid/container_with_placeholder_in_fstring.yaml @@ -27,7 +27,7 @@ deploymentSpec: - my_program - prefix-{{$.inputs.parameters['text1']}} - '{{$.outputs.artifacts[''output_artifact''].uri}}/0' - image: python:3.9 + image: python:3.11 pipelineInfo: name: container-with-placeholder-in-fstring root: diff --git a/test_data/sdk_compiled_pipelines/valid/containerized_python_component.py b/test_data/sdk_compiled_pipelines/valid/containerized_python_component.py index e32d4ad454c..e91fda48e85 100644 --- a/test_data/sdk_compiled_pipelines/valid/containerized_python_component.py +++ b/test_data/sdk_compiled_pipelines/valid/containerized_python_component.py @@ -14,7 +14,7 @@ from kfp import dsl -@dsl.component(base_image='python:3.9', target_image='kfp-image') +@dsl.component(base_image='python:3.11', target_image='kfp-image') def concat_message(message1: str, message2: str) -> str: return message1 + message2 diff --git a/test_data/sdk_compiled_pipelines/valid/create_pod_metadata_complex.yaml b/test_data/sdk_compiled_pipelines/valid/create_pod_metadata_complex.yaml index ca4729b41de..f1c0b96449e 100644 --- a/test_data/sdk_compiled_pipelines/valid/create_pod_metadata_complex.yaml +++ b/test_data/sdk_compiled_pipelines/valid/create_pod_metadata_complex.yaml @@ -112,7 +112,7 @@ deploymentSpec: \ label = os.getenv(label_path)\n if label != \"\":\n raise\ \ ValueError(\n f\"Pod label is {label} but is supposed to be\ \ None.\"\n )\n return True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata: container: args: @@ -166,7 +166,7 @@ deploymentSpec: \ is supposed to be {label_exp_val_2}.\"\n )\n validated_metadata_count+=1\n\ \ if validated_metadata_count <1:\n raise RuntimeError(f\"No pod\ \ metadata found to validate.\")\n return True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata-2: container: args: @@ -220,7 +220,7 @@ deploymentSpec: \ is supposed to be {label_exp_val_2}.\"\n )\n validated_metadata_count+=1\n\ \ if validated_metadata_count <1:\n raise RuntimeError(f\"No pod\ \ metadata found to validate.\")\n return True\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-pod-metadata root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/add_numbers.yaml b/test_data/sdk_compiled_pipelines/valid/critical/add_numbers.yaml index fcf983e001e..48a68443d70 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/add_numbers.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/add_numbers.yaml @@ -47,7 +47,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: add-numbers root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/artifact_cache.yaml b/test_data/sdk_compiled_pipelines/valid/critical/artifact_cache.yaml index 455ee6ae3ec..463f50b5cfc 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/artifact_cache.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/artifact_cache.yaml @@ -91,7 +91,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef core_comp(dataset: dsl.Output[dsl.Dataset]):\n with open(dataset.path,\ \ 'w') as f:\n f.write('foo')\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -120,7 +120,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(input: dsl.Dataset):\n with open(input.path, 'r')\ \ as f:\n print('input: ', f.read())\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: artifact-cache-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/artifact_crust.yaml b/test_data/sdk_compiled_pipelines/valid/critical/artifact_crust.yaml index 8bfb71fe1e9..c3399674883 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/artifact_crust.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/artifact_crust.yaml @@ -89,7 +89,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef core_comp(dataset: dsl.Output[dsl.Dataset]):\n with open(dataset.path,\ \ 'w') as f:\n f.write('foo')\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -118,7 +118,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(input: dsl.Dataset):\n with open(input.path, 'r')\ \ as f:\n print('input: ', f.read())\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: artifact-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/collected_parameters.yaml b/test_data/sdk_compiled_pipelines/valid/critical/collected_parameters.yaml index 3a772330981..64b7886fa05 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/collected_parameters.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/collected_parameters.yaml @@ -185,7 +185,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef consume_ids(ids: List[str]) -> str:\n for id in ids:\n \ \ print(f'Consuming: {id}')\n return 'completed'\n\n" - image: python:3.9 + image: python:3.11 exec-consume-ids-2: container: args: @@ -214,7 +214,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef consume_ids(ids: List[str]) -> str:\n for id in ids:\n \ \ print(f'Consuming: {id}')\n return 'completed'\n\n" - image: python:3.9 + image: python:3.11 exec-consume-single-id: container: args: @@ -243,7 +243,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef consume_single_id(id: str) -> str:\n print(f'Consuming single:\ \ {id}')\n return 'completed'\n\n" - image: python:3.9 + image: python:3.11 exec-prepend-id: container: args: @@ -272,7 +272,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef prepend_id(content: str) -> str:\n print(f\"prepending: {content}\ \ with 'model_id'\")\n return f'model_id_{content}'\n\n" - image: python:3.9 + image: python:3.11 exec-split-ids: container: args: @@ -300,7 +300,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef split_ids(ids: str) -> list:\n return ids.split(',')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: collected-param-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/component_with_optional_inputs.yaml b/test_data/sdk_compiled_pipelines/valid/critical/component_with_optional_inputs.yaml index 716b616d28b..bc8291f057b 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/component_with_optional_inputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/component_with_optional_inputs.yaml @@ -48,7 +48,7 @@ deploymentSpec: \ {input1}, type: {type(input1)}')\n print(f'input2: {input2}, type:\ \ {type(input2)}')\n print(f'input3: {input3}, type: {type(input3)}')\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: v2-component-optional-input root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.py b/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.py index da0c8e78735..2ef8cd58438 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.py +++ b/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.py @@ -19,7 +19,7 @@ @dsl.container_component def hello_world_container(): return dsl.ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo', 'hello world'], args=[], ) diff --git a/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.yaml b/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.yaml index 7c2f0f3aed4..904e264de67 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/container_component_with_no_inputs.yaml @@ -10,7 +10,7 @@ deploymentSpec: command: - echo - hello world - image: python:3.9 + image: python:3.11 pipelineInfo: name: v2-container-component-no-input root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/flip_coin.yaml b/test_data/sdk_compiled_pipelines/valid/critical/flip_coin.yaml index 5dd95963f8d..b785fc65323 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/flip_coin.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/flip_coin.yaml @@ -294,7 +294,7 @@ deploymentSpec: \ tails randomly.\"\"\"\n import random # noqa: PLC0415\n\n result\ \ = \"heads\" if random.randint(0, 1) == 0 else \"tails\"\n print(result)\n\ \ return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-msg: container: args: @@ -323,7 +323,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-msg-2: container: args: @@ -352,7 +352,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-msg-3: container: args: @@ -381,7 +381,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-msg-4: container: args: @@ -410,7 +410,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_msg(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-random-num: container: args: @@ -441,7 +441,7 @@ deploymentSpec: \ random number between low and high.\"\"\"\n import random # noqa:\ \ PLC0415\n\n result = random.randint(low, high)\n print(result)\n\ \ return result\n\n" - image: python:3.9 + image: python:3.11 exec-random-num-2: container: args: @@ -472,7 +472,7 @@ deploymentSpec: \ random number between low and high.\"\"\"\n import random # noqa:\ \ PLC0415\n\n result = random.randint(low, high)\n print(result)\n\ \ return result\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: Shows how to use dsl.If(). name: conditional-execution-pipeline diff --git a/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml b/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml index 988b0482a4f..159f5d9e570 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/loop_consume_upstream.yaml @@ -104,7 +104,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef create_file(file: Output[Artifact], content: str):\n with\ \ open(file.path, 'w') as f:\n f.write(content)\n\n" - image: python:3.9 + image: python:3.11 exec-print-input: container: args: @@ -133,7 +133,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_input(input: list):\n for item in input:\n print(f'Input\ \ item: {item}')\n\n" - image: python:3.9 + image: python:3.11 exec-read-file: container: args: @@ -162,7 +162,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef read_file(file: Input[Artifact]) -> str:\n with open(file.path,\ \ 'r') as f:\n print(f.read())\n return file.path\n\n" - image: python:3.9 + image: python:3.11 exec-split-input: container: args: @@ -191,7 +191,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef split_input(input: str) -> list:\n return input.split(',')\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: displayName: Loop Consume Upstream name: loop-consume-upstream diff --git a/test_data/sdk_compiled_pipelines/valid/critical/mixed_parameters.yaml b/test_data/sdk_compiled_pipelines/valid/critical/mixed_parameters.yaml index ea516dfa225..4b777dd3605 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/mixed_parameters.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/mixed_parameters.yaml @@ -82,7 +82,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef core_comp() -> int:\n return 1\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -110,7 +110,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(x: int, y: int):\n print('sum :', x + y)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: mixed-parameters-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/modelcar.yaml b/test_data/sdk_compiled_pipelines/valid/critical/modelcar.yaml index d56fc38ad65..c5df2d62286 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/modelcar.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/modelcar.yaml @@ -66,7 +66,7 @@ deploymentSpec: \ *\n\ndef build_model_car(model: dsl.Output[dsl.Model]):\n # Simulate\ \ pushing the Modelcar to an OCI registry\n model.uri = \"oci://registry.domain.local/org/repo:v1.0\"\ \n\n" - image: python:3.9 + image: python:3.11 exec-get-model-files-list: container: args: @@ -106,7 +106,7 @@ deploymentSpec: \ \"The model does not have expected files: \"\n +\ \ \", \".join(sorted(expected_files.difference(filesInPath)))\n )\n\ \n return \", \".join(sorted(filesInPath))\n\n" - image: python:3.9 + image: python:3.11 exec-importer: importer: artifactUri: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/multiple_artifacts_namedtuple.yaml b/test_data/sdk_compiled_pipelines/valid/critical/multiple_artifacts_namedtuple.yaml index 3d9ec439ed1..df96afdda60 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/multiple_artifacts_namedtuple.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/multiple_artifacts_namedtuple.yaml @@ -114,7 +114,7 @@ deploymentSpec: \ *\n\ndef core_comp(ds1: dsl.Output[dsl.Dataset], ds2: dsl.Output[dsl.Dataset]):\n\ \ with open(ds1.path, 'w') as f:\n f.write('foo')\n with open(ds2.path,\ \ 'w') as f:\n f.write('bar')\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -144,7 +144,7 @@ deploymentSpec: \ *\n\ndef crust_comp(\n ds1: dsl.Dataset,\n ds2: dsl.Dataset,\n):\n\ \ with open(ds1.path, 'r') as f:\n print('ds1: ', f.read())\n\ \ with open(ds2.path, 'r') as f:\n print('ds2: ', f.read())\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: multiple-artifacts-namedtuple-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/multiple_parameters_namedtuple.yaml b/test_data/sdk_compiled_pipelines/valid/critical/multiple_parameters_namedtuple.yaml index 123b9e2282b..8609a1f0c0a 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/multiple_parameters_namedtuple.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/multiple_parameters_namedtuple.yaml @@ -98,7 +98,7 @@ deploymentSpec: \ *\n\ndef core_comp() -> NamedTuple('outputs', val1=str, val2=str): #\ \ type: ignore\n outputs = NamedTuple('outputs', val1=str, val2=str)\n\ \ return outputs('foo', 'bar')\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -127,7 +127,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(val1: str, val2: str):\n print('val1: ', val1)\n\ \ print('val2: ', val2)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: multiple-parameters-namedtuple-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml index b2cbc610ebb..9b51dbd460d 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_input_child_level_compiled.yaml @@ -163,7 +163,7 @@ deploymentSpec: \ *\n\ndef component_a_bool(componentInputBool: bool = None):\n if componentInputBool\ \ != True:\n raise ValueError(f\"componentInputBool should be True\ \ but is {componentInputBool}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-a-int: container: args: @@ -193,7 +193,7 @@ deploymentSpec: \ *\n\ndef component_a_int(componentInputInt: int = None):\n if componentInputInt\ \ != 1:\n raise ValueError(f\"componentInputInt should be 1 but is\ \ {componentInputInt}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-a-str: container: args: @@ -223,7 +223,7 @@ deploymentSpec: \ *\n\ndef component_a_str(componentInputStr: str = None):\n if componentInputStr\ \ != 'Input - pipeline':\n raise ValueError(f\"componentInputStr\ \ should be 'Input - pipeline' but is {componentInputStr}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-b-bool: container: args: @@ -253,7 +253,7 @@ deploymentSpec: \ *\n\ndef component_b_bool(componentInputBool: bool = None):\n if componentInputBool\ \ != False:\n raise ValueError(f\"componentInputBool should be False\ \ but is {componentInputBool}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-b-int: container: args: @@ -283,7 +283,7 @@ deploymentSpec: \ *\n\ndef component_b_int(componentInputInt: int = None):\n if componentInputInt\ \ != 0:\n raise ValueError(f\"componentInputInt should be 0 but is\ \ {componentInputInt}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-b-str: container: args: @@ -313,7 +313,7 @@ deploymentSpec: \ *\n\ndef component_b_str(componentInputStr: str = None):\n if componentInputStr\ \ != 'Input 2 - nested pipeline':\n raise ValueError(f\"componentInputStr\ \ should be 'Input 2 - nested pipeline' but is {componentInputStr}\")\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nested-pipeline-opt-input-child-level root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml index 993a54c95da..53461e0464c 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -97,7 +97,7 @@ deploymentSpec: \ *\n\ndef component_bool(componentInput: bool = None):\n if componentInput\ \ is not None:\n raise ValueError(f\"componentInput should be None\ \ but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-int: container: args: @@ -127,7 +127,7 @@ deploymentSpec: \ *\n\ndef component_int(componentInput: int = None):\n if componentInput\ \ is not None:\n raise ValueError(f\"componentInput should be None\ \ but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-str: container: args: @@ -157,7 +157,7 @@ deploymentSpec: \ *\n\ndef component_str(componentInput: str = None):\n if componentInput\ \ is not None:\n raise ValueError(f\"componentInput should be None\ \ but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nested-pipeline-opt-inputs-nil root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml index e8eeeb92fd3..d3cc55a8fee 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -173,7 +173,7 @@ deploymentSpec: \ *\n\ndef component_bool_default(componentInput: bool = False):\n if\ \ componentInput != True:\n raise ValueError(f\"componentInput should\ \ be True but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-int-default: container: args: @@ -203,7 +203,7 @@ deploymentSpec: \ *\n\ndef component_int_default(componentInput: int = 0):\n if componentInput\ \ != 1:\n raise ValueError(f\"componentInput should be 1 but is {componentInput}\"\ )\n\n" - image: python:3.9 + image: python:3.11 exec-component-nil-bool-default: container: args: @@ -233,7 +233,7 @@ deploymentSpec: \ *\n\ndef component_nil_bool_default(componentInput: bool = None):\n \ \ if componentInput != True:\n raise ValueError(f\"componentInput\ \ should be True but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-nil-int-default: container: args: @@ -263,7 +263,7 @@ deploymentSpec: \ *\n\ndef component_nil_int_default(componentInput: int = None):\n if\ \ componentInput != 1:\n raise ValueError(f\"componentInput should\ \ be 1 but is {componentInput}\")\n\n" - image: python:3.9 + image: python:3.11 exec-component-nil-str-default: container: args: @@ -294,7 +294,7 @@ deploymentSpec: \ componentInput != 'Input - parent pipeline':\n raise ValueError(f\"\ componentInput should be 'Input - parent pipeline' but is {componentInput}\"\ )\n\n" - image: python:3.9 + image: python:3.11 exec-component-str-default: container: args: @@ -325,7 +325,7 @@ deploymentSpec: \ if componentInput != 'Input - parent pipeline':\n raise ValueError(f\"\ componentInput should be 'Input - parent pipeline' but is {componentInput}\"\ )\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nested-pipeline-opt-inputs-parent-level root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/notebook_component_simple.yaml b/test_data/sdk_compiled_pipelines/valid/critical/notebook_component_simple.yaml index 1e226889802..fa60b21e8b0 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/notebook_component_simple.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/notebook_component_simple.yaml @@ -155,7 +155,7 @@ deploymentSpec: \n with open(\"/tmp/kfp_nb_outputs/log.txt\", \"r\", encoding=\"utf-8\"\ ) as f:\n log = f.read()\n\n assert log == text + \" \" + \"default2\"\ \n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nb-simple root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/parallel_for_after_dependency.yaml b/test_data/sdk_compiled_pipelines/valid/critical/parallel_for_after_dependency.yaml index 761f788b207..f0a23c05615 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/parallel_for_after_dependency.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/parallel_for_after_dependency.yaml @@ -80,7 +80,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str) -> str:\n print(message)\n return\ \ message\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -109,7 +109,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str) -> str:\n print(message)\n return\ \ message\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -138,7 +138,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str) -> str:\n print(message)\n return\ \ message\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: loop-with-after-dependency-set root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/parameter_cache.yaml b/test_data/sdk_compiled_pipelines/valid/critical/parameter_cache.yaml index 30032a66ea4..aed26096184 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/parameter_cache.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/parameter_cache.yaml @@ -82,7 +82,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef core_comp() -> str:\n return 'foo'\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -110,7 +110,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(input: str):\n print('input :', input)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: parameter-cache-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/parameter_oneof.yaml b/test_data/sdk_compiled_pipelines/valid/critical/parameter_oneof.yaml index bb508e500b0..67bf8ddf41e 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/parameter_oneof.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/parameter_oneof.yaml @@ -214,7 +214,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef core_comp(input: str) -> str:\n print('input :', input)\n\ \ return input\n\n" - image: python:3.9 + image: python:3.11 exec-core-output-comp: container: args: @@ -244,7 +244,7 @@ deploymentSpec: \ *\n\ndef core_output_comp(input: str, output_key: dsl.OutputPath(str)):\n\ \ print('input :', input)\n with open(output_key, 'w') as f:\n \ \ f.write(input)\n\n" - image: python:3.9 + image: python:3.11 exec-crust-comp: container: args: @@ -272,7 +272,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef crust_comp(input: str):\n print('input :', input)\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin: container: args: @@ -301,7 +301,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef flip_coin() -> str:\n import random\n return 'heads' if\ \ random.randint(0, 1) == 0 else 'tails'\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: parameter-oneof-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/parameters_simple.yaml b/test_data/sdk_compiled_pipelines/valid/critical/parameters_simple.yaml index 962bc68ed06..38869e4aa08 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/parameters_simple.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/parameters_simple.yaml @@ -90,7 +90,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add(nums: List[int]) -> int:\n return sum(nums)\n\n" - image: python:3.9 + image: python:3.11 exec-add-container: container: args: @@ -128,7 +128,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_artifact_upload_download.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_artifact_upload_download.yaml index baaa51fbc4a..12001857d6d 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_artifact_upload_download.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_artifact_upload_download.yaml @@ -62,7 +62,7 @@ deploymentSpec: \ dataset_name)\n print(\"Downloaded Hugging Face data\")\n print(f\"\ Now saving to {output_dataset.path}\")\n ds.save_to_disk(output_dataset.path)\n\ \ print(f\"Saved to {output_dataset.path}\")\n\n" - image: python:3.9 + image: python:3.11 exec-print-dataset-info: container: args: @@ -94,7 +94,7 @@ deploymentSpec: \ dataset.uri)\n assert \"download-dataset-and-upload-as-artifact\" in\ \ dataset.uri, \"The URI of the downloaded artifact does not match the expected\ \ function's name that generated it\"\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: Download Hugging Face Data Set, upload it as an artifact and print its metadata diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_env.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_env.yaml index c0fb8a775ed..c89a7d181c2 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_env.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_env.yaml @@ -60,7 +60,7 @@ deploymentSpec: env: - name: ENV1 value: val1 - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-env root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_input_status_state.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_input_status_state.yaml index 4dd27499f0d..f6963ccd57a 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_input_status_state.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_input_status_state.yaml @@ -53,7 +53,7 @@ deploymentSpec: \ assert(status.pipeline_task_name == 'exit-handler-1')\n #TODO: Add\ \ assert statements to validate status.error_code and status.error_message\ \ values once those fields have been implemented.\n\n" - image: python:3.9 + image: python:3.11 exec-some-task: container: args: @@ -81,7 +81,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef some_task():\n print('Executing some_task()...')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: status-state-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_placeholders.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_placeholders.yaml index dc2ba8e4851..6e096255bd7 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_placeholders.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_placeholders.yaml @@ -53,7 +53,7 @@ deploymentSpec: \ == \"print-all-placeholders\"\n assert job_resource_name.startswith(\"\ pipeline-with-placeholders-\")\n\n output = \", \".join(allPlaceholders)\n\ \ print(output)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-placeholders root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_pod_metadata.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_pod_metadata.yaml index 574ad15fd98..c7fde31aa03 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_pod_metadata.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_pod_metadata.yaml @@ -135,7 +135,7 @@ deploymentSpec: \ if label != '' or label_2 != '':\n raise ValueError(\n \ \ f'Pod label is {label} but is supposed to be None.'\n )\n\ \ return True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata-task-a: container: args: @@ -176,7 +176,7 @@ deploymentSpec: \n label_val_2 = os.getenv(label_path_2)\n if label_val_2 != label_exp_val_2:\n\ \ raise ValueError(\n f'Pod label is {label_val_2} but\ \ is supposed to be {label_exp_val_2}.'\n )\n return True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata-task-b: container: args: @@ -235,7 +235,7 @@ deploymentSpec: \ label_val_3 = os.getenv(label_path_3)\n if label_val_3 != label_exp_val_3:\n\ \ raise ValueError(\n f'Pod label is {label_val_3} but\ \ is supposed to be {label_exp_val_3}.'\n )\n return True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata-task-c: container: args: @@ -273,7 +273,7 @@ deploymentSpec: \ raise ValueError(\n f'Pod annotation is {annotation_val_2}\ \ but is supposed to be {annotation_exp_val_2}.'\n )\n return\ \ True\n\n" - image: python:3.9 + image: python:3.11 exec-validate-pod-metadata-task-d: container: args: @@ -314,7 +314,7 @@ deploymentSpec: \ != label_exp_val_3:\n raise ValueError(\n f'Pod label\ \ is {label_val_3} but is supposed to be {label_exp_val_3}.'\n )\n\ \ return True\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-pod-metadata root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_retry.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_retry.yaml index 012ccc619cc..3c026dc08f6 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_retry.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_retry.yaml @@ -45,7 +45,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add(a: float, b: float) -> float:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: test-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_secret_as_env.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_secret_as_env.yaml index 5d4e3a16879..55f0c450c99 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_secret_as_env.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_secret_as_env.yaml @@ -43,7 +43,7 @@ deploymentSpec: , \"\")\n psw1 = os.getenv(\"PASSWORD_VAR1\", \"\")\n psw2 = os.getenv(\"\ PASSWORD_VAR2\", \"\")\n assert username == \"user1\"\n assert psw1\ \ == \"psw1\"\n assert psw2 == \"psw2\"\n\n" - image: python:3.9 + image: python:3.11 exec-generate-secret-name: container: args: @@ -73,7 +73,7 @@ deploymentSpec: \ *\n\ndef generate_secret_name(some_output: OutputPath(str)):\n secret_name\ \ = \"test-secret-3\"\n with open(some_output, 'w') as f:\n f.write(secret_name)\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-secret-env root: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_workspace.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_workspace.yaml index db933f09cac..4d7d56b9f15 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_workspace.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pipeline_with_workspace.yaml @@ -60,7 +60,7 @@ deploymentSpec: \ assert content == \"Hello from workspace!\"\n return content\n\ \ else:\n print(f\"File not found at: {file_path}\")\n \ \ return \"File not found\"\n\n" - image: python:3.9 + image: python:3.11 exec-write-to-workspace: container: args: @@ -93,7 +93,7 @@ deploymentSpec: , \"test_file.txt\")\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n\ \n with open(file_path, \"w\") as f:\n f.write(\"Hello from workspace!\"\ )\n\n print(f\"Wrote file to: {file_path}\")\n return file_path\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: A pipeline that demonstrates workspace functionality name: pipeline-with-workspace diff --git a/test_data/sdk_compiled_pipelines/valid/critical/pythonic_artifacts_test_pipeline.yaml b/test_data/sdk_compiled_pipelines/valid/critical/pythonic_artifacts_test_pipeline.yaml index 2e12ac6ea74..4b650df9ce4 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/pythonic_artifacts_test_pipeline.yaml @@ -55,7 +55,7 @@ deploymentSpec: \ with open(dataset.path, \"w\") as f:\n f.write(\"some data\"\ )\n\n dataset.metadata[\"length\"] = len(\"some data\")\n return dataset\n\ \n" - image: python:3.9 + image: python:3.11 exec-train-model: container: args: @@ -88,7 +88,7 @@ deploymentSpec: \ = Model(uri=dsl.get_uri(\"model\"))\n with open(model_artifact.path,\ \ \"w\") as f:\n f.write(\"model trained\")\n\n return model_artifact\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pythonic-artifacts-test root: diff --git a/test_data/sdk_compiled_pipelines/valid/cross_loop_after_topology.yaml b/test_data/sdk_compiled_pipelines/valid/cross_loop_after_topology.yaml index 445c720a2ce..51b6012b27a 100644 --- a/test_data/sdk_compiled_pipelines/valid/cross_loop_after_topology.yaml +++ b/test_data/sdk_compiled_pipelines/valid/cross_loop_after_topology.yaml @@ -233,7 +233,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -261,7 +261,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -289,7 +289,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-4: container: args: @@ -317,7 +317,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-5: container: args: @@ -345,7 +345,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-6: container: args: @@ -373,7 +373,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-7: container: args: @@ -401,7 +401,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-8: container: args: @@ -429,7 +429,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/dict_input.yaml b/test_data/sdk_compiled_pipelines/valid/dict_input.yaml index 308db5f70b4..140b779a2a9 100644 --- a/test_data/sdk_compiled_pipelines/valid/dict_input.yaml +++ b/test_data/sdk_compiled_pipelines/valid/dict_input.yaml @@ -38,7 +38,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef dict_input(struct: Dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: dict-input root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_index_urls.yaml b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_index_urls.yaml index 7bbbcd9ed7a..b0cb231cab3 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_index_urls.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_index_urls.yaml @@ -34,7 +34,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef component_op():\n import yapf\n print(dir(yapf))\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: v2-component-pip-index-urls root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install.yaml b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install.yaml index 4db48882c2b..96aa320b079 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install.yaml @@ -35,7 +35,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef component_with_pip_install():\n import yapf\n print(dir(yapf))\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: component-with-pip-install root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install_in_venv.yaml b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install_in_venv.yaml index d80a97723de..04a67bf3e39 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install_in_venv.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/component_with_pip_install_in_venv.yaml @@ -37,7 +37,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef component_with_pip_install():\n import yapf\n\n print(dir(yapf))\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: component-with-pip-install root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/concat_message.yaml b/test_data/sdk_compiled_pipelines/valid/essential/concat_message.yaml index 6c28c9a9c34..8029e3b715c 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/concat_message.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/concat_message.yaml @@ -48,7 +48,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef concat_message(message1: str, message2: str) -> str:\n return\ \ message1 + message2\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: concat-message root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.py b/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.py index f4f1fb526d0..9c42a2fdc13 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.py +++ b/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.py @@ -18,7 +18,7 @@ @container_component def container_no_input(): return ContainerSpec( - image='python:3.9', + image='python:3.11', command=['echo', 'hello world'], args=[], ) diff --git a/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.yaml b/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.yaml index 44d21e33edc..49b8aa97d01 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/container_no_input.yaml @@ -10,7 +10,7 @@ deploymentSpec: command: - echo - hello world - image: python:3.9 + image: python:3.11 pipelineInfo: name: container-no-input root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/dict_input.yaml b/test_data/sdk_compiled_pipelines/valid/essential/dict_input.yaml index a8194c148e1..61aa6714125 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/dict_input.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/dict_input.yaml @@ -38,7 +38,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef dict_input(struct: Dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: dict-input root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/embedded_artifact.yaml b/test_data/sdk_compiled_pipelines/valid/essential/embedded_artifact.yaml index 2e4994533eb..96bfcd491ac 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/embedded_artifact.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/embedded_artifact.yaml @@ -49,7 +49,7 @@ deploymentSpec: \ import os\n\n with open(os.path.join(artifact.path, \"log.txt\"\ ), \"r\", encoding=\"utf-8\") as f:\n log = f.read()\n\n assert\ \ log == \"Hello, world!\"\n\n" - image: python:3.9 + image: python:3.11 exec-read-embedded-artifact-file: container: args: @@ -92,7 +92,7 @@ deploymentSpec: \ 'log.txt')\n\n\n\n\ndef read_embedded_artifact_file(artifact: dsl.EmbeddedInput[dsl.Dataset]):\n\ \ with open(artifact.path, \"r\", encoding=\"utf-8\") as f:\n \ \ log = f.read()\n\n assert log == \"Hello, world!\"\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nb-simple root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/iris_pipeline_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/essential/iris_pipeline_compiled.yaml index cfccf10b81d..e27d2c34557 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/iris_pipeline_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/iris_pipeline_compiled.yaml @@ -108,7 +108,7 @@ deploymentSpec: , \"Sepal_Width\", \"Petal_Length\", \"Petal_Width\", \"Labels\"]\n df\ \ = pd.read_csv(StringIO(data), names=col_names)\n\n with open(iris_dataset.path,\ \ \"w\") as f:\n df.to_csv(f)\n\n" - image: python:3.9 + image: python:3.11 exec-normalize-dataset: container: args: @@ -146,7 +146,7 @@ deploymentSpec: \ normalized_iris_dataset.metadata[\"state\"] = \"Normalized\"\n with\ \ open(normalized_iris_dataset.path, \"w\") as f:\n df.to_csv(f)\n\ \n" - image: python:3.9 + image: python:3.11 exec-train-model: container: args: @@ -190,7 +190,7 @@ deploymentSpec: \ predictions).tolist(), # .tolist() to convert np array to list.\n \ \ )\n\n model.metadata[\"framework\"] = \"scikit-learn\"\n with open(model.path,\ \ \"wb\") as f:\n pickle.dump(clf, f)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: iris-training-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_pipeline.yaml b/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_pipeline.yaml index f9087b137ee..d7dc9b18686 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_pipeline.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_pipeline.yaml @@ -119,7 +119,7 @@ deploymentSpec: \ 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with\ \ open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\ \n" - image: python:3.9 + image: python:3.11 exec-train: container: args: @@ -170,7 +170,7 @@ deploymentSpec: \ {i}\\n{line}\\n=====\\n')\n\n # model is an instance of Model artifact,\ \ which has a .metadata dictionary\n # to store arbitrary metadata for\ \ the output artifact.\n model.metadata['accuracy'] = 0.9\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-test-pipeline-beta root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_with_outputs.yaml b/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_with_outputs.yaml index 63dc4138f21..98a1564d9da 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_with_outputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/lightweight_python_functions_with_outputs.yaml @@ -95,7 +95,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_numbers(first: int, second: int) -> int:\n return first\ \ + second\n\n" - image: python:3.9 + image: python:3.11 exec-concat-message: container: args: @@ -124,7 +124,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef concat_message(first: str, second: str) -> str:\n return first\ \ + second\n\n" - image: python:3.9 + image: python:3.11 exec-output-artifact: container: args: @@ -153,7 +153,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef output_artifact(number: int, message: str) -> Dataset:\n result\ \ = [message for _ in range(number)]\n return '\\n'.join(result)\n\n" - image: python:3.9 + image: python:3.11 exec-output-named-tuple: container: args: @@ -189,7 +189,7 @@ deploymentSpec: \ = 'Model contents: ' + artifact_contents\n\n from collections import\ \ namedtuple\n output = namedtuple('Outputs', ['scalar', 'metrics', 'model'])\n\ \ return output(scalar, metrics, model)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: functions-with-outputs root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml index 56e8d2c2c0f..acbc0da27cd 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline.yaml @@ -90,7 +90,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op1-2: container: args: @@ -119,7 +119,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op2: container: command: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml index 64e1d9346a9..9e377c6beac 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_complex.yaml @@ -177,7 +177,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op1-2: container: args: @@ -206,7 +206,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op2: container: command: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml index 5a752ff0390..534078298ef 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -168,7 +168,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op1-2: container: args: @@ -197,7 +197,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op1-3: container: args: @@ -226,7 +226,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(data: Input[Artifact]):\n with open(data.path, 'r')\ \ as f:\n print(f.read())\n\n" - image: python:3.9 + image: python:3.11 exec-print-op2: container: args: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_condition.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_condition.yaml index e7e1e4709ba..3458ef394df 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_condition.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_condition.yaml @@ -105,7 +105,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op-2: container: args: @@ -135,7 +135,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -164,7 +164,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -193,7 +193,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -222,7 +222,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: single-condition-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml index c4e1e7af9b0..8cf4bec3f96 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_loops.yaml @@ -187,7 +187,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef args_generator_op() -> List[Dict[str, str]]:\n return [{'A_a':\ \ '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]\n\n" - image: python:3.9 + image: python:3.11 exec-print-struct: container: args: @@ -215,7 +215,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: Dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 exec-print-struct-2: container: args: @@ -243,7 +243,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: Dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text: container: args: @@ -271,7 +271,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-2: container: args: @@ -299,7 +299,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-3: container: args: @@ -327,7 +327,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-4: container: args: @@ -355,7 +355,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-5: container: args: @@ -383,7 +383,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-loops root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_metrics_outputs.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_metrics_outputs.yaml index 3d42ec7d9b6..444860f457d 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_metrics_outputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_metrics_outputs.yaml @@ -63,7 +63,7 @@ deploymentSpec: \ that outputs metrics with a random accuracy.\"\"\"\n import random\n\ \ result = random.randint(0, 100)\n metrics.log_metric('accuracy',\ \ result)\n\n" - image: python:3.9 + image: python:3.11 exec-output-metrics-2: container: args: @@ -94,7 +94,7 @@ deploymentSpec: \ that outputs metrics with a random accuracy.\"\"\"\n import random\n\ \ result = random.randint(0, 100)\n metrics.log_metric('accuracy',\ \ result)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-metrics-outputs root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_nested_conditions.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_nested_conditions.yaml index 86333309ea3..7c1325bc646 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_nested_conditions.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_nested_conditions.yaml @@ -164,7 +164,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op-2: container: args: @@ -194,7 +194,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op-3: container: args: @@ -224,7 +224,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op-4: container: args: @@ -254,7 +254,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -283,7 +283,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -312,7 +312,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -341,7 +341,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-4: container: args: @@ -370,7 +370,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str):\n \"\"\"Print a message.\"\"\"\n print(msg)\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nested-conditions-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_outputs.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_outputs.yaml index f7deac25493..0b989412e80 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_outputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_outputs.yaml @@ -120,7 +120,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op1-2: container: args: @@ -149,7 +149,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op1(msg: str) -> str:\n print(msg)\n return msg\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op2: container: args: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_params_containing_format.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_params_containing_format.yaml index 58e08a6fb3f..13ac0c849bd 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_params_containing_format.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_params_containing_format.yaml @@ -90,7 +90,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(text: str) -> str:\n print(text)\n return text\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -119,7 +119,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(text: str) -> str:\n print(text)\n return text\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-op2: container: args: @@ -148,7 +148,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op2(text1: str, text2: str) -> str:\n print(text1 +\ \ text2)\n return text1 + text2\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-pipelineparam-containing-format root: diff --git a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_reused_component.yaml b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_reused_component.yaml index fd28b26ae31..a491654fe11 100644 --- a/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_reused_component.yaml +++ b/test_data/sdk_compiled_pipelines/valid/essential/pipeline_with_reused_component.yaml @@ -69,7 +69,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 exec-add-numbers-2: container: args: @@ -97,7 +97,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 exec-add-numbers-3: container: args: @@ -125,7 +125,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-reused-component root: diff --git a/test_data/sdk_compiled_pipelines/valid/failing/fail_v2.yaml b/test_data/sdk_compiled_pipelines/valid/failing/fail_v2.yaml index f0fff944e55..b57ce19592c 100644 --- a/test_data/sdk_compiled_pipelines/valid/failing/fail_v2.yaml +++ b/test_data/sdk_compiled_pipelines/valid/failing/fail_v2.yaml @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail():\n '''Fails'''\n import sys\n sys.exit(1)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: fail-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_exit_handler.yaml b/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_exit_handler.yaml index 708129f8939..2c702cf113f 100644 --- a/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_exit_handler.yaml +++ b/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_exit_handler.yaml @@ -81,7 +81,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ \ print(message)\n sys.exit(1)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -110,7 +110,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -139,7 +139,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-exit-handler root: diff --git a/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_multiple_exit_handlers.yaml b/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_multiple_exit_handlers.yaml index 5c7eb1f60ea..5f0b9390bf0 100644 --- a/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/sdk_compiled_pipelines/valid/failing/pipeline_with_multiple_exit_handlers.yaml @@ -141,7 +141,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ \ print(message)\n sys.exit(1)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -170,7 +170,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -199,7 +199,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -228,7 +228,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-4: container: args: @@ -257,7 +257,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-5: container: args: @@ -286,7 +286,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-6: container: args: @@ -315,7 +315,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-multiple-exit-handlers root: diff --git a/test_data/sdk_compiled_pipelines/valid/identity.yaml b/test_data/sdk_compiled_pipelines/valid/identity.yaml index 43bb467fba0..d190e22ca05 100644 --- a/test_data/sdk_compiled_pipelines/valid/identity.yaml +++ b/test_data/sdk_compiled_pipelines/valid/identity.yaml @@ -44,7 +44,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef identity(value: str) -> str:\n return value\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: identity root: diff --git a/test_data/sdk_compiled_pipelines/valid/if_elif_else_complex.yaml b/test_data/sdk_compiled_pipelines/valid/if_elif_else_complex.yaml index 8cbfc427e7f..6aedbadfb66 100644 --- a/test_data/sdk_compiled_pipelines/valid/if_elif_else_complex.yaml +++ b/test_data/sdk_compiled_pipelines/valid/if_elif_else_complex.yaml @@ -712,7 +712,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef int_0_to_9999() -> int:\n import random\n return random.randint(0,\ \ 9999)\n\n" - image: python:3.9 + image: python:3.11 exec-is-even-or-odd: container: args: @@ -741,7 +741,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef is_even_or_odd(num: int) -> str:\n return 'odd' if num % 2\ \ else 'even'\n\n" - image: python:3.9 + image: python:3.11 exec-is-even-or-odd-2: container: args: @@ -770,7 +770,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef is_even_or_odd(num: int) -> str:\n return 'odd' if num % 2\ \ else 'even'\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return: container: args: @@ -799,7 +799,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-2: container: args: @@ -828,7 +828,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-3: container: args: @@ -857,7 +857,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-4: container: args: @@ -886,7 +886,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-5: container: args: @@ -915,7 +915,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-6: container: args: @@ -944,7 +944,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-7: container: args: @@ -973,7 +973,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-8: container: args: @@ -1002,7 +1002,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-9: container: args: @@ -1031,7 +1031,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-ints: container: args: @@ -1059,7 +1059,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_ints(ints: List[int]):\n print(ints)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: lucky-number-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/if_elif_else_with_oneof_parameters.yaml b/test_data/sdk_compiled_pipelines/valid/if_elif_else_with_oneof_parameters.yaml index 31e5c613d3d..2bba2003d03 100644 --- a/test_data/sdk_compiled_pipelines/valid/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/sdk_compiled_pipelines/valid/if_elif_else_with_oneof_parameters.yaml @@ -261,7 +261,7 @@ deploymentSpec: \ random.randint(0, 2)\n\n if val == 0:\n return 'heads'\n \ \ elif val == 1:\n return 'tails'\n else:\n return 'draw'\n\ \n" - image: python:3.9 + image: python:3.11 exec-print-and-return: container: args: @@ -290,7 +290,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-2: container: args: @@ -319,7 +319,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-3: container: args: @@ -348,7 +348,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-special-print-and-return: container: args: @@ -378,7 +378,7 @@ deploymentSpec: \ *\n\ndef special_print_and_return(text: str, output_key: dsl.OutputPath(str)):\n\ \ print('Got the special state:', text)\n with open(output_key, 'w')\ \ as f:\n f.write(text)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: outer-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_artifacts.yaml b/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_artifacts.yaml index 65e8f0fb448..dd0246c0f1d 100644 --- a/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_artifacts.yaml +++ b/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_artifacts.yaml @@ -232,7 +232,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef flip_coin() -> str:\n import random\n return 'heads' if\ \ random.randint(0, 1) == 0 else 'tails'\n\n" - image: python:3.9 + image: python:3.11 exec-param-to-artifact: container: args: @@ -261,7 +261,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef param_to_artifact(val: str, a: Output[Artifact]):\n with open(a.path,\ \ 'w') as f:\n f.write(val)\n\n" - image: python:3.9 + image: python:3.11 exec-param-to-artifact-2: container: args: @@ -290,7 +290,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef param_to_artifact(val: str, a: Output[Artifact]):\n with open(a.path,\ \ 'w') as f:\n f.write(val)\n\n" - image: python:3.9 + image: python:3.11 exec-print-artifact: container: args: @@ -319,7 +319,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_artifact(a: Input[Artifact]):\n with open(a.path) as\ \ f:\n print(f.read())\n\n" - image: python:3.9 + image: python:3.11 exec-print-artifact-2: container: args: @@ -348,7 +348,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_artifact(a: Input[Artifact]):\n with open(a.path) as\ \ f:\n print(f.read())\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: outer-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_parameters.yaml b/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_parameters.yaml index 04ff338058d..9cd040663db 100644 --- a/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_parameters.yaml +++ b/test_data/sdk_compiled_pipelines/valid/if_else_with_oneof_parameters.yaml @@ -171,7 +171,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef flip_coin() -> str:\n import random\n return 'heads' if\ \ random.randint(0, 1) == 0 else 'tails'\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return: container: args: @@ -200,7 +200,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-2: container: args: @@ -229,7 +229,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 exec-print-and-return-3: container: args: @@ -258,7 +258,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ \ text\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: flip-coin-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/input_artifact.yaml b/test_data/sdk_compiled_pipelines/valid/input_artifact.yaml index 5479462c5b5..bbb3339ed08 100644 --- a/test_data/sdk_compiled_pipelines/valid/input_artifact.yaml +++ b/test_data/sdk_compiled_pipelines/valid/input_artifact.yaml @@ -41,7 +41,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef input_artifact(data: Input[Dataset]):\n print(data.name)\n\ \ print(data.uri)\n print(data.metadata)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: input-artifact root: diff --git a/test_data/sdk_compiled_pipelines/valid/log_streaming_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/log_streaming_compiled.yaml index efb3cd6615b..18b6e7776ce 100644 --- a/test_data/sdk_compiled_pipelines/valid/log_streaming_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/log_streaming_compiled.yaml @@ -42,7 +42,7 @@ deploymentSpec: \ import time # noqa: PLC0415\n\n t_end = time.time() + 60\n while\ \ time.time() < t_end:\n print(message + \" (\" + str(datetime.datetime.now())\ \ + \")\")\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: Pipeline that prints a hello message in a loop to test log streaming in Dashboard diff --git a/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.py b/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.py index 8cc8f68d533..890250bcc95 100644 --- a/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.py +++ b/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.py @@ -24,7 +24,7 @@ @component( packages_to_install=['scikit-learn'], - base_image='python:3.9', + base_image='python:3.11', kfp_package_path=_KFP_PACKAGE_PATH, ) def digit_classification(metrics: Output[Metrics]): @@ -68,7 +68,7 @@ def digit_classification(metrics: Output[Metrics]): @component( packages_to_install=['scikit-learn'], - base_image='python:3.9', + base_image='python:3.11', kfp_package_path=_KFP_PACKAGE_PATH, ) def wine_classification(metrics: Output[ClassificationMetrics]): @@ -99,7 +99,7 @@ def wine_classification(metrics: Output[ClassificationMetrics]): @component( packages_to_install=['scikit-learn'], - base_image='python:3.9', + base_image='python:3.11', kfp_package_path=_KFP_PACKAGE_PATH, ) def iris_sgdclassifier(test_samples_fraction: float, diff --git a/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.yaml b/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.yaml index 58d5114e2a5..10b80b9b70b 100644 --- a/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.yaml +++ b/test_data/sdk_compiled_pipelines/valid/metrics_visualization_v2.yaml @@ -89,7 +89,7 @@ deploymentSpec: \ model.fit(X_train, y_train)\n\n #accuracy on test set\n result\ \ = model.score(X_test, y_test)\n metrics.log_metric('accuracy', (result\ \ * 100.0))\n\n" - image: python:3.9 + image: python:3.11 exec-html-visualization: container: args: @@ -120,7 +120,7 @@ deploymentSpec: \ = '

Hello world

'\n \ \ with open(html_artifact.path, 'w') as f:\n f.write(html_content)\n\ \n" - image: python:3.9 + image: python:3.11 exec-iris-sgdclassifier: container: args: @@ -161,7 +161,7 @@ deploymentSpec: \ 'Versicolour', 'Virginica'],\n confusion_matrix(\n train_y,\n\ \ predictions).tolist() # .tolist() to convert np array to list.\n\ \ )\n\n" - image: python:3.9 + image: python:3.11 exec-markdown-visualization: container: args: @@ -192,7 +192,7 @@ deploymentSpec: \ markdown_content = '## Hello world \\n\\n Markdown content'\n with\ \ open(markdown_artifact.path, 'w') as f:\n f.write(markdown_content)\n\ \n" - image: python:3.9 + image: python:3.11 exec-wine-classification: container: args: @@ -234,7 +234,7 @@ deploymentSpec: \ 1], pos_label=True)\n\n # avoid inf thresholds\n epsilon = 1e-6\n\ \ thresholds = [1 - epsilon if t == float('inf') else t for t in thresholds]\n\ \n metrics.log_roc_curve(fpr, tpr, thresholds)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: metrics-visualization-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/nested_return.yaml b/test_data/sdk_compiled_pipelines/valid/nested_return.yaml index 4036968620b..e560379f33b 100644 --- a/test_data/sdk_compiled_pipelines/valid/nested_return.yaml +++ b/test_data/sdk_compiled_pipelines/valid/nested_return.yaml @@ -39,7 +39,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef nested_return() -> List[Dict[str, str]]:\n return [{'A_a':\ \ '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}]\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nested-return root: diff --git a/test_data/sdk_compiled_pipelines/valid/nested_with_parameters.yaml b/test_data/sdk_compiled_pipelines/valid/nested_with_parameters.yaml index 6d6f5ebce4d..41385f3cf92 100644 --- a/test_data/sdk_compiled_pipelines/valid/nested_with_parameters.yaml +++ b/test_data/sdk_compiled_pipelines/valid/nested_with_parameters.yaml @@ -166,7 +166,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add(nums: List[List[int]]) -> int:\n import itertools\n \ \ return sum(itertools.chain(*nums))\n\n" - image: python:3.9 + image: python:3.11 exec-add-two-nums: container: args: @@ -194,7 +194,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_two_nums(x: int, y: int) -> int:\n return x + y\n\n" - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -222,7 +222,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 exec-double-2: container: args: @@ -250,7 +250,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/notebook_component_mixed.yaml b/test_data/sdk_compiled_pipelines/valid/notebook_component_mixed.yaml index ea0b4ce623c..9861f04c91b 100644 --- a/test_data/sdk_compiled_pipelines/valid/notebook_component_mixed.yaml +++ b/test_data/sdk_compiled_pipelines/valid/notebook_component_mixed.yaml @@ -193,7 +193,7 @@ deploymentSpec: \ == {\"score\": float(len(model_text))}\n\n for metric_name, metric_value\ \ in metrics_dict.items():\n metrics.log_metric(metric_name, metric_value)\n\ \n" - image: python:3.9 + image: python:3.11 exec-preprocess: container: args: @@ -224,7 +224,7 @@ deploymentSpec: \ import re\n\n cleaned_text = re.sub(r\"\\s+\", \" \", text).strip()\n\ \ with open(dataset.path, \"w\", encoding=\"utf-8\") as f:\n f.write(cleaned_text)\n\ \n" - image: python:3.9 + image: python:3.11 exec-train-model: container: args: @@ -372,7 +372,7 @@ deploymentSpec: , model.path)\n\n with open(model.path, \"r\", encoding=\"utf-8\") as\ \ f:\n model_text = f.read()\n\n assert model_text == cleaned_text.upper()\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: nb-mixed root: diff --git a/test_data/sdk_compiled_pipelines/valid/output_metrics.yaml b/test_data/sdk_compiled_pipelines/valid/output_metrics.yaml index dc7461949f6..d40ebd9d139 100644 --- a/test_data/sdk_compiled_pipelines/valid/output_metrics.yaml +++ b/test_data/sdk_compiled_pipelines/valid/output_metrics.yaml @@ -44,7 +44,7 @@ deploymentSpec: \ that outputs metrics with a random accuracy.\"\"\"\n import random\n\ \ result = random.randint(0, 100)\n metrics.log_metric('accuracy',\ \ result)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: output-metrics root: diff --git a/test_data/sdk_compiled_pipelines/valid/parameters_complex.yaml b/test_data/sdk_compiled_pipelines/valid/parameters_complex.yaml index a8d9b48f618..2f16d4c564e 100644 --- a/test_data/sdk_compiled_pipelines/valid/parameters_complex.yaml +++ b/test_data/sdk_compiled_pipelines/valid/parameters_complex.yaml @@ -240,7 +240,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add_two_numbers(x: List[int], y: List[int]) -> int:\n return\ \ sum(x) + sum(y)\n\n" - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -268,7 +268,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 exec-double-2: container: args: @@ -296,7 +296,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 exec-nested-add: container: args: @@ -325,7 +325,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef nested_add(nums: List[List[int]]) -> int:\n import itertools\n\ \ return sum(itertools.chain(*nums))\n\n" - image: python:3.9 + image: python:3.11 exec-nested-add-2: container: args: @@ -354,7 +354,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef nested_add(nums: List[List[int]]) -> int:\n import itertools\n\ \ return sum(itertools.chain(*nums))\n\n" - image: python:3.9 + image: python:3.11 exec-simple-add: container: args: @@ -382,7 +382,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef simple_add(nums: List[int]) -> int:\n return sum(nums)\n\n" - image: python:3.9 + image: python:3.11 exec-simple-add-2: container: args: @@ -410,7 +410,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef simple_add(nums: List[int]) -> int:\n return sum(nums)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_as_exit_task.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_as_exit_task.yaml index 57619355145..f39946b6f8d 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_as_exit_task.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_as_exit_task.yaml @@ -145,7 +145,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ \ print(message)\n sys.exit(1)\n\n" - image: python:3.9 + image: python:3.11 exec-get-run-state: container: args: @@ -174,7 +174,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef get_run_state(status: dict) -> str:\n print('Pipeline status:\ \ ', status)\n return status['state']\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -203,7 +203,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -232,7 +232,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-task-final-status-conditional root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_producer_consumer.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_producer_consumer.yaml index 068d30f88e7..20a2e7bffb4 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_producer_consumer.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_producer_consumer.yaml @@ -222,7 +222,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef add(nums: List[List[int]]) -> int:\n import itertools\n \ \ return sum(itertools.chain(*nums))\n\n" - image: python:3.9 + image: python:3.11 exec-double: container: args: @@ -250,7 +250,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef double(num: int) -> int:\n return 2 * num\n\n" - image: python:3.9 + image: python:3.11 exec-echo-and-return: container: args: @@ -279,7 +279,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef echo_and_return(string: str) -> str:\n print(string)\n \ \ return string\n\n" - image: python:3.9 + image: python:3.11 exec-join-and-print: container: args: @@ -308,7 +308,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef join_and_print(strings: List[str]):\n print(''.join(strings))\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: math-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 36c6a36ae81..4cfadc3d0f2 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -255,7 +255,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_count() -> int:\n return 1\n\n" - image: python:3.9 + image: python:3.11 exec-accelerator-type: container: args: @@ -283,7 +283,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_type() -> str:\n return 'NVIDIA_TESLA_P4'\n\n" - image: python:3.9 + image: python:3.11 exec-custom-training-job: container: args: @@ -344,7 +344,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef flip_biased_coin_op() -> str:\n \"\"\"Flip a coin and output\ \ heads.\"\"\"\n return 'heads'\n\n" - image: python:3.9 + image: python:3.11 exec-machine-type: container: args: @@ -372,7 +372,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef machine_type() -> str:\n return 'n1-standard-4'\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-dynamic-condition-output root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_importer_metadata.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_importer_metadata.yaml index 6966ea280ba..3bd9b7af74d 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_importer_metadata.yaml @@ -109,7 +109,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef make_name(name: str) -> str:\n return name\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-importer root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_task_output_custom_training_job.yaml index d10d4d9f55c..a4b2cd0f1c3 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -159,7 +159,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_count() -> int:\n return 1\n\n" - image: python:3.9 + image: python:3.11 exec-accelerator-type: container: args: @@ -187,7 +187,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_type() -> str:\n return 'NVIDIA_TESLA_P4'\n\n" - image: python:3.9 + image: python:3.11 exec-custom-training-job: container: args: @@ -243,7 +243,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef machine_type() -> str:\n return 'n1-standard-4'\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_google_artifact_type.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_google_artifact_type.yaml index d09851bdef2..ea88188b00c 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_google_artifact_type.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_google_artifact_type.yaml @@ -79,7 +79,7 @@ deploymentSpec: \ print('Dataset')\n print('artifact.type: ', type(dataset))\n print('artifact.name:\ \ ', dataset.name)\n print('artifact.uri: ', dataset.uri)\n print('artifact.metadata:\ \ ', dataset.metadata)\n\n" - image: python:3.9 + image: python:3.11 exec-model-producer: container: args: @@ -110,7 +110,7 @@ deploymentSpec: \ *\nimport aiplatform\n\ndef model_producer(model: Output[aiplatform.VertexModel]):\n\ \n assert isinstance(model, aiplatform.VertexModel), type(model)\n \ \ with open(model.path, 'w') as f:\n f.write('my model')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-google-types root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.json b/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.json index 88a13886300..02d10954242 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.json +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.json @@ -200,7 +200,7 @@ "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef train(\n dataset: Input[Dataset]\n) -> NamedTuple('Outputs', [\n ('scalar', str),\n ('model', Model),\n]):\n \"\"\"Dummy Training step.\"\"\"\n with open(dataset.path) as f:\n data = f.read()\n print('Dataset:', data)\n\n scalar = '123'\n model = f'My model trained using data: {data}'\n\n from collections import namedtuple\n output = namedtuple('Outputs', ['scalar', 'model'])\n return output(scalar, model)\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } }, "exec-train-2": { @@ -220,7 +220,7 @@ "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef train(\n dataset: Input[Dataset]\n) -> NamedTuple('Outputs', [\n ('scalar', str),\n ('model', Model),\n]):\n \"\"\"Dummy Training step.\"\"\"\n with open(dataset.path) as f:\n data = f.read()\n print('Dataset:', data)\n\n scalar = '123'\n model = f'My model trained using data: {data}'\n\n from collections import namedtuple\n output = namedtuple('Outputs', ['scalar', 'model'])\n return output(scalar, model)\n\n" ], - "image": "python:3.9" + "image": "python:3.11" } } } diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.yaml index e71d6193dc0..fff8c3d158c 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_importer.yaml @@ -148,7 +148,7 @@ deploymentSpec: \ trained using data: {data}'\n\n from collections import namedtuple\n\ \ output = namedtuple('Outputs', ['scalar', 'model'])\n return output(scalar,\ \ model)\n\n" - image: python:3.9 + image: python:3.11 exec-train-2: container: args: @@ -182,7 +182,7 @@ deploymentSpec: \ trained using data: {data}'\n\n from collections import namedtuple\n\ \ output = namedtuple('Outputs', ['scalar', 'model'])\n return output(scalar,\ \ model)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-importer root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_loops_and_conditions.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_loops_and_conditions.yaml index 7368df7932c..1af0ca14b7d 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_loops_and_conditions.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_loops_and_conditions.yaml @@ -620,7 +620,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-args-generator-op-2: container: args: @@ -651,7 +651,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.9 + image: python:3.11 exec-flip-coin-op: container: args: @@ -681,7 +681,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.9 + image: python:3.11 exec-print-struct: container: args: @@ -709,7 +709,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text: container: args: @@ -738,7 +738,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-2: container: args: @@ -767,7 +767,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-3: container: args: @@ -796,7 +796,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-4: container: args: @@ -825,7 +825,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-5: container: args: @@ -854,7 +854,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-6: container: args: @@ -883,7 +883,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-7: container: args: @@ -912,7 +912,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-8: container: args: @@ -941,7 +941,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-9: container: args: @@ -970,7 +970,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-loops-and-conditions-multi-layers root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_metadata_fields.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_metadata_fields.yaml index 750f5043b04..e031546fe4d 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_metadata_fields.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_metadata_fields.yaml @@ -84,7 +84,7 @@ deploymentSpec: \ as f:\n content_b = f.read()\n\n concatenated_string = content_a\ \ + content_b\n with open(out_dataset.path, 'w') as f:\n f.write(concatenated_string)\n\ \n return concatenated_string\n\n" - image: python:3.9 + image: python:3.11 exec-str-to-dataset: container: args: @@ -115,7 +115,7 @@ deploymentSpec: \"\"Convert string to dataset.\n\n Args:\n string: The string.\n\ \n Returns:\n dataset: The dataset.\n \"\"\"\n with open(dataset.path,\ \ 'w') as f:\n f.write(string)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: A pipeline that joins string to in_dataset. name: dataset-concatenator diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_nested_loops.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_nested_loops.yaml index 16015edebdc..f962fe7cf31 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_nested_loops.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_nested_loops.yaml @@ -161,7 +161,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-2: container: args: @@ -190,7 +190,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 exec-print-op-3: container: args: @@ -219,7 +219,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-nested-loops root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_list_artifacts_GH-12033.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_list_artifacts_GH-12033.yaml index 50abb4878d3..44509868a11 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_list_artifacts_GH-12033.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_list_artifacts_GH-12033.yaml @@ -274,7 +274,7 @@ deploymentSpec: \ *\n\ndef make_artifact(data: str) -> Artifact:\n artifact = Artifact(uri=dsl.get_uri(),\ \ metadata={'length': len(data)})\n with open(artifact.path, 'w') as\ \ f:\n f.write(data)\n return artifact\n\n" - image: python:3.9 + image: python:3.11 exec-make-dataset: container: args: @@ -304,7 +304,7 @@ deploymentSpec: \ *\n\ndef make_dataset(data: str) -> Dataset:\n dataset = Dataset(uri=dsl.get_uri(),\ \ metadata={'length': len(data)})\n with open(dataset.path, 'w') as f:\n\ \ f.write(data)\n return dataset\n\n" - image: python:3.9 + image: python:3.11 exec-print-artifact-name: container: args: @@ -333,7 +333,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_artifact_name(artifact: Artifact) -> str:\n print(artifact.name)\n\ \ return artifact.name\n\n" - image: python:3.9 + image: python:3.11 exec-print-artifact-name-2: container: args: @@ -362,7 +362,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_artifact_name(artifact: Artifact) -> str:\n print(artifact.name)\n\ \ return artifact.name\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-parallelfor-artifacts root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_parallelism.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_parallelism.yaml index 7c8542ea783..88b244a9f10 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_parallelfor_parallelism.yaml @@ -388,7 +388,7 @@ deploymentSpec: \ are strings and values are integers. For testing type \n handling during\ \ compilation.\"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}, {'a':\ \ 3, 'b': 4}]\n\n" - image: python:3.9 + image: python:3.11 exec-list-dict-maker-1: container: args: @@ -419,7 +419,7 @@ deploymentSpec: \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" - image: python:3.9 + image: python:3.11 exec-list-dict-maker-1-2: container: args: @@ -450,7 +450,7 @@ deploymentSpec: \ dictionary typing (no enforcement of specific key or\n value types).\n\ \n Tests flexibility in type handling.\n \"\"\"\n return [{'a':\ \ 1, 'b': 2}, {'a': 2, 'b': 3}, {'a': 3, 'b': 4}]\n\n" - image: python:3.9 + image: python:3.11 exec-list-dict-maker-2: container: args: @@ -481,7 +481,7 @@ deploymentSpec: \ of dictionaries without type enforcement.\n\n Tests flexibility in\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" - image: python:3.9 + image: python:3.11 exec-list-dict-maker-3: container: args: @@ -512,7 +512,7 @@ deploymentSpec: \ (no typing or structure guarantees).\n\n Tests the limits of compiler\ \ type handling.\n \"\"\"\n return [{'a': 1, 'b': 2}, {'a': 2, 'b':\ \ 3}, {'a': 3, 'b': 4}]\n\n" - image: python:3.9 + image: python:3.11 exec-print-int: container: args: @@ -540,7 +540,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-int-2: container: args: @@ -568,7 +568,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-int-3: container: args: @@ -596,7 +596,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-int-4: container: args: @@ -624,7 +624,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-int-5: container: args: @@ -652,7 +652,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-int-6: container: args: @@ -680,7 +680,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_int(x: int):\n print(x)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text: container: args: @@ -708,7 +708,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-2: container: args: @@ -736,7 +736,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-3: container: args: @@ -764,7 +764,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-4: container: args: @@ -792,7 +792,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-5: container: args: @@ -820,7 +820,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 exec-print-text-6: container: args: @@ -848,7 +848,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str):\n print(msg)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-loops root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_secret_as_volume.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_secret_as_volume.yaml index bbb28c08124..e4d18f21dae 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_secret_as_volume.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_secret_as_volume.yaml @@ -40,7 +40,7 @@ deploymentSpec: \ secret_file.read()\n\n # Decode the secret\n username = username.decode('utf-8')\n\ \n # Print the secret\n print(f\"username: {username}\")\n assert\ \ username == \"user1\"\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-secret-volume root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_semaphore.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_semaphore.yaml index 298029d7b43..ce908e5e53a 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_semaphore.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_semaphore.yaml @@ -32,7 +32,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef comp():\n pass\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-semaphore root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_pipeline_input.yaml index 2e01f7bea38..2d7e3059c30 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -47,7 +47,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef sum_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 resources: accelerator: resourceCount: '{{$.inputs.parameters[''pipelinechannel--accelerator_limit'']}}' diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_task_output.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_task_output.yaml index eb062b2ed76..58cbaba6480 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_string_machine_fields_task_output.yaml @@ -66,7 +66,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_limit() -> str:\n return '1'\n\n" - image: python:3.9 + image: python:3.11 exec-accelerator-type: container: args: @@ -94,7 +94,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef accelerator_type() -> str:\n return 'NVIDIA_TESLA_P4'\n\n" - image: python:3.9 + image: python:3.11 exec-cpu-limit: container: args: @@ -122,7 +122,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef cpu_limit() -> str:\n return '4000m'\n\n" - image: python:3.9 + image: python:3.11 exec-memory-limit: container: args: @@ -150,7 +150,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef memory_limit() -> str:\n return '15G'\n\n" - image: python:3.9 + image: python:3.11 exec-sum-numbers: container: args: @@ -178,7 +178,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef sum_numbers(a: int, b: int) -> int:\n return a + b\n\n" - image: python:3.9 + image: python:3.11 resources: accelerator: resourceCount: '{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}' diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status.yaml index a2d61308664..6e4e1cbe63b 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status.yaml @@ -88,7 +88,7 @@ deploymentSpec: \ print('Pipeline task name: ', status.pipeline_task_name)\n print('Error\ \ code: ', status.error_code)\n print('Error message: ', status.error_message)\n\ \n" - image: python:3.9 + image: python:3.11 exec-fail-op: container: args: @@ -117,7 +117,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ \ print(message)\n sys.exit(1)\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -146,7 +146,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ \ print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-task-final-status root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status_yaml.py b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status_yaml.py index caada43bc53..cd1cd5e396a 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status_yaml.py +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_final_status_yaml.py @@ -24,7 +24,7 @@ - {name: status, type: PipelineTaskFinalStatus} implementation: container: - image: python:3.9 + image: python:3.11 command: - echo - "user input:" @@ -39,7 +39,7 @@ - {name: message, type: String} implementation: container: - image: python:3.9 + image: python:3.11 command: - echo - {inputValue: message} diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_using_ignore_upstream_failure.yaml index 3f8294172b3..8b6843595c8 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -51,7 +51,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef fail_op(message: str) -> str:\n \"\"\"Fails.\"\"\"\n import\ \ sys\n print(message)\n sys.exit(1)\n return message\n\n" - image: python:3.9 + image: python:3.11 exec-print-op: container: args: @@ -80,7 +80,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_op(message: str = 'default'):\n \"\"\"Prints a message.\"\ \"\"\n print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: my-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_utils.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_utils.yaml index 2b56e016542..c0259da9160 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_utils.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_utils.yaml @@ -42,7 +42,7 @@ deploymentSpec: \n\n\ndef util_func2(msg: str) -> str:\n return f\"{msg} from util_func2\"\ \n\n\ndef echo(msg: str):\n assert util_func(msg) == f\"{msg} from util_func\"\ \n assert util_func2(msg) == f\"{msg} from util_func2\"\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: A simple hello world pipeline name: pipeline-with-utils diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.yaml index d7b63f2f86b..f1cddbcbf74 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.yaml @@ -113,7 +113,7 @@ deploymentSpec: \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ \ content = file.read()\n print(content)\n return content\n\ \n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -150,7 +150,7 @@ deploymentSpec: \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ \ as file:\n content = file.read()\n print(content)\n return\ \ content\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-volume root: diff --git a/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume_no_cache.yaml b/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume_no_cache.yaml index 3496c4ca9a4..e32c2ec0d1e 100644 --- a/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume_no_cache.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pipeline_with_volume_no_cache.yaml @@ -113,7 +113,7 @@ deploymentSpec: \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ \ content = file.read()\n print(content)\n return content\n\ \n" - image: python:3.9 + image: python:3.11 exec-createpvc: container: image: argostub/createpvc @@ -150,7 +150,7 @@ deploymentSpec: \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ \ as file:\n content = file.read()\n print(content)\n return\ \ content\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pipeline-with-volume-no-cache root: diff --git a/test_data/sdk_compiled_pipelines/valid/preprocess.yaml b/test_data/sdk_compiled_pipelines/valid/preprocess.yaml index f0e808fc3b5..e7089907c43 100644 --- a/test_data/sdk_compiled_pipelines/valid/preprocess.yaml +++ b/test_data/sdk_compiled_pipelines/valid/preprocess.yaml @@ -97,7 +97,7 @@ deploymentSpec: \ 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with\ \ open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\ \n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: preprocess root: diff --git a/test_data/sdk_compiled_pipelines/valid/pvc_mount.yaml b/test_data/sdk_compiled_pipelines/valid/pvc_mount.yaml index 73137fc1a57..194adbea0e4 100644 --- a/test_data/sdk_compiled_pipelines/valid/pvc_mount.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pvc_mount.yaml @@ -41,7 +41,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef consumer() -> None:\n with open('/data/file.txt', 'r') as\ \ f:\n print(f.read())\n\n" - image: python:3.9 + image: python:3.11 exec-producer: container: args: @@ -71,7 +71,7 @@ deploymentSpec: \ *\n\ndef producer() -> str:\n with open('/data/file.txt', 'w') as f:\n\ \ f.write('hello')\n with open('/data/file.txt', 'r') as f:\n\ \ return f.read()\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: pvc-mount-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pythonic_artifact_with_single_return.yaml b/test_data/sdk_compiled_pipelines/valid/pythonic_artifact_with_single_return.yaml index d8d354265c4..7822a0c24a0 100644 --- a/test_data/sdk_compiled_pipelines/valid/pythonic_artifact_with_single_return.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pythonic_artifact_with_single_return.yaml @@ -74,7 +74,7 @@ deploymentSpec: \ str) -> str:\n return x\n\n model = Model(\n uri=dsl.get_uri(suffix='model'),\n\ \ metadata={'data': text_dataset.name},\n )\n\n with open(model.path,\ \ 'wb') as f:\n dill.dump(dummy_model, f)\n\n return model\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: make-language-model-pipeline root: diff --git a/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_list_of_artifacts.yaml index 966c6432bb4..5b876ab4152 100644 --- a/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_list_of_artifacts.yaml @@ -97,7 +97,7 @@ deploymentSpec: \ = []\n for dataset in datasets:\n with open(dataset.path, 'r')\ \ as f:\n texts.append(f.read())\n\n return ''.join(texts)\n\ \n" - image: python:3.9 + image: python:3.11 exec-make-dataset: container: args: @@ -127,7 +127,7 @@ deploymentSpec: \ *\n\ndef make_dataset(text: str) -> Dataset:\n dataset = Dataset(uri=dsl.get_uri(),\ \ metadata={'length': len(text)})\n with open(dataset.path, 'w') as f:\n\ \ f.write(text)\n return dataset\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: make-and-join-datasets root: diff --git a/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_multiple_returns.yaml b/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_multiple_returns.yaml index 723cd964087..b886a7f12e7 100644 --- a/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/sdk_compiled_pipelines/valid/pythonic_artifacts_with_multiple_returns.yaml @@ -110,7 +110,7 @@ deploymentSpec: \ f.write(out_data2)\n\n outputs = NamedTuple(\n 'outputs',\n\ \ dataset1=Dataset,\n dataset2=Dataset,\n )\n return\ \ outputs(dataset1=dataset1, dataset2=dataset2)\n\n" - image: python:3.9 + image: python:3.11 exec-make-dataset: container: args: @@ -140,7 +140,7 @@ deploymentSpec: \ *\n\ndef make_dataset() -> Artifact:\n artifact = Artifact(uri=dsl.get_uri('dataset'))\n\ \ with open(artifact.path, 'w') as f:\n f.write('Hello, world')\n\ \ return artifact\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: split-datasets-and-return-first root: diff --git a/test_data/sdk_compiled_pipelines/valid/take_nap_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/take_nap_compiled.yaml index ff6ff70e1a2..4ae54608d6f 100644 --- a/test_data/sdk_compiled_pipelines/valid/take_nap_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/take_nap_compiled.yaml @@ -52,7 +52,7 @@ deploymentSpec: \"\"\n from time import sleep # noqa: PLC0415\n\n print(f\"Sleeping\ \ for {naptime_secs} seconds: Zzzzzz ...\")\n sleep(naptime_secs)\n \ \ return \"I'm awake now. Did I snore?\"\n\n" - image: python:3.9 + image: python:3.11 exec-wake-up: container: args: @@ -81,7 +81,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef wake_up(message: str):\n \"\"\"Wakes up from nap printing\ \ a message\"\"\"\n print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: Pipeline that sleeps for 15 mins (900 secs) name: take-nap-pipeline diff --git a/test_data/sdk_compiled_pipelines/valid/take_nap_pipeline_root_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/take_nap_pipeline_root_compiled.yaml index ee4dbbf9a9b..6d9294b9691 100644 --- a/test_data/sdk_compiled_pipelines/valid/take_nap_pipeline_root_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/take_nap_pipeline_root_compiled.yaml @@ -53,7 +53,7 @@ deploymentSpec: \"\"\n from time import sleep # noqa: PLC0415\n\n print(f\"Sleeping\ \ for {naptime_secs} seconds: Zzzzzz ...\")\n sleep(naptime_secs)\n \ \ return \"I'm awake now. Did I snore?\"\n\n" - image: python:3.9 + image: python:3.11 exec-wake-up: container: args: @@ -82,7 +82,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef wake_up(message: str):\n \"\"\"Wakes up from nap printing\ \ a message\"\"\"\n print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: description: Pipeline that sleeps for 15 mins (900 secs) name: take-nap-pipeline diff --git a/test_data/sdk_compiled_pipelines/valid/upload_download_compiled.yaml b/test_data/sdk_compiled_pipelines/valid/upload_download_compiled.yaml index 88c33338118..b53deb1c3fc 100644 --- a/test_data/sdk_compiled_pipelines/valid/upload_download_compiled.yaml +++ b/test_data/sdk_compiled_pipelines/valid/upload_download_compiled.yaml @@ -80,7 +80,7 @@ deploymentSpec: ) as f:\n b = f.read(1)\n print(\"read byte: %s\" % b)\n \ \ f.close()\n\n print(\"copying in %s to out %s\" % (incomingfile,\ \ saveartifact))\n shutil.copyfile(incomingfile, saveartifact)\n\n" - image: python:3.9 + image: python:3.11 exec-send-file: container: args: @@ -117,7 +117,7 @@ deploymentSpec: \ file...\")\n file_path = \"/tmp/large_file.txt\"\n create_large_file(file_path,\ \ file_size_bytes)\n zip_file(file_path, outgoingfile)\n print(f\"\ saved: {outgoingfile}\")\n\n" - image: python:3.9 + image: python:3.11 exec-test-uploaded-artifact: container: args: @@ -166,7 +166,7 @@ deploymentSpec: \ bytes_written)\n diff = round((bytes_written / file_size_bytes) - 1,\ \ 3)\n print(diff)\n # if not matching, the test will fail\n assert\ \ diff == 0\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: test-data-passing-pipeline-1 root: