fix: use unique name for integration job hyperparameter tuning job (#204)

Author: laurenyu

.flake8

@@ -1,3 +1,3 @@
 [flake8]
-application_import_names = sagemaker_tensorflow_container, test
+application_import_names = sagemaker_tensorflow_container, test, utils
 import-order-style = google

.gitignore

@@ -6,4 +6,5 @@ dist
 *.iml
 **/.ipynb_checkpoints
 **/.python-version
-.tox
+.tox
+*~

test/integration/sagemaker/test_horovod.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -17,6 +17,8 @@
 import sagemaker
 from sagemaker.tensorflow import TensorFlow
 
+from utils import unique_name_from_base
+
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 
 
@@ -41,7 +43,7 @@ def test_distributed_training_horovod(sagemaker_session,
                          'sagemaker_mpi_num_of_processes_per_host': 1},
         sagemaker_session=sagemaker_session)
 
-    estimator.fit()
+    estimator.fit(job_name=unique_name_from_base('test-tf-horovod'))
 
     model_data_source = sagemaker.local.data.get_data_source_instance(
         estimator.model_data, sagemaker_session)

test/integration/sagemaker/test_mnist.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -19,6 +19,7 @@
 from six.moves.urllib.parse import urlparse
 
 from sagemaker_tensorflow_container.training import SAGEMAKER_PARAMETER_SERVER_ENABLED
+from utils import unique_name_from_base
 
 
 def test_mnist(sagemaker_session, ecr_image, instance_type, framework_version):
@@ -31,12 +32,11 @@ def test_mnist(sagemaker_session, ecr_image, instance_type, framework_version):
                            sagemaker_session=sagemaker_session,
                            image_name=ecr_image,
                            framework_version=framework_version,
-                           py_version='py3',
-                           base_job_name='test-sagemaker-mnist')
+                           script_mode=True)
     inputs = estimator.sagemaker_session.upload_data(
         path=os.path.join(resource_path, 'mnist', 'data'),
         key_prefix='scriptmode/mnist')
-    estimator.fit(inputs)
+    estimator.fit(inputs, job_name=unique_name_from_base('test-sagemaker-mnist'))
     _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)
 
 
@@ -50,12 +50,11 @@ def test_distributed_mnist_no_ps(sagemaker_session, ecr_image, instance_type, fr
                            sagemaker_session=sagemaker_session,
                            image_name=ecr_image,
                            framework_version=framework_version,
-                           py_version='py3',
-                           base_job_name='test-tf-sm-distributed-mnist')
+                           script_mode=True)
     inputs = estimator.sagemaker_session.upload_data(
         path=os.path.join(resource_path, 'mnist', 'data'),
         key_prefix='scriptmode/mnist')
-    estimator.fit(inputs)
+    estimator.fit(inputs, job_name=unique_name_from_base('test-tf-sm-distributed-mnist'))
     _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)
 
 
@@ -70,12 +69,11 @@ def test_distributed_mnist_ps(sagemaker_session, ecr_image, instance_type, frame
                            sagemaker_session=sagemaker_session,
                            image_name=ecr_image,
                            framework_version=framework_version,
-                           py_version='py3',
-                           base_job_name='test-tf-sm-distributed-mnist')
+                           script_mode=True)
     inputs = estimator.sagemaker_session.upload_data(
         path=os.path.join(resource_path, 'mnist', 'data-distributed'),
         key_prefix='scriptmode/mnist-distributed')
-    estimator.fit(inputs)
+    estimator.fit(inputs, job_name=unique_name_from_base('test-tf-sm-distributed-mnist'))
     _assert_checkpoint_exists(sagemaker_session.boto_region_name, estimator.model_dir, 0)
     _assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)
 
@@ -104,9 +102,9 @@ def test_s3_plugin(sagemaker_session, ecr_image, instance_type, region, framewor
                            sagemaker_session=sagemaker_session,
                            image_name=ecr_image,
                            framework_version=framework_version,
-                           py_version='py3',
-                           base_job_name='test-tf-sm-s3-mnist')
-    estimator.fit('s3://sagemaker-sample-data-{}/tensorflow/mnist'.format(region))
+                           script_mode=True)
+    estimator.fit('s3://sagemaker-sample-data-{}/tensorflow/mnist'.format(region),
+                  job_name=unique_name_from_base('test-tf-sm-s3-mnist'))
     _assert_s3_file_exists(region, estimator.model_data)
     _assert_checkpoint_exists(region, estimator.model_dir, 200)
 

test/integration/sagemaker/test_tuning_model_dir.py

@@ -17,6 +17,8 @@
 from sagemaker.tensorflow import TensorFlow
 from sagemaker.tuner import HyperparameterTuner, IntegerParameter
 
+from utils import unique_name_from_base
+
 
 def test_model_dir_with_training_job_name(sagemaker_session, ecr_image, instance_type, framework_version):
     resource_path = os.path.join(os.path.dirname(__file__), '../..', 'resources')
@@ -36,9 +38,8 @@ def test_model_dir_with_training_job_name(sagemaker_session, ecr_image, instance
                                 hyperparameter_ranges={'arbitrary_value': IntegerParameter(0, 1)},
                                 metric_definitions=[{'Name': 'accuracy', 'Regex': 'accuracy=([01])'}],
                                 max_jobs=1,
-                                max_parallel_jobs=1,
-                                base_tuning_job_name='test-tf-tuning-model-dir')
+                                max_parallel_jobs=1)
 
     # User script has logic to check for the correct model_dir
-    tuner.fit()
+    tuner.fit(job_name=unique_name_from_base('test-tf-model-dir', max_length=32))
     tuner.wait()

test/integration/sagemaker/utils.py

@@ -0,0 +1,24 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import random
+import time
+
+
+def unique_name_from_base(base, max_length=63):
+    unique = '%04x' % random.randrange(16**4)  # 4-digit hex
+    ts = str(int(time.time()))
+    available_length = max_length - 2 - len(ts) - len(unique)
+    trimmed = base[:available_length]
+    return '{}-{}-{}'.format(trimmed, ts, unique)