Update apache beam to the latest version (#470)

Remove the constrains on apache beam version.
Update the input pattern and some unit tests since the match method in filesystems is changed in the newer version of beam.

Author: allieychen

gcp_variant_transforms/pipeline_common_test.py

@@ -45,9 +45,11 @@ def test_validation_failure_for_invalid_input_pattern(self):
 
   def test_get_mode_optimize_set(self):
     args = self._create_mock_args(
-        input_pattern='*', input_file=None, optimize_for_large_inputs=True)
-
-    self.assertEqual(self._get_pipeline_mode(args), PipelineModes.LARGE)
+        input_pattern='**', input_file=None, optimize_for_large_inputs=True)
+    match_result = collections.namedtuple('MatchResult', ['metadata_list'])
+    match = match_result([None for _ in range(100)])
+    with mock.patch.object(FileSystems, 'match', return_value=[match]):
+      self.assertEqual(self._get_pipeline_mode(args), PipelineModes.LARGE)
 
   def test_get_mode_small(self):
     args = self._create_mock_args(

gcp_variant_transforms/testing/integration/vcf_to_bq_tests/presubmit_tests/large_tests/option_optimize_for_large_inputs.json

@@ -2,7 +2,7 @@
   {
     "test_name": "option-optimize-for-large-inputs",
     "table_name": "option_optimize_for_large_inputs",
-    "input_pattern": "gs://gcp-variant-transforms-testfiles/large_tests/valid-70000-copies/*.vcf",
+    "input_pattern": "gs://gcp-variant-transforms-testfiles/large_tests/valid-70000-copies/**.vcf",
     "variant_merge_strategy": "MOVE_TO_CALLS",
     "optimize_for_large_inputs": true,
     "runner": "DataflowRunner",

gcp_variant_transforms/testing/integration/vcf_to_bq_tests/presubmit_tests/medium_tests/test_1000_copies_of_valid_4_2_no_merge.json

@@ -2,7 +2,7 @@
   {
     "test_name": "test-1000-copies-of-valid-4-2-no-merge",
     "table_name": "test_1000_copies_of_valid_4_2_no_merge",
-    "input_pattern": "gs://gcp-variant-transforms-testfiles/medium_tests/valid-1000-copies/*.vcf",
+    "input_pattern": "gs://gcp-variant-transforms-testfiles/medium_tests/valid-1000-copies/**.vcf",
     "runner": "DataflowRunner",
     "worker_machine_type": "n1-standard-16",
     "max_num_workers": "20",

gcp_variant_transforms/testing/integration/vcf_to_bq_tests/presubmit_tests/medium_tests/test_1000_copies_of_valid_4_2_with_merge.json

@@ -2,7 +2,7 @@
   {
     "test_name": "test-1000-copies-of-valid-4-2-with-merge",
     "table_name": "test_1000_copies_of_valid_4_2_with_merge",
-    "input_pattern": "gs://gcp-variant-transforms-testfiles/medium_tests/valid-1000-copies/*.vcf",
+    "input_pattern": "gs://gcp-variant-transforms-testfiles/medium_tests/valid-1000-copies/**.vcf",
     "variant_merge_strategy": "MOVE_TO_CALLS",
     "runner": "DataflowRunner",
     "worker_machine_type": "n1-standard-16",

gcp_variant_transforms/vcf_to_bq.py

@@ -83,6 +83,7 @@
 _ANNOTATE_FILES_JOB_NAME = 'annotate-files'
 _SHARD_VCF_FILES_JOB_NAME = 'shard-files'
 _SHARDS_FOLDER = 'shards'
+_GCS_RECURSIVE_WILDCARD = '**'
 
 
 def _read_variants(all_patterns, # type: List[str]
@@ -194,7 +195,8 @@ def _shard_variants(known_args, pipeline_args, pipeline_mode):
              beam.pvalue.AsSingleton(call_names),
              known_args.number_of_variants_per_shard))
 
-  return [vep_runner_util.format_dir_path(vcf_shards_output_dir)]
+  return [vep_runner_util.format_dir_path(vcf_shards_output_dir) +
+          _GCS_RECURSIVE_WILDCARD]
 
 
 def _annotate_vcf_files(all_patterns, known_args, pipeline_args):

gcp_variant_transforms/vcf_to_bq_test.py

@@ -18,9 +18,9 @@
 import unittest
 
 from gcp_variant_transforms import vcf_to_bq
+from gcp_variant_transforms.libs.variant_merge import move_to_calls_strategy
 from gcp_variant_transforms.options.variant_transform_options import MergeOptions
 from gcp_variant_transforms.vcf_to_bq import _get_variant_merge_strategy
-from gcp_variant_transforms.libs.variant_merge import move_to_calls_strategy
 
 
 class VcfToBqTest(unittest.TestCase):
@@ -56,7 +56,7 @@ def test_valid_merge_strategy(self):
                           move_to_calls_strategy.MoveToCallsStrategy)
 
   def test_invalid_annotation_output_directory_raises_error(self):
-    known_args = self._create_mock_args(annotation_output_dir='*')
+    known_args = self._create_mock_args(annotation_output_dir='./*')
     pipeline_args = []
     with self.assertRaisesRegexp(ValueError, 'directory .* already exists'):
       vcf_to_bq._validate_annotation_pipeline_args(known_args, pipeline_args)

setup.py

@@ -18,8 +18,7 @@
 
 REQUIRED_PACKAGES = [
     'cython>=0.28.1',
-    # TODO(bashir2): Drop the <=2.4 condition once the build is fixed with 2.5.
-    'apache-beam[gcp]>=2.3,<=2.4',
+    'apache-beam[gcp]',
     # Note that adding 'google-api-python-client>=1.6' causes some dependency
     # mismatch issues. This is fatal if using 'setup.py install', but works on
     # 'pip install .' as it ignores conflicting versions. See Issue #71.
@@ -30,16 +29,11 @@
     # Nucleus needs uptodate protocol buffer compiler (protoc).
     'protobuf>=3.6.1',
     'mmh3<2.6',
-    # Need to explicitly install v<=1.2.0. apache-beam requires
-    # google-cloud-pubsub 0.26.0, which relies on google-cloud-core<0.26dev,
-    # >=0.25.0. google-cloud-storage also has requirements on google-cloud-core,
-    # and version 1.2.0 resolves the dependency conflicts.
-    'google-cloud-storage<=1.2.0'
-]
-
-INTEGRATION_TEST_REQUIREMENTS = [
-    # Need to explicitly install v>0.25 as the BigQuery python API has changed.
-    'google-cloud-bigquery>0.25'
+    # Need to explicitly install v<=1.13.0. apache-beam requires
+    # google-cloud-pubsub 0.39.0, which relies on google-cloud-core<0.30dev,
+    # >=0.29.0. google-cloud-storage also has requirements on google-cloud-core,
+    # and version 1.13.0 resolves the dependency conflicts.
+    'google-cloud-storage<=1.13.0'
 ]
 
 REQUIRED_SETUP_PACKAGES = [
@@ -69,9 +63,6 @@
 
     setup_requires=REQUIRED_SETUP_PACKAGES,
     install_requires=REQUIRED_PACKAGES,
-    extras_require={
-        'int_test': INTEGRATION_TEST_REQUIREMENTS,
-    },
     test_suite='nose.collector',
     packages=setuptools.find_packages(),
     package_data={