Merge branch '396_alternative_android_module_license_output' into develop

Author: chinyeungli

REFERENCE.rst

@@ -151,6 +151,10 @@ gen
 
 ::
 
+    --android                           Generate MODULE_LICENSE_XXX (XXX will be
+                                        replaced by license key) and NOTICE as the same
+                                        design as from Android.
+
     --fetch-license api_url api_key     Fetch licenses data from DejaCode License
                                         Library and create <license>.LICENSE
                                         side-by-side with the generated .ABOUT file.
@@ -178,6 +182,17 @@ Options
 
 ::
 
+    --android
+
+        Create an empty file named `MODULE_LICENSE_XXX` where `XXX` is the license
+        key and create a NOTICE file which these two files follow the design from
+        Android Open Source Project.
+
+        The input **must** have the license key information as this is needed to
+        create the empty MODULE_LICENSE_XXX
+
+    $ about gen --android LOCATION OUTPUT
+
     --fetch-license
 
         Fetch licenses text from a DejaCode API. and create <license>.LICENSE side-by-side

src/attributecode/cmd.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -46,7 +46,7 @@
 
 
 __copyright__ = """
-    Copyright (c) 2013-2019 nexB Inc and others. All rights reserved.
+    Copyright (c) 2013-2020 nexB Inc and others. All rights reserved.
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
@@ -210,6 +210,11 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
     metavar='OUTPUT',
     type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True))
 
+@click.option('--android',
+    is_flag=True,
+    help='Generate MODULE_LICENSE_XXX (XXX will be replaced by license key) and NOTICE '
+         'as the same design as from Android.')
+
 # FIXME: the CLI UX should be improved with two separate options for API key and URL
 @click.option('--fetch-license',
     nargs=2,
@@ -233,7 +238,7 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
 
 @click.help_option('-h', '--help')
 
-def gen(location, output, fetch_license, reference, quiet, verbose):
+def gen(location, output, android, fetch_license, reference, quiet, verbose):
     """
 Generate .ABOUT files in OUTPUT from an inventory of .ABOUT files at LOCATION.
 
@@ -245,12 +250,14 @@ def gen(location, output, fetch_license, reference, quiet, verbose):
         print_version()
         click.echo('Generating .ABOUT files...')
 
+    #FIXME: This should be checked in the `click`
     if not location.endswith(('.csv', '.json',)):
         raise click.UsageError('ERROR: Invalid input file extension: must be one .csv or .json.')
 
     errors, abouts = generate_about_files(
         location=location,
         base_dir=output,
+        android=android,
         reference_dir=reference,
         fetch_license=fetch_license,
     )

src/attributecode/gen.py

@@ -210,12 +210,13 @@ def load_inventory(location, base_dir, reference_dir=None):
 def update_about_resource(self):
     pass
 
-def generate(location, base_dir, reference_dir=None, fetch_license=False):
+def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False):
     """
     Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
     base_dir. Return errors and about objects.
     """
     not_exist_errors = []
+    notice_dict = {}
     api_url = ''
     api_key = ''
     gen_license = False
@@ -312,6 +313,21 @@ def generate(location, base_dir, reference_dir=None, fetch_license=False):
 
             about.dump(dump_loc)
 
+            if android:
+                """
+                Create MODULE_LICENSE_XXX and get context to create NOTICE file
+                follow the standard from Android Open Source Project
+                """
+                import os
+                parent_path = os.path.dirname(util.to_posix(dump_loc))
+
+                about.android_module_license(parent_path)
+                notice_path, notice_context = about.android_notice(parent_path)
+                if notice_path in notice_dict.keys():
+                    notice_dict[notice_path] += '\n\n' + notice_context
+                else:
+                    notice_dict[notice_path] = notice_context
+
             for e in not_exist_errors:
                 errors.append(Error(INFO, e))
 
@@ -323,4 +339,14 @@ def generate(location, base_dir, reference_dir=None, fetch_license=False):
                    u'%(dump_loc)s '
                    u'with error: %(emsg)s' % locals())
             errors.append(Error(ERROR, msg))
+
+    if android:
+        # Check if there is already a NOTICE file present
+        for path in notice_dict.keys():
+            if os.path.exists(path):
+                msg = (u'NOTICE file already exist at: %s' % path)
+                errors.append(Error(ERROR, msg))
+            else:
+                about.dump_android_notice(path, notice_dict[path])
+
     return unique(errors), abouts

src/attributecode/model.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf8 -*-
 # ============================================================================
-#  Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -677,7 +677,7 @@ def validate_fields(fields, about_file_path, running_inventory, base_dir,
 def validate_field_name(name):
     if not is_valid_name(name):
         msg = ('Field name: %(name)r contains illegal name characters: '
-               '0 to 9, a to z, A to Z and _.')
+               '0 to 9, a to z, A to Z and _. (or empty spaces)')
         return Error(CRITICAL, msg % locals())
 
 
@@ -1095,6 +1095,53 @@ def dump(self, location):
             dumped.write(genereated_tk_version)
             dumped.write(self.dumps())
 
+    def dump_android_notice(self, path, context):
+        """
+        Write the NOITCE file consist of copyright, notice and license
+        """
+        if on_windows:
+            path = add_unc(path)
+
+        with io.open(path, mode='w', encoding='utf-8') as dumped:
+            dumped.write(context)
+
+    def android_module_license(self, about_parent_path):
+        """
+        Create MODULE_LICENSE_XXX which the XXX is the value of license key.
+        """
+        for lic_key in self.license_key.value:
+            # Make uppercase and with dash and spaces and dots replaced by underscore
+            # just to look similar and consistent.
+            name = 'MODULE_LICENSE_' + lic_key.replace('.', '_').replace('-', '_').replace(' ', '_').upper()
+            module_lic_path = os.path.join(about_parent_path, name)
+            # Create an empty MODULE_LICESE_XXX file
+            open(module_lic_path, 'a').close()
+
+    def android_notice(self, about_parent_path):
+        """
+        Return a notice dictionary which the path of the notice file going
+        to create will be the key and its context will be the value of the dict.
+        """
+        # Create NOTICE file with the combination context of copyright,
+        # notice_file and license_file
+        notice_path = posixpath.join(about_parent_path, 'NOTICE')
+        notice_context = ''
+        if self.copyright.value:
+            notice_context += self.copyright.value
+        if self.notice_file.value:
+            notice_file_dict = self.notice_file.value
+            notice_file_key = notice_file_dict.keys()
+            for key in notice_file_key:
+                if notice_file_dict[key]:
+                    notice_context += '\n' + notice_file_dict[key] + '\n'
+        if self.license_file.value:
+            lic_file_dict = self.license_file.value
+            lic_file_key = lic_file_dict.keys()
+            for key in lic_file_key:
+                if lic_file_dict[key]:
+                    notice_context += '\n\n' + lic_file_dict[key] + '\n\n'
+        return notice_path, notice_context
+
     def dump_lic(self, location, license_dict):
         """
         Write LICENSE files and return the a list of key, name, context and the url

tests/test_model.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2014-2020 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -21,6 +21,7 @@
 from collections import OrderedDict
 import io
 import json
+import os
 import posixpath
 import shutil
 import unittest
@@ -40,6 +41,7 @@
 from attributecode.util import replace_tab_with_spaces
 
 from testing_utils import extract_test_loc
+from testing_utils import get_temp_dir
 from testing_utils import get_temp_file
 from testing_utils import get_test_loc
 
@@ -582,7 +584,7 @@ def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self):
         test_file = get_test_loc('test_model/parse/non_ascii_field_name_value.about')
         a = model.About(test_file)
         expected = [
-            Error(CRITICAL, "Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _.")
+            Error(CRITICAL, "Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _. (or empty spaces)")
         ]
         assert expected == a.errors
 
@@ -971,6 +973,41 @@ def test_write_output_json(self):
         expected = get_test_loc('test_model/expected.json')
         check_json(expected, result)
 
+    def test_android_module_license(self):
+        path = 'test_model/android/single_license.c.ABOUT'
+        test_file = get_test_loc(path)
+        abouts = model.About(location=test_file, about_file_path=path)
+
+        parent_dir = get_temp_dir()
+        abouts.android_module_license(parent_dir)
+        assert os.path.exists(os.path.join(parent_dir, 'MODULE_LICENSE_PUBLIC_DOMAIN'))
+
+    def test_android_module_multi_licenses(self):
+        path = 'test_model/android/multi_license.c.ABOUT'
+        test_file = get_test_loc(path)
+        abouts = model.About(location=test_file, about_file_path=path)
+
+        parent_dir = get_temp_dir()
+        abouts.android_module_license(parent_dir)
+        assert os.path.exists(os.path.join(parent_dir, 'MODULE_LICENSE_BSD_NEW'))
+        assert os.path.exists(os.path.join(parent_dir, 'MODULE_LICENSE_BSD_SIMPLIFIED'))
+
+    def test_android_notice(self):
+        path = 'test_model/android/single_license.c.ABOUT'
+        test_file = get_test_loc(path)
+        abouts = model.About(location=test_file, about_file_path=path)
+
+        parent_dir = get_temp_dir()
+        notice_path, notice_context = abouts.android_notice(parent_dir)
+        expected_path = os.path.join(parent_dir, 'NOTICE')
+        assert os.path.normpath(notice_path) == expected_path
+
+        expected_notice = '''Copyright (c) xyz
+
+This component is released to the public domain by the author.
+
+'''
+        assert notice_context == expected_notice
 
 class CollectorTest(unittest.TestCase):
 

tests/testdata/test_cmd/help/about_gen_help.txt

@@ -8,6 +8,9 @@ Usage: about gen [OPTIONS] LOCATION OUTPUT
   OUTPUT: Path to a directory where ABOUT files are generated.
 
 Options:
+  --android                Generate MODULE_LICENSE_XXX (XXX will be replaced by
+                           license key) and NOTICE as the same design as from
+                           Android.
   --fetch-license URL KEY  Fetch license data and text files from a DejaCode
                            License Library API URL using the API KEY.
   --reference DIR          Path to a directory with reference license data and

tests/testdata/test_model/android/expected_NOTICE

@@ -0,0 +1,3 @@
+Copyright (c) xyz
+
+This component is released to the public domain by the author.

tests/testdata/test_model/android/multi_license.c.ABOUT

@@ -0,0 +1,17 @@
+# Generated with AboutCode Toolkit Version 4.0.1 
+
+about_resource: multi_license.c
+name: multi_license.c
+license_expression: bsd-new OR bsd-simplified
+copyright: |
+  Copyright (c) xyz
+  Copyright (c) Blah blah BlAh
+licenses:
+  - key: bsd-new
+    name: BSD-3-Clause
+    file: bsd-new.LICENSE
+    url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new
+  - key: bsd-simplified
+    name: BSD-2-Clause
+    file: bsd-simplified.LICENSE
+    url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-simplified

tests/testdata/test_model/android/public-domain.LICENSE

@@ -0,0 +1 @@
+This component is released to the public domain by the author.

tests/testdata/test_model/android/single_license.c.ABOUT

@@ -0,0 +1,11 @@
+# Generated with AboutCode Toolkit Version 4.0.1 
+
+about_resource: single_license.c
+name: single_license.c
+license_expression: public-domain
+copyright: Copyright (c) xyz
+licenses:
+  - key: public-domain
+    name: Public Domain
+    file: public-domain.LICENSE
+    url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain