Merge branch 'develop'

Author: chinyeungli

NOTICE

@@ -1,4 +1,4 @@
- Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
+ Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved.
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

REFERENCE.rst

@@ -151,6 +151,10 @@ gen
 
 ::
 
+    --android                           Generate MODULE_LICENSE_XXX (XXX will be
+                                        replaced by license key) and NOTICE as the same
+                                        design as from Android.
+
     --fetch-license api_url api_key     Fetch licenses data from DejaCode License
                                         Library and create <license>.LICENSE
                                         side-by-side with the generated .ABOUT file.
@@ -178,6 +182,17 @@ Options
 
 ::
 
+    --android
+
+        Create an empty file named `MODULE_LICENSE_XXX` where `XXX` is the license
+        key and create a NOTICE file which these two files follow the design from
+        Android Open Source Project.
+
+        The input **must** have the license key information as this is needed to
+        create the empty MODULE_LICENSE_XXX
+
+    $ about gen --android LOCATION OUTPUT
+
     --fetch-license
 
         Fetch licenses text from a DejaCode API. and create <license>.LICENSE side-by-side
@@ -324,7 +339,7 @@ Options
         Show configuration file format help and exit.
         This option will print out examples of the the YAML configuration file.
         
-        Keys configuration are: `column_renamings`, `required_columns` and `column_filters`
+        Keys configuration are: `field_renamings`, `required_fields` and `field_filters`
 
     $ about transform --help-format
 
@@ -335,5 +350,5 @@ Options
 
 Special Notes
 =============
-When using the `column_filters` configuration, all the standard required columns
-(`about_resource` and `name`) and the user defined `required_columns` need to be included.
+When using the `field_filters` configuration, all the standard required columns
+(`about_resource` and `name`) and the user defined `required_fields` need to be included.

about.ABOUT

@@ -1,6 +1,6 @@
 about_resource: .
 name: AboutCode-toolkit
-version: 4.0.2
+version: 5.0.0
 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
 copyright: Copyright (c) 2013-2020 nexB Inc.
 description: |

docs/CHANGELOG.rst

@@ -1,3 +1,14 @@
+2020-08-11
+    Release 5.0.0
+
+	* Enhance the `transform` to also work with JSON file
+    * Update transform code (See #427 and #428)
+    * Fixed #431 - Error handling for empty "_file" fields
+    * Fixed #432 - Handled UTF-8 variant invented by Microsoft
+    * Fixed #433 - problem was caused by the different multi-lic file format between json and CSV (CSV with '\n' line break)
+    * Fixed #436 - issue about copy with the `--reference` option
+    * Fixed #396 - support for alternative output for Android
+
 2020-05-05
     Release 4.0.2
 

docs/UsingAboutCodetoDocumentYourSoftwareAssets.md

@@ -245,22 +245,22 @@ A transform configuration file is used to describe which transformations and val
 
 The attributes that can be set in a configuration file are:
 
-* column_renamings:
+* field_renamings:
 An optional map of source CSV column name to target CSV new column name that
 is used to rename CSV columns.
 
 For instance with this configuration the columns "Directory/Location" will be
 renamed to "about_resource" and "foo" to "bar":
 
-    column_renamings:
-        'Directory/Location' : about_resource
-        foo : bar
+    field_renamings:
+        about_resource : 'Directory/Location'
+        bar : foo
 
 The renaming is always applied first before other transforms and checks. All
 other column names referenced below are these that exist AFTER the renaming
 have been applied to the existing column names.
 
-* required_columns:
+* required_fields:
 An optional list of required column names that must have a value, beyond the
 standard columns names. If a source CSV does not have such a column or a row is
 missing a value for a required column, an error is reported.
@@ -269,11 +269,11 @@ For instance with this configuration an error will be reported if the columns
 "name" and "version" are missing or if any row does not have a value set for
 these columns:
 
-    required_columns:
+    required_fields:
         - name
         - version
 
-* column_filters:
+* field_filters:
 An optional list of column names that should be kept in the transformed CSV. If
 this list is provided, all the columns from the source CSV that should be kept
 in the target CSV must be listed be even if they are standard or required
@@ -283,10 +283,24 @@ transformed target CSV.
 For instance with this configuration the target CSV will only contains the "name"
 and "version" columns and no other column:
 
-    column_filters:
+    field_filters:
         - name
         - version
 
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+
+    exclude_fields:
+        - type
+        - temp
+
 
 ## <a name="RungentoGenerateAboutCodeToolkitFiles">Run gen to Generate AboutCode Toolkit Files</a>
 

docs/UsingAboutCodetoDocumentYourSoftwareAssets.pdf

@@ -139,7 +139,7 @@ def appveyor_vars():
         response = line.split()
 
         if response[0] == 'PING':
-            irc_file.send('PONG {}\r\n'.format(reponse[1]).encode())
+            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
 
         elif response[1] == '433':
             irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())

etc/scripts/irc-notify.py

@@ -24,7 +24,7 @@ def read(*names, **kwargs):
 
 setup(
     name='aboutcode-toolkit',
-    version='4.0.2',
+    version='5.0.0',
     license='Apache-2.0',
     description=(
         'AboutCode-toolkit is a tool to document the provenance (origin and license) of '
@@ -69,7 +69,7 @@ def read(*names, **kwargs):
     install_requires=[
         'jinja2 >= 2.9, < 3.0',
 
-        'click >= 6.7, < 7.0',
+        'click',
 
         "backports.csv ; python_version<'3.6'",
 

setup.py

@@ -31,7 +31,7 @@
 
 import saneyaml
 
-__version__ = '4.0.2'
+__version__ = '5.0.0'
 
 __about_spec_version__ = '3.1.4'
 

src/attributecode/__init__.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -46,7 +46,7 @@
 
 
 __copyright__ = """
-    Copyright (c) 2013-2019 nexB Inc and others. All rights reserved.
+    Copyright (c) 2013-2020 nexB Inc and others. All rights reserved.
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
@@ -210,6 +210,11 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
     metavar='OUTPUT',
     type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True))
 
+@click.option('--android',
+    is_flag=True,
+    help='Generate MODULE_LICENSE_XXX (XXX will be replaced by license key) and NOTICE '
+         'as the same design as from Android.')
+
 # FIXME: the CLI UX should be improved with two separate options for API key and URL
 @click.option('--fetch-license',
     nargs=2,
@@ -233,7 +238,7 @@ def inventory(location, output, format, quiet, verbose):  # NOQA
 
 @click.help_option('-h', '--help')
 
-def gen(location, output, fetch_license, reference, quiet, verbose):
+def gen(location, output, android, fetch_license, reference, quiet, verbose):
     """
 Generate .ABOUT files in OUTPUT from an inventory of .ABOUT files at LOCATION.
 
@@ -245,12 +250,14 @@ def gen(location, output, fetch_license, reference, quiet, verbose):
         print_version()
         click.echo('Generating .ABOUT files...')
 
+    #FIXME: This should be checked in the `click`
     if not location.endswith(('.csv', '.json',)):
         raise click.UsageError('ERROR: Invalid input file extension: must be one .csv or .json.')
 
     errors, abouts = generate_about_files(
         location=location,
         base_dir=output,
+        android=android,
         reference_dir=reference,
         fetch_license=fetch_license,
     )
@@ -401,17 +408,17 @@ def print_config_help(ctx, param, value):
 
 
 @about.command(cls=AboutCommand,
-    short_help='Transform a CSV by applying renamings, filters and checks.')
+    short_help='Transform a CSV/JSON by applying renamings, filters and checks.')
 
 @click.argument('location',
     required=True,
-    callback=partial(validate_extensions, extensions=('.csv',)),
+    callback=partial(validate_extensions, extensions=('.csv', '.json',)),
     metavar='LOCATION',
     type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True))
 
 @click.argument('output',
     required=True,
-    callback=partial(validate_extensions, extensions=('.csv',)),
+    callback=partial(validate_extensions, extensions=('.csv', '.json',)),
     metavar='OUTPUT',
     type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True))
 
@@ -438,30 +445,39 @@ def print_config_help(ctx, param, value):
 
 def transform(location, output, configuration, quiet, verbose):  # NOQA
     """
-Transform the CSV file at LOCATION by applying renamings, filters and checks
-and write a new CSV to OUTPUT.
+Transform the CSV/JSON file at LOCATION by applying renamings, filters and checks
+and write a new CSV/JSON to OUTPUT.
 
-LOCATION: Path to a CSV file.
+LOCATION: Path to a CSV/JSON file.
 
-OUTPUT: Path to CSV inventory file to create.
+OUTPUT: Path to CSV/JSON inventory file to create.
     """
     from attributecode.transform import transform_csv_to_csv
+    from attributecode.transform import transform_json_to_json
     from attributecode.transform import Transformer
 
-    if not quiet:
-        print_version()
-        click.echo('Transforming CSV...')
 
     if not configuration:
         transformer = Transformer.default()
     else:
         transformer = Transformer.from_file(configuration)
 
-    errors = transform_csv_to_csv(location, output, transformer)
+    if location.endswith('.csv') and output.endswith('.csv'):
+        errors = transform_csv_to_csv(location, output, transformer)
+    elif location.endswith('.json') and output.endswith('.json'):
+        errors = transform_json_to_json(location, output, transformer)
+    else:
+        msg = 'Extension for the input and output need to be the same.'
+        click.echo(msg)
+        sys.exit()
+
+    if not quiet:
+        print_version()
+        click.echo('Transforming...')
 
     errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log')
     if not quiet and not errors:
-        msg = 'Transformed CSV written to {output}.'.format(**locals())
+        msg = 'Transformed file written to {output}.'.format(**locals())
         click.echo(msg)
     sys.exit(errors_count)