Use saneyaml library

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

setup.py

@@ -70,7 +70,8 @@ def read(*names, **kwargs):
         'jinja2 >= 2.9, < 3.0',
         'click >= 6.7, < 7.0',
         "backports.csv ; python_version<'3.6'",
-        'PyYAML >= 3.0, < 4.0',
+        'PyYAML >= 3.11, <=3.13',
+        'saneyaml',
         'boolean.py >= 3.5, < 4.0',
         'license_expression >= 0.94, < 1.0',
     ],

src/attributecode/model.py

@@ -53,14 +53,14 @@
     from urllib.error import HTTPError  # NOQA
 
 from license_expression import Licensing
+import saneyaml
 
 from attributecode import CRITICAL
 from attributecode import ERROR
 from attributecode import INFO
 from attributecode import WARNING
 from attributecode import api
 from attributecode import Error
-from attributecode import saneyaml
 from attributecode import util
 from attributecode.util import add_unc
 from attributecode.util import copy_license_notice_files
@@ -1015,8 +1015,6 @@ def load(self, location, use_mapping=False, mapping_file=None):
             loc = add_unc(loc)
             with codecs.open(loc, encoding='utf-8') as txt:
                 input_text = txt.read()
-            # Check for duplicated key
-            yaml.load(input_text, Loader=util.NoDuplicateLoader)
             """
             The running_inventory defines if the current process is 'inventory' or not.
             This is used for the validation of the path of the 'about_resource'.
@@ -1030,7 +1028,9 @@ def load(self, location, use_mapping=False, mapping_file=None):
             # wrap the value of the boolean field in quote to avoid
             # automatically conversion from yaml.load
             input = util.wrap_boolean_value(input_text)  # NOQA
-            errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, use_mapping, mapping_file)
+            data = saneyaml.load(input, allow_duplicate_keys=False)
+
+            errs = self.load_dict(data, base_dir, running_inventory, use_mapping, mapping_file)
             errors.extend(errs)
         except Exception as e:
             msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e)
@@ -1121,7 +1121,8 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
                 lic_dict['url'] = lic_group[3]
             about_data.setdefault('licenses', []).append(lic_dict)
         formatted_about_data = util.format_output(about_data, use_mapping, mapping_file)
-        return saneyaml.dump(formatted_about_data)
+
+        return saneyaml.dump(formatted_about_data, indent=2)
 
     def dump(self, location, use_mapping=False, mapping_file=False, with_absent=False, with_empty=True):
         """

src/attributecode/saneyaml.py

@@ -70,12 +70,10 @@ def test_load_inventory(self):
             else:
                 assert e.message in expected_error_messages
 
-        expected = [u'about_resource: .\n'
-                    u'name: AboutCode\n'
-                    u'version: 0.11.0\n'
-                    u'description: |-\n'
-                    u'    multi\n'
-                    u'    line\n']
+        expected = ['about_resource: .\n'
+                    'name: AboutCode\n'
+                    'version: 0.11.0\n'
+                    'description: |\n  multi\n  line\n']
         result = [a.dumps(use_mapping=False, mapping_file=False, with_absent=False, with_empty=False)
                         for a in abouts]
         assert expected == result
@@ -101,15 +99,14 @@ def test_load_inventory_with_mapping(self):
             else:
                 assert e.message in expected_error_messages
 
-        expected = [u'about_resource: .\n'
-                    u'name: AboutCode\n'
-                    u'version: 0.11.0\n'
-                    u'copyright: Copyright (c) nexB, Inc.\n'
-                    u'resource: this.ABOUT\n'
-                    u'description: |-\n'
-                    u'    multi\n'
-                    u'    line\n'
-                    ]
+        expected = [
+            'about_resource: .\n'
+            'name: AboutCode\n'
+            'version: 0.11.0\n'
+            'copyright: Copyright (c) nexB, Inc.\n'
+            'resource: this.ABOUT\n'
+            'description: |\n  multi\n  line\n'
+        ]
         result = [a.dumps(use_mapping, mapping_file=False, with_absent=False, with_empty=False)
                         for a in abouts]
         assert expected == result
@@ -171,9 +168,9 @@ def test_generate(self):
         expected = (u'about_resource: .\n'
                     u'name: AboutCode\n'
                     u'version: 0.11.0\n'
-                    u'description: |-\n'
-                    u'    multi\n'
-                    u'    line\n')
+                    u'description: |\n'
+                    u'  multi\n'
+                    u'  line\n')
         assert expected == in_mem_result
 
     def test_generate_not_overwrite_original_license_file(self):
@@ -190,7 +187,7 @@ def test_generate_not_overwrite_original_license_file(self):
                     u'name: AboutCode\n'
                     u'version: 0.11.0\n'
                     u'licenses:\n'
-                    u'    -   file: this.LICENSE\n')
+                    u'  - file: this.LICENSE\n')
         assert expected == in_mem_result
 
     def test_deduplicate(self):

tests/test_gen.py

@@ -709,24 +709,24 @@ def test_About_dumps(self):
         a = model.About(test_file)
         assert [] == a.errors
 
-        expected = u'''about_resource: .
+        expected = '''about_resource: .
 name: AboutCode
 version: 0.11.0
 copyright: Copyright (c) 2013-2014 nexB Inc.
 license_expression: apache-2.0
 author:
-    - Jillian Daguil
-    - Chin Yeung Li
-    - Philippe Ombredanne
-    - Thomas Druez
-description: |-
-    AboutCode is a tool
-    to process ABOUT files.
-    An ABOUT file is a file.
+  - Jillian Daguil
+  - Chin Yeung Li
+  - Philippe Ombredanne
+  - Thomas Druez
+description: |
+  AboutCode is a tool
+  to process ABOUT files.
+  An ABOUT file is a file.
 homepage_url: http://dejacode.org
 licenses:
-    -   file: apache-2.0.LICENSE
-        key: apache-2.0
+  - file: apache-2.0.LICENSE
+    key: apache-2.0
 notice_file: NOTICE
 owner: nexB Inc.
 vcs_repository: https://github.com/dejacode/about-code-tool.git

tests/test_model.py

@@ -0,0 +1,20 @@
+about_resource: saneyaml-0.1-py2.py3-none-any.whl
+attribute: true
+checksum_md5: 53509e4f1ee9f6565158163a56513b7c
+checksum_sha1: a66920309794ead47711473a21f1c7d003acd005
+contact: http://www.nexb.com/contactus.html
+copyright: Copyright (c) nexB Inc. and others.
+download_url: https://files.pythonhosted.org/packages/21/13/3d639adcd97fc22cfc4acfd23330cc26f96a85a5b992d93086684e24ac14/saneyaml-0.1-py2.py3-none-any.whl
+homepage_url: https://github.com/nexB/saneyaml
+license_expression: apache-2.0
+licenses:
+-   file: apache-2.0.LICENSE
+    key: apache-2.0
+    name: Apache License 2.0
+name: saneyaml
+owner: nexB
+owner_url: http://www.nexb.com/
+package_url: pkg:pypi/[email protected]
+track_changes: true
+vcs_repository: https://github.com/nexB/saneyaml
+version: '0.1'