Collect and handle the "matched_text" from the --license-text option from the scancode-toolkit

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -1,12 +1,13 @@
 ==============================
 Changelog
 ==============================
-2022-09-21
+2022-xx-xx
     Release 7.1.0
 
     * Fixed version mismatch (https://github.com/nexB/aboutcode-toolkit/issues/510)
     * Improve `check` performance (https://github.com/nexB/aboutcode-toolkit/issues/511)
     * Relax the requirement to have the same format for input and output for `transform`
+    * Collect and handle the "matched_text" from the `--license-text` option from the scancode-toolkit
 
 
 2022-03-21

src/attributecode/attrib.py

@@ -114,41 +114,69 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
     if scancode:
         meet_score_licenses_list = []
         for about in abouts:
+            # See if the input has 'matched_text'
+            matched_text_exist = False
+            try:
+                if about.matched_text:
+                    matched_text_exist = True
+            except:
+                pass
             # We will use a dictionary to keep the unique license key
             # which the dictionary key is the license key and the dictionary value
-            # is (lic_score, lic_name)
+            # is (lic_score, lic_name) or (lic_score, lic_name, matched_text)
             if about.license_key.value:
                 updated_dict = {}
                 lic_key = about.license_key.value
                 lic_name = about.license_name.value
                 lic_score = about.license_score.value
+                if matched_text_exist:
+                    matched_text = about.matched_text.value
+                    assert len(lic_key) == len(matched_text)
                 assert len(lic_key) == len(lic_name)
                 assert len(lic_key) == len(lic_score)
                 if lic_key:
                     index = 0
                     for key in lic_key:
                         if key in updated_dict:
-                            previous_score, _name = updated_dict[key]
+                            if matched_text_exist:
+                                previous_score, _name, _detected_text = updated_dict[key]
+                            else:
+                                previous_score, _name = updated_dict[key]
                             current_score = lic_score[index]
                             if current_score > previous_score:
-                                updated_dict[key] = (lic_score[index], lic_name[index])
+                                if matched_text_exist:
+                                    updated_dict[key] = (lic_score[index], lic_name[index], matched_text[index])
+                                else:
+                                    updated_dict[key] = (lic_score[index], lic_name[index])
                         else:
-                            updated_dict[key] = (lic_score[index], lic_name[index])
+                            if matched_text_exist:
+                                updated_dict[key] = (lic_score[index], lic_name[index], matched_text[index])
+                            else:
+                                updated_dict[key] = (lic_score[index], lic_name[index])
                         index = index + 1
                 updated_lic_key = []
                 updated_lic_name = []
                 updated_lic_score = []
+                if matched_text_exist:
+                    updated_matched_text = []
                 for lic in updated_dict:
-                    score, name = updated_dict[lic]
+                    if matched_text_exist:
+                        score, name, text = updated_dict[lic]
+                    else:
+                        score, name = updated_dict[lic]
                     if score >= min_license_score:
                         updated_lic_key.append(lic)
                         updated_lic_score.append(score)
                         updated_lic_name.append(name)
+                        if matched_text_exist:
+                            updated_matched_text.append(text)
                         if not lic in meet_score_licenses_list:
                             meet_score_licenses_list.append(lic)
                 about.license_key.value = updated_lic_key
                 about.license_name.value = updated_lic_name
                 about.license_score.value = updated_lic_score
+                if matched_text_exist:
+                    about.matched_text.value = updated_matched_text
 
         for lic in licenses_list:
             if not lic.key in meet_score_licenses_list:
@@ -180,7 +208,6 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score,
 
     # Sort the license object by key
     licenses_list = sorted(licenses_list, key=lambda x: x.key)
-
     rendered = template.render(
         abouts=abouts,
         common_licenses=COMMON_LICENSES,

src/attributecode/gen.py

@@ -235,7 +235,6 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r
             running_inventory=False,
             reference_dir=reference_dir,
         )
-
         for severity, message in ld_errors:
             if 'Custom Field' in message:
                 field_name = message.replace('Custom Field: ', '').strip()

src/attributecode/model.py

@@ -1051,7 +1051,7 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru
                 continue
             if key == u'licenses':
                 # FIXME: use a license object instead
-                lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score = ungroup_licenses(value)
+                lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, lic_matched_text = ungroup_licenses(value)
                 if lic_key:
                     fields.append(('license_key', lic_key))
                 if lic_name:
@@ -1067,6 +1067,8 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru
                 # The license score is a key from scancode license scan
                 if lic_score:
                     fields.append(('license_score', lic_score))
+                if lic_matched_text:
+                    fields.append(('matched_text', lic_matched_text))
                 # The licenses field has been ungrouped and can be removed.
                 # Otherwise, it will gives the following INFO level error
                 # 'Field licenses is a custom field.'
@@ -1660,7 +1662,7 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a
         url = 'https://scancode-licensedb.aboutcode.org/'
     if util.have_network_connection():
         if not valid_api_url(url):
-            msg = u"URL not reachable. Invalid 'URL'. License generation is skipped."
+            msg = u"URL not reachable. Invalid 'URL. License generation is skipped."
             errors.append(Error(ERROR, msg))
     else:
         msg = u'Network problem. Please check your Internet connection. License generation is skipped.'

src/attributecode/util.py

@@ -472,6 +472,7 @@ def ungroup_licenses(licenses):
     lic_url = []
     spdx_lic_key = []
     lic_score = []
+    lic_matched_text = []
     for lic in licenses:
         if 'key' in lic:
             lic_key.append(lic['key'])
@@ -485,7 +486,9 @@ def ungroup_licenses(licenses):
             spdx_lic_key.append(lic['spdx_license_key'])
         if 'score' in lic:
             lic_score.append(lic['score'])
-    return lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score
+        if 'matched_text' in lic:
+            lic_matched_text.append(lic['matched_text'])
+    return lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, lic_matched_text
 
 
 # FIXME: add docstring

tests/test_util.py

@@ -572,7 +572,7 @@ def test_ungroup_licenses(self):
             u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit',
             u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new']
         expected_spdx = [u'MIT', u'BSD-3-Clause']
-        lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score = util.ungroup_licenses(about)
+        lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, _matched_text = util.ungroup_licenses(about)
         assert expected_lic_key == lic_key
         assert expected_lic_name == lic_name
         assert expected_lic_file == lic_file