Only scan for bianry packages optionally

Introduce a new option --binary-packages which looks for
package/dependency data in binaries.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

docs/source/rst_snippets/basic_options.rst

@@ -33,6 +33,11 @@ documenting a program's options. For example:
 --system-package             Scan ``<input>`` for installed system package
                              databases.
 
+-b, --binary-package         Scan <input> for package and dependency related
+                             data in binaries. Note that looking for packages
+                             in binaries makes package scan slower.
+                             Currently supported binaries: Go, Rust.
+
 --package-only               Scan ``<input>`` for system and application
                              only for package metadata, without license/
                              copyright detection and package assembly.

src/packagedcode/__init__.py

@@ -251,32 +251,27 @@
 # detect these binaries instead of datafile path patterns
 # as these are optionally installed, we can skip checking
 # for filetype if these are not available
-BINARY_HANDLERS_PRESENT = False
 BINARY_PACKAGE_DATAFILE_HANDLERS = []
 
 try:
     from go_inspector.binary import get_go_binary_handler
     handler = get_go_binary_handler()
-    APPLICATION_PACKAGE_DATAFILE_HANDLERS.append(handler)
     BINARY_PACKAGE_DATAFILE_HANDLERS.append(handler)
-    BINARY_HANDLERS_PRESENT = True
 except ImportError:
     pass
 
 try:
     from rust_inspector.packages import get_rust_binary_handler
     handler = get_rust_binary_handler()
-    APPLICATION_PACKAGE_DATAFILE_HANDLERS.append(handler)
     BINARY_PACKAGE_DATAFILE_HANDLERS.append(handler)
-    BINARY_HANDLERS_PRESENT = True
 except ImportError:
     pass
 
 ALL_DATAFILE_HANDLERS = (
     APPLICATION_PACKAGE_DATAFILE_HANDLERS + [
         p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS
         if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS
-    ]
+    ] + BINARY_PACKAGE_DATAFILE_HANDLERS
 )
 
 # registry of all handler classes keyed by datasource_id

src/packagedcode/plugin_package.py

@@ -170,6 +170,20 @@ class PackageScanner(ScanPlugin):
             help_group=SCAN_GROUP,
             sort_order=21,
         ),
+        PluggableCommandLineOption(
+            (
+                '-b',
+                '--binary-package',
+            ),
+            is_flag=True,
+            default=False,
+            help=(
+                'Scan <input> for package and dependency related data in binaries. '
+                'Currently supported binaries: Go, Rust.'
+            ),
+            help_group=SCAN_GROUP,
+            sort_order=22,
+        ),
         PluggableCommandLineOption(
             (
                 '--package-only',
@@ -182,7 +196,7 @@ class PackageScanner(ScanPlugin):
                 'license/copyright detection and top-level package creation.'
             ),
             help_group=SCAN_GROUP,
-            sort_order=22,
+            sort_order=23,
         ),
         PluggableCommandLineOption(
             ('--list-packages',),
@@ -195,10 +209,17 @@ class PackageScanner(ScanPlugin):
         ),
     ]
 
-    def is_enabled(self, package, system_package, package_only, **kwargs):
-        return package or system_package or package_only
+    def is_enabled(self, package, system_package, binary_package, package_only, **kwargs):
+        return package or system_package or binary_package or package_only
 
-    def get_scanner(self, package=True, system_package=False, package_only=False, **kwargs):
+    def get_scanner(
+        self,
+        package=True,
+        system_package=False,
+        binary_package=False,
+        package_only=False,
+        **kwargs
+    ):
         """
         Return a scanner callable to scan a file for package data.
         """
@@ -208,6 +229,7 @@ def get_scanner(self, package=True, system_package=False, package_only=False, **
             get_package_data,
             application=package,
             system=system_package,
+            binary=binary_package,
             package_only=package_only,
         )
 
@@ -464,7 +486,7 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                 resource.scan_errors.append(msg)
                 resource.save(codebase)
 
-                if TRACE:
+                if TRACE_ASSEMBLY:
                     raise Exception(msg) from e
 
     return packages, dependencies

src/packagedcode/recognize.py

@@ -14,7 +14,6 @@
 from commoncode.fileutils import as_posixpath
 
 from packagedcode import HANDLER_BY_DATASOURCE_ID
-from packagedcode import BINARY_HANDLERS_PRESENT
 from packagedcode import BINARY_PACKAGE_DATAFILE_HANDLERS
 from packagedcode import models
 from packagedcode.cache import get_cache
@@ -47,6 +46,7 @@ def recognize_package_data(
     location,
     application=True,
     system=False,
+    binary=False,
     package_only=False,
 ):
     """
@@ -61,16 +61,18 @@ def recognize_package_data(
 
     return list(_parse(
         location=location,
-        package_only=package_only,
         application=application,
         system=system,
+        binary=binary,
+        package_only=package_only,
     ))
 
 
 def _parse(
     location,
     application=True,
     system=False,
+    binary=False,
     package_only=False,
 ):
     """
@@ -83,30 +85,39 @@ def _parse(
     package_path = as_posixpath(location)
     package_patterns = get_cache()
 
-    assert application or system or package_only
+    has_patterns = application or system or package_only
+    assert has_patterns or binary
     if package_only or (application and system):
         package_matcher = package_patterns.all_package_matcher
     elif application:
         package_matcher = package_patterns.application_package_matcher
     elif system:
         package_matcher = package_patterns.system_package_matcher
 
-    matched_patterns = package_matcher.match(package_path)
+    matched_patterns = []
+    if has_patterns:
+        matched_patterns = package_matcher.match(package_path)
 
-    datafile_handlers = []
+    all_handler_ids = []
     for matched_pattern in matched_patterns:
         regex, _match = matched_pattern
         handler_ids = package_patterns.handler_by_regex.get(regex.pattern)
         if TRACE:
             logger_debug(f'_parse:.handler_ids: {handler_ids}')
 
-        datafile_handlers.extend([
-            HANDLER_BY_DATASOURCE_ID.get(handler_id)
+        all_handler_ids.extend([
+            handler_id
             for handler_id in handler_ids
+            if handler_id not in all_handler_ids
         ])
 
+    datafile_handlers = [
+        HANDLER_BY_DATASOURCE_ID.get(handler_id)
+        for handler_id in all_handler_ids
+    ]
+
     if not datafile_handlers:
-        if BINARY_HANDLERS_PRESENT:
+        if binary:
             datafile_handlers.extend(BINARY_PACKAGE_DATAFILE_HANDLERS)
         elif TRACE:
             logger_debug(f'_parse: no package datafile detected at {package_path}')

src/scancode/api.py

@@ -256,20 +256,28 @@ def get_licenses(
 SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
 
 
-def _get_package_data(location, application=True, system=False, package_only=False, **kwargs):
+def _get_package_data(
+    location,
+    application=True,
+    system=False,
+    binary=False,
+    package_only=False, 
+    **kwargs
+):
     """
     Return a mapping of package manifest information detected in the file at ``location``.
     Include ``application`` packages (such as pypi) and/or ``system`` packages.
     Note that all exceptions are caught if there are any errors while parsing a
     package manifest.
     """
-    assert application or system or package_only
+    assert application or system or binary or package_only
     from packagedcode.recognize import recognize_package_data
     try:
         return recognize_package_data(
             location=location,
             application=application,
             system=system,
+            binary=binary,
             package_only=package_only,
         ) or []
 
@@ -300,7 +308,14 @@ def get_package_info(location, **kwargs):
     return dict(packages=[p.to_dict() for p in packages])
 
 
-def get_package_data(location, application=True, system=False, package_only=False, **kwargs):
+def get_package_data(
+    location,
+    application=True,
+    system=False,
+    binary=False,
+    package_only=False,
+    **kwargs
+):
     """
     Return a mapping of package manifest information detected in the file at
     `location`.
@@ -313,6 +328,7 @@ def get_package_data(location, application=True, system=False, package_only=Fals
         location=location,
         application=application,
         system=system,
+        binary=binary,
         package_only=package_only,
         **kwargs,
     ) or []

tests/packagedcode/test_cargo.py

@@ -159,7 +159,7 @@ def test_scan_works_on_rust_binary_with_inspector(self):
         test_file = self.get_test_loc('cargo/binary/cargo_dependencies')
         expected_file = self.get_test_loc('cargo/binary/cargo-binary.expected.json')
         result_file = self.get_temp_file('results.json')
-        run_scan_click(['--package', test_file, '--json', result_file])
+        run_scan_click(['--binary-package', test_file, '--json', result_file])
         check_json_scan(
             expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
         )

tests/scancode/data/help/help_linux.txt

@@ -8,15 +8,18 @@ Usage: scancode [OPTIONS] <OUTPUT FORMAT OPTION(s)> <input>...
 Options:
 
   primary scans:
-    -l, --license     Scan <input> for licenses.
-    -p, --package     Scan <input> for application package and dependency
-                      manifests, lockfiles and related data.
-    --system-package  Scan <input> for installed system package databases.
-    --package-only    Scan for system and application package data and skip
-                      license/copyright detection and top-level package creation.
-    -c, --copyright   Scan <input> for copyrights.
-    --go-symbol       Collect Go symbols.
-    --rust-symbol     Collect Rust symbols from rust binaries.
+    -l, --license         Scan <input> for licenses.
+    -p, --package         Scan <input> for application package and dependency
+                          manifests, lockfiles and related data.
+    --system-package      Scan <input> for installed system package databases.
+    -b, --binary-package  Scan <input> for package and dependency related data in
+                          binaries. Currently supported binaries: Go, Rust.
+    --package-only        Scan for system and application package data and skip
+                          license/copyright detection and top-level package
+                          creation.
+    -c, --copyright       Scan <input> for copyrights.
+    --go-symbol           Collect Go symbols.
+    --rust-symbol         Collect Rust symbols from rust binaries.
 
   other scans:
     -i, --info   Scan <input> for file information (size, checksums, etc).