Cache multiregex matchers instead of patterns

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/cache.py

@@ -8,12 +8,13 @@
 #
 
 import os
-import json
-import attr
 import fnmatch
+import pickle
+import multiregex
 
-from commoncode.fileutils import create_dir
+import attr
 
+from commoncode.fileutils import create_dir
 from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS
 from packagedcode import SYSTEM_PACKAGE_DATAFILE_HANDLERS
 
@@ -29,6 +30,9 @@
 # global in-memory cache of the PkgManifestPatternsCache
 _PACKAGE_CACHE = None
 
+# This is the Pickle protocol we use, which was added in Python 3.4.
+PICKLE_PROTOCOL = 4
+
 PACKAGE_INDEX_LOCK_TIMEOUT = 60 * 6
 PACKAGE_INDEX_DIR = 'package_patterns_index'
 PACKAGE_INDEX_FILENAME = 'index_cache'
@@ -45,23 +49,21 @@ class PkgManifestPatternsCache:
     """
 
     handler_by_regex = attr.ib(default=attr.Factory(dict))
-    system_multiregex_patterns = attr.ib(default=attr.Factory(list))
-    application_multiregex_patterns = attr.ib(default=attr.Factory(list))
+    system_package_matcher = attr.ib(default=None)
+    application_package_matcher = attr.ib(default=None)
+    all_package_matcher = attr.ib(default=None)
 
     @staticmethod
-    def all_multiregex_patterns(self):
-        return self.application_multiregex_patterns + [
+    def all_multiregex_patterns(application_multiregex_patterns, system_multiregex_patterns):
+        return application_multiregex_patterns + [
             multiregex_pattern
-            for multiregex_pattern in self.system_multiregex_patterns
-            if multiregex_pattern not in self.application_multiregex_patterns
+            for multiregex_pattern in system_multiregex_patterns
+            if multiregex_pattern not in application_multiregex_patterns
         ]
 
     @classmethod
-    def from_mapping(cls, cache_mapping):
-        return cls(**cache_mapping)
-
-    @staticmethod
     def load_or_build(
+        cls,
         packagedcode_cache_dir=packagedcode_cache_dir,
         scancode_cache_dir=scancode_cache_dir,
         force=False,
@@ -94,7 +96,6 @@ def load_or_build(
                 print(str(e))
                 print(traceback.format_exc())
 
-
         from scancode import lockfile
         lock_file = os.path.join(scancode_cache_dir, PACKAGE_LOCKFILE_NAME)
 
@@ -109,29 +110,31 @@ def load_or_build(
                 application_multiregex_patterns, application_handlers_by_regex = build_mappings_and_multiregex_patterns(
                     datafile_handlers=application_package_datafile_handlers,
                 )
-                package_cache = PkgManifestPatternsCache(
+                all_multiregex_matcher = PkgManifestPatternsCache.all_multiregex_patterns(
+                    application_multiregex_patterns, system_multiregex_patterns,
+                )
+                system_package_matcher = multiregex.RegexMatcher(system_multiregex_patterns)
+                application_package_matcher = multiregex.RegexMatcher(application_multiregex_patterns)
+                all_package_matcher = multiregex.RegexMatcher(all_multiregex_matcher)
+                package_cache = cls(
                     handler_by_regex=system_handlers_by_regex | application_handlers_by_regex,
-                    system_multiregex_patterns=system_multiregex_patterns,
-                    application_multiregex_patterns=application_multiregex_patterns,
+                    system_package_matcher=system_package_matcher,
+                    application_package_matcher=application_package_matcher,
+                    all_package_matcher=all_package_matcher,
                 )
                 package_cache.dump(cache_file)
                 return package_cache
 
         except lockfile.LockTimeout:
             # TODO: handle unable to lock in a nicer way
-            raise
+            raise 
 
     def dump(self, cache_file):
         """
-        Dump this package cache on disk at ``cache_file``.
+        Dump this license cache on disk at ``cache_file``.
         """
-        package_cache = {
-            "handler_by_regex": self.handler_by_regex,
-            "system_multiregex_patterns": self.system_multiregex_patterns,
-            "application_multiregex_patterns": self.application_multiregex_patterns,
-        }
-        with open(cache_file, 'w') as f:
-            json.dump(package_cache, f)
+        with open(cache_file, 'wb') as fn:
+            pickle.dump(self, fn, protocol=PICKLE_PROTOCOL)
 
 
 def get_prematchers_from_glob_pattern(pattern):
@@ -203,20 +206,16 @@ def get_cache(
 
 def load_cache_file(cache_file):
     """
-    Return a PkgManifestPatternsCache loaded from JSON ``cache_file``.
+    Return a PkgManifestPatternsCache loaded from ``cache_file``.
     """
-    with open(cache_file) as f:
-        cache = json.load(f)
-
-    # convert multiregex patterns from list to tuples while loading
-    cache_transformed = {"handler_by_regex": cache.get("handler_by_regex")}
-    cache_transformed["system_multiregex_patterns"] = [
-        tuple(multiregex_pattern)
-        for multiregex_pattern in cache.get("system_multiregex_patterns")
-    ]
-    cache_transformed["application_multiregex_patterns"] = [
-        tuple(multiregex_pattern)
-        for multiregex_pattern in cache.get("application_multiregex_patterns")
-    ]
-
-    return PkgManifestPatternsCache.from_mapping(cache_transformed)
+    with open(cache_file, 'rb') as lfc:
+        try:
+            return pickle.load(lfc)
+        except Exception as e:
+            msg = (
+                'ERROR: Failed to load package cache (the file may be corrupted ?).\n'
+                f'Please delete "{cache_file}" and retry.\n'
+                'If the problem persists, copy this error message '
+                'and submit a bug report at https://github.com/nexB/scancode-toolkit/issues/'
+            )
+            raise Exception(msg) from e

src/packagedcode/recognize.py

@@ -10,8 +10,6 @@
 import os
 import sys
 
-import multiregex
-
 from commoncode import filetype
 from commoncode.fileutils import as_posixpath
 
@@ -87,13 +85,12 @@ def _parse(
 
     assert application or system or package_only
     if package_only or (application and system):
-        multiregex_patterns = package_patterns.all_multiregex_patterns
+        package_matcher = package_patterns.all_package_matcher
     elif application:
-        multiregex_patterns = package_patterns.application_multiregex_patterns
+        package_matcher = package_patterns.application_package_matcher
     elif system:
-        multiregex_patterns = package_patterns.system_multiregex_patterns
+        package_matcher = package_patterns.system_package_matcher
 
-    package_matcher = multiregex.RegexMatcher(multiregex_patterns)
     matched_patterns = package_matcher.match(package_path)
 
     datafile_handlers = []
@@ -103,19 +100,14 @@ def _parse(
         if TRACE:
             logger_debug(f'_parse:.handler_ids: {handler_ids}')
 
-        datafile_handlers = [
+        datafile_handlers.extend([
             HANDLER_BY_DATASOURCE_ID.get(handler_id)
             for handler_id in handler_ids
-        ]
+        ])
 
     if not datafile_handlers:
-        if BINARY_HANDLERS_PRESENT:
-            datafile_handlers = BINARY_PACKAGE_DATAFILE_HANDLERS
-        else:
-            if TRACE:
-                logger_debug(f'_parse: no package datafile detected at {package_path}')
-
-            return
+        if TRACE:
+            logger_debug(f'_parse: no package datafile detected at {package_path}')
 
     for handler in datafile_handlers:
         if TRACE:

tests/packagedcode/data/cache/.gitignore

@@ -0,0 +1 @@
+/package_patterns_index/

tests/packagedcode/data/cache/package_patterns_index/package_patterns_index/index_cache

@@ -10,6 +10,8 @@
 import os.path
 
 from packagedcode import cache
+from commoncode.fileutils import as_posixpath
+
 from packages_test_utils import PackageTester
 from scancode_config import REGEN_TEST_FIXTURES
 from scancode.cli_test_utils import run_scan_click
@@ -33,17 +35,27 @@ def test_build_mappings_and_multiregex_patterns_works(self):
 
     def test_build_package_cache_works(self):
         from packagedcode.about import AboutFileHandler
+        from packagedcode.bower import BowerJsonHandler
 
-        package_cache_dir = self.get_test_loc('cache/package_patterns_index')
+        package_cache_dir = self.get_test_loc('cache/')
         package_cache = cache.PkgManifestPatternsCache.load_or_build(
             packagedcode_cache_dir=package_cache_dir,
             application_package_datafile_handlers=[AboutFileHandler],
-            system_package_datafile_handlers=[],
+            system_package_datafile_handlers=[BowerJsonHandler],
             force=True,
         )
+        test_path = "scancode-toolkit.ABOUT"
+
+        assert not package_cache.system_package_matcher.match(test_path)
+        assert package_cache.application_package_matcher.match(test_path)
 
-        assert not package_cache.system_multiregex_patterns
-        assert len(package_cache.application_multiregex_patterns) == 1
-        assert '(?s:.*\\.ABOUT)\\Z' in package_cache.handler_by_regex
+        regex, _match = package_cache.all_package_matcher.match(test_path).pop()
+        assert package_cache.handler_by_regex.get(regex.pattern).pop() == AboutFileHandler.datasource_id
+
+    def check_empty_file_scan_works(self):
 
+        test_file = self.get_test_loc('cache/.gitignore')
+        package_path = as_posixpath(test_file)
+        package_matcher = cache.get_cache()
 
+        assert not package_matcher.match(package_path)

tests/packagedcode/test_cache.py

@@ -202,3 +202,8 @@ def test_recognize_rpmdb_sqlite(self):
         packages = recognize_package_data(test_file, system=True)
         assert packages
         assert isinstance(packages[0], models.PackageData)
+
+    def test_recognize_non_package_manifest_file(self):
+        test_file = self.get_test_loc('cache/.gitignore')
+        packages = recognize_package_data(test_file)
+        assert not packages