better error recovery for file encoding and bad json

beet/core/file.py

@@ -23,6 +23,9 @@
 
 import io
 import json
+
+import chardet
+import pyjson5
 import shutil
 from copy import deepcopy
 from dataclasses import dataclass, replace
@@ -436,14 +439,41 @@ def deserialize(self, content: Union[ValueType, str]) -> ValueType:
 
     @classmethod
     def from_path(cls, path: FileSystemPath, start: int, stop: int) -> str:
-        with open(path, "r", encoding="utf-8") as f:
-            if start > 0:
-                f.seek(start)
-            return f.read(stop - start) if stop >= -1 else f.read()
+        # We assume that the file is utf-8 since its most common
+        # In the event the file fails to be parsed in utf-8, we'll attempt to detect
+        # the correct encoding and retry. If that fails than another exception will be raised
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                if start > 0:
+                    f.seek(start)
+                return f.read(stop - start) if stop >= -1 else f.read()
+        except ValueError:
+            with open(path, "rb") as f:
+                if start > 0:
+                    f.seek(start)
+                file_bytes = f.read(stop - start) if stop >= -1 else f.read()
+
+                encoding = chardet.detect(file_bytes)["encoding"]
+
+                if encoding == None:
+                    encoding = "utf-8"
+
+                return file_bytes.decode(encoding=encoding)
+
 
     @classmethod
     def from_zip(cls, origin: ZipFile, name: str) -> str:
-        return origin.read(name).decode()
+        file_bytes = origin.read(name)
+
+        try:
+            return file_bytes.decode()
+        except UnicodeDecodeError:
+            encoding = chardet.detect(file_bytes)["encoding"]
+
+            if encoding == None:
+                encoding = "utf-8"
+
+            return file_bytes.decode(encoding=encoding)
 
     def dump_path(self, path: FileSystemPath, raw: str) -> None:
         with open(
@@ -634,7 +664,7 @@ def __post_init__(self):
         if not self.encoder:  # type: ignore
             self.encoder = dump_json
         if not self.decoder:  # type: ignore
-            self.decoder = json.loads
+            self.decoder = pyjson5.loads
 
 
 @dataclass(eq=False, repr=False)

beet/library/base.py

@@ -26,6 +26,8 @@
     "get_output_scope",
     "PACK_COMPRESSION",
     "LATEST_MINECRAFT_VERSION",
+    "NamespaceFileScope",
+    "get_output_scope",
 ]
 
 
@@ -430,6 +432,20 @@ class NamespacePin(Pin[Type[NamespaceFileType], NamespaceContainer[NamespaceFile
     """Descriptor for accessing namespace containers by attribute lookup."""
 
 
+def create_scope_map(
+    pins: Dict[str, Pin[type[NamespaceFile], NamespaceContainer[NamespaceFile]]]
+):
+    scope_map: Mapping[Tuple[Tuple[str, ...], str], Type[NamespaceFile]] = {}
+    for pin in pins.values():
+        if isinstance(scopes := pin.key.scope, tuple):
+            scope_map[(scopes, pin.key.extension)] = pin.key
+        else:
+            for scope in scopes.values():
+                scope_map[(scope, pin.key.extension)] = pin.key
+
+    return scope_map
+
+
 class Namespace(
     MergeMixin,
     Container[Type[NamespaceFile], NamespaceContainer[NamespaceFile]],
@@ -611,6 +627,7 @@ def scan(
             _update_with_none(extra_info, extend_namespace_extra)
 
         scope_map = dict(cls.scope_map)
+
         for file_type in extend_namespace:
             for scope in list_input_scopes(file_type.scope):
                 scope_map[scope, file_type.extension] = file_type