Add duplicate entry handling (Fix #179)

- Running "Fix Unlinked Entries" will no longer result in duplicate entries if the directory was refreshed after the original entries became unlinked.
- A "Duplicate Entries" section is added to the "Fix Unlinked Entries" modal to help repair existing affected libraries.

Author: CyanVoxel

tagstudio/src/core/library.py

@@ -32,7 +32,7 @@ def __init__(self, library: "Library", driver: "QtDriver"):
         super().__init__()
         self.lib = library
         self.driver = driver
-        self.setWindowTitle(f"Delete Unlinked Entries")
+        self.setWindowTitle("Delete Unlinked Entries")
         self.setWindowModality(Qt.WindowModality.ApplicationModal)
         self.setMinimumSize(500, 400)
         self.root_layout = QVBoxLayout(self)
@@ -81,20 +81,6 @@ def refresh_list(self):
             self.model.appendRow(QStandardItem(i))
 
     def delete_entries(self):
-        # pb = QProgressDialog('', None, 0, len(self.lib.missing_files))
-        # # pb.setMaximum(len(self.lib.missing_files))
-        # pb.setFixedSize(432, 112)
-        # pb.setWindowFlags(pb.windowFlags() & ~Qt.WindowType.WindowCloseButtonHint)
-        # pb.setWindowTitle('Deleting Entries')
-        # pb.setWindowModality(Qt.WindowModality.ApplicationModal)
-        # pb.show()
-
-        # r = CustomRunnable(lambda: self.lib.ref(pb))
-        # r.done.connect(lambda: self.done.emit())
-        # # r.done.connect(lambda: self.model.clear())
-        # QThreadPool.globalInstance().start(r)
-        # # r.run()
-
         iterator = FunctionIterator(self.lib.remove_missing_files)
 
         pw = ProgressWidget(
@@ -119,23 +105,3 @@ def delete_entries(self):
         r = CustomRunnable(lambda: iterator.run())
         QThreadPool.globalInstance().start(r)
         r.done.connect(lambda: (pw.hide(), pw.deleteLater(), self.done.emit()))
-
-    # def delete_entries_runnable(self):
-    # 	deleted = []
-    # 	for i, missing in enumerate(self.lib.missing_files):
-    # 		# pb.setValue(i)
-    # 		# pb.setLabelText(f'Deleting {i}/{len(self.lib.missing_files)} Unlinked Entries')
-    # 		try:
-    # 			id = self.lib.get_entry_id_from_filepath(missing)
-    # 			logging.info(f'Removing Entry ID {id}:\n\t{missing}')
-    # 			self.lib.remove_entry(id)
-    # 			self.driver.purge_item_from_navigation(ItemType.ENTRY, id)
-    # 			deleted.append(missing)
-    # 		except KeyError:
-    # 			logging.info(
-    # 				f'{ERROR} \"{id}\" was reported as missing, but is not in the file_to_entry_id map.')
-    # 		yield i
-    # 	for d in deleted:
-    # 		self.lib.missing_files.remove(d)
-    # 	# self.driver.filter_items('')
-    # 	# self.done.emit()

tagstudio/src/qt/modals/delete_unlinked.py

@@ -6,80 +6,95 @@
 import logging
 import typing
 
-from PySide6.QtCore import QThread, Qt, QThreadPool
+from PySide6.QtCore import Qt, QThreadPool
 from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLabel, QPushButton
 
 from src.core.library import Library
 from src.qt.helpers.function_iterator import FunctionIterator
 from src.qt.helpers.custom_runnable import CustomRunnable
 from src.qt.modals.delete_unlinked import DeleteUnlinkedEntriesModal
 from src.qt.modals.relink_unlinked import RelinkUnlinkedEntries
+from src.qt.modals.merge_dupe_entries import MergeDuplicateEntries
 from src.qt.widgets.progress import ProgressWidget
 
 # Only import for type checking/autocompletion, will not be imported at runtime.
 if typing.TYPE_CHECKING:
     from src.qt.ts_qt import QtDriver
 
 
-ERROR = f"[ERROR]"
-WARNING = f"[WARNING]"
-INFO = f"[INFO]"
+ERROR = "[ERROR]"
+WARNING = "[WARNING]"
+INFO = "[INFO]"
 
 logging.basicConfig(format="%(message)s", level=logging.INFO)
 
 
 class FixUnlinkedEntriesModal(QWidget):
-    # done = Signal(int)
     def __init__(self, library: "Library", driver: "QtDriver"):
         super().__init__()
         self.lib = library
         self.driver = driver
-        self.count = -1
-        self.setWindowTitle(f"Fix Unlinked Entries")
+        self.missing_count = -1
+        self.dupe_count = -1
+        self.setWindowTitle("Fix Unlinked Entries")
         self.setWindowModality(Qt.WindowModality.ApplicationModal)
         self.setMinimumSize(400, 300)
         self.root_layout = QVBoxLayout(self)
         self.root_layout.setContentsMargins(6, 6, 6, 6)
 
-        self.desc_widget = QLabel()
-        self.desc_widget.setObjectName("descriptionLabel")
-        self.desc_widget.setWordWrap(True)
-        self.desc_widget.setStyleSheet(
-            # 'background:blue;'
-            "text-align:left;"
-            # 'font-weight:bold;'
-            # 'font-size:14px;'
-            # 'padding-top: 6px'
-            ""
+        self.unlinked_desc_widget = QLabel()
+        self.unlinked_desc_widget.setObjectName("unlinkedDescriptionLabel")
+        self.unlinked_desc_widget.setWordWrap(True)
+        self.unlinked_desc_widget.setStyleSheet("text-align:left;")
+        self.unlinked_desc_widget.setText(
+            """Each library entry is linked to a file in one of your directories. If a file linked to an entry is moved or deleted outside of TagStudio, it is then considered unlinked. Unlinked entries may be automatically relinked via searching your directories, manually relinked by the user, or deleted if desired."""
         )
-        self.desc_widget.setText("""Each library entry is linked to a file in one of your directories. If a file linked to an entry is moved or deleted outside of TagStudio, it is then considered unlinked.
-		Unlinked entries may be automatically relinked via searching your directories, manually relinked by the user, or deleted if desired.""")
-        self.desc_widget.setAlignment(Qt.AlignmentFlag.AlignCenter)
-
-        self.missing_count = QLabel()
-        self.missing_count.setObjectName("missingCountLabel")
-        self.missing_count.setStyleSheet(
-            # 'background:blue;'
-            # 'text-align:center;'
-            "font-weight:bold;"
-            "font-size:14px;"
-            # 'padding-top: 6px'
-            ""
+
+        self.dupe_desc_widget = QLabel()
+        self.dupe_desc_widget.setObjectName("dupeDescriptionLabel")
+        self.dupe_desc_widget.setWordWrap(True)
+        self.dupe_desc_widget.setStyleSheet("text-align:left;")
+        self.dupe_desc_widget.setText(
+            """Duplicate entries are defined as multiple entries which point to the same file on disk. Merging these will combine the tags and metadata from all duplicates into a single consolidated entry. These are not to be confused with "duplicate files", which are duplicates of your files themselves outside of TagStudio."""
+        )
+
+        self.missing_count_label = QLabel()
+        self.missing_count_label.setObjectName("missingCountLabel")
+        self.missing_count_label.setStyleSheet("font-weight:bold;" "font-size:14px;")
+        self.missing_count_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
+
+        self.dupe_count_label = QLabel()
+        self.dupe_count_label.setObjectName("dupeCountLabel")
+        self.dupe_count_label.setStyleSheet("font-weight:bold;" "font-size:14px;")
+        self.dupe_count_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
+
+        self.refresh_unlinked_button = QPushButton()
+        self.refresh_unlinked_button.setText("&Refresh All")
+        self.refresh_unlinked_button.clicked.connect(
+            lambda: self.refresh_missing_files()
         )
-        self.missing_count.setAlignment(Qt.AlignmentFlag.AlignCenter)
-        # self.missing_count.setText('Missing Files: N/A')
 
-        self.refresh_button = QPushButton()
-        self.refresh_button.setText("&Refresh")
-        self.refresh_button.clicked.connect(lambda: self.refresh_missing_files())
+        self.merge_class = MergeDuplicateEntries(self.lib, self.driver)
+        self.relink_class = RelinkUnlinkedEntries(self.lib, self.driver)
 
         self.search_button = QPushButton()
         self.search_button.setText("&Search && Relink")
-        self.relink_class = RelinkUnlinkedEntries(self.lib, self.driver)
-        self.relink_class.done.connect(lambda: self.refresh_missing_files())
-        self.relink_class.done.connect(lambda: self.driver.update_thumbs())
+        self.relink_class.done.connect(
+            lambda: self.refresh_and_repair_dupe_entries(self.merge_class)
+        )
         self.search_button.clicked.connect(lambda: self.relink_class.repair_entries())
 
+        self.refresh_dupe_button = QPushButton()
+        self.refresh_dupe_button.setText("Refresh Duplicate Entries")
+        self.refresh_dupe_button.clicked.connect(lambda: self.refresh_dupe_entries())
+
+        self.merge_dupe_button = QPushButton()
+        self.merge_dupe_button.setText("&Merge Duplicate Entries")
+        self.merge_class.done.connect(lambda: self.set_dupe_count(-1))
+        self.merge_class.done.connect(lambda: self.set_missing_count(-1))
+        self.merge_class.done.connect(lambda: self.driver.filter_items())
+        self.merge_dupe_button.clicked.connect(lambda: self.merge_class.merge_entries())
+
         self.manual_button = QPushButton()
         self.manual_button.setText("&Manual Relink")
 
@@ -92,65 +107,46 @@ def __init__(self, library: "Library", driver: "QtDriver"):
         self.delete_button.setText("De&lete Unlinked Entries")
         self.delete_button.clicked.connect(lambda: self.delete_modal.show())
 
-        # self.combo_box = QComboBox()
-        # self.combo_box.setEditable(False)
-        # # self.combo_box.setMaxVisibleItems(5)
-        # self.combo_box.setStyleSheet('combobox-popup:0;')
-        # self.combo_box.view().setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
-        # for df in self.lib.default_fields:
-        # 	self.combo_box.addItem(f'{df["name"]} ({df["type"].replace("_", " ").title()})')
-
         self.button_container = QWidget()
         self.button_layout = QHBoxLayout(self.button_container)
         self.button_layout.setContentsMargins(6, 6, 6, 6)
         self.button_layout.addStretch(1)
 
         self.done_button = QPushButton()
         self.done_button.setText("&Done")
-        # self.save_button.setAutoDefault(True)
         self.done_button.setDefault(True)
         self.done_button.clicked.connect(self.hide)
-        # self.done_button.clicked.connect(lambda: self.done.emit(self.combo_box.currentIndex()))
-        # self.save_button.clicked.connect(lambda: save_callback(widget.get_content()))
         self.button_layout.addWidget(self.done_button)
 
-        # self.returnPressed.connect(lambda: self.done.emit(self.combo_box.currentIndex()))
-
-        # self.done.connect(lambda x: callback(x))
-
-        self.root_layout.addWidget(self.desc_widget)
-        self.root_layout.addWidget(self.missing_count)
-        self.root_layout.addWidget(self.refresh_button)
+        self.root_layout.addWidget(self.missing_count_label)
+        self.root_layout.addWidget(self.unlinked_desc_widget)
+        self.root_layout.addWidget(self.refresh_unlinked_button)
         self.root_layout.addWidget(self.search_button)
         self.manual_button.setHidden(True)
         self.root_layout.addWidget(self.manual_button)
         self.root_layout.addWidget(self.delete_button)
-        # self.root_layout.setStretch(1,2)
         self.root_layout.addStretch(1)
+        self.root_layout.addWidget(self.dupe_count_label)
+        self.root_layout.addWidget(self.dupe_desc_widget)
+        self.root_layout.addWidget(self.refresh_dupe_button)
+        self.root_layout.addWidget(self.merge_dupe_button)
+        self.root_layout.addStretch(2)
         self.root_layout.addWidget(self.button_container)
 
-        self.set_missing_count(self.count)
+        self.set_missing_count(self.missing_count)
+        self.set_dupe_count(self.dupe_count)
 
     def refresh_missing_files(self):
-        logging.info(f"Start RMF: {QThread.currentThread()}")
-        # pb = QProgressDialog(f'Scanning Library for Unlinked Entries...', None, 0,len(self.lib.entries))
-        # pb.setFixedSize(432, 112)
-        # pb.setWindowFlags(pb.windowFlags() & ~Qt.WindowType.WindowCloseButtonHint)
-        # pb.setWindowTitle('Scanning Library')
-        # pb.setWindowModality(Qt.WindowModality.ApplicationModal)
-        # pb.show()
-
         iterator = FunctionIterator(self.lib.refresh_missing_files)
         pw = ProgressWidget(
             window_title="Scanning Library",
-            label_text=f"Scanning Library for Unlinked Entries...",
+            label_text="Scanning Library for Unlinked Entries...",
             cancel_button_text=None,
             minimum=0,
             maximum=len(self.lib.entries),
         )
         pw.show()
         iterator.value.connect(lambda v: pw.update_progress(v + 1))
-        # rmf.value.connect(lambda v: pw.update_label(f'Progress: {v}'))
         r = CustomRunnable(lambda: iterator.run())
         QThreadPool.globalInstance().start(r)
         r.done.connect(
@@ -159,30 +155,76 @@ def refresh_missing_files(self):
                 pw.deleteLater(),
                 self.set_missing_count(len(self.lib.missing_files)),
                 self.delete_modal.refresh_list(),
+                self.refresh_dupe_entries(),
             )
         )
 
-        # r = CustomRunnable(lambda: self.lib.refresh_missing_files(lambda v: self.update_scan_value(pb, v)))
-        # r.done.connect(lambda: (pb.hide(), pb.deleteLater(), self.set_missing_count(len(self.lib.missing_files)), self.delete_modal.refresh_list()))
-        # QThreadPool.globalInstance().start(r)
-        # # r.run()
-        # pass
+    def refresh_dupe_entries(self):
+        iterator = FunctionIterator(self.lib.refresh_dupe_entries)
+        pw = ProgressWidget(
+            window_title="Scanning Library",
+            label_text="Scanning Library for Duplicate Entries...",
+            cancel_button_text=None,
+            minimum=0,
+            maximum=len(self.lib.entries),
+        )
+        pw.show()
+        iterator.value.connect(lambda v: pw.update_progress(v + 1))
+        r = CustomRunnable(lambda: iterator.run())
+        QThreadPool.globalInstance().start(r)
+        r.done.connect(
+            lambda: (
+                pw.hide(),
+                pw.deleteLater(),
+                self.set_dupe_count(len(self.lib.dupe_entries)),
+            )
+        )
 
-    # def update_scan_value(self, pb:QProgressDialog, value=int):
-    # 	# pb.setLabelText(f'Scanning Library for Unlinked Entries ({value}/{len(self.lib.entries)})...')
-    # 	pb.setValue(value)
+    def refresh_and_repair_dupe_entries(self, merge_class: MergeDuplicateEntries):
+        iterator = FunctionIterator(self.lib.refresh_dupe_entries)
+        pw = ProgressWidget(
+            window_title="Scanning Library",
+            label_text="Scanning Library for Duplicate Entries...",
+            cancel_button_text=None,
+            minimum=0,
+            maximum=len(self.lib.entries),
+        )
+        pw.show()
+        iterator.value.connect(lambda v: pw.update_progress(v + 1))
+        r = CustomRunnable(lambda: iterator.run())
+        QThreadPool.globalInstance().start(r)
+        r.done.connect(
+            lambda: (
+                pw.hide(),
+                pw.deleteLater(),
+                self.set_dupe_count(len(self.lib.dupe_entries)),
+                merge_class.merge_entries(),
+            )
+        )
 
     def set_missing_count(self, count: int):
-        self.count = count
-        if self.count < 0:
+        self.missing_count = count
+        if self.missing_count < 0:
             self.search_button.setDisabled(True)
             self.delete_button.setDisabled(True)
-            self.missing_count.setText(f"Unlinked Entries: N/A")
-        elif self.count == 0:
+            self.missing_count_label.setText("Unlinked Entries: N/A")
+        elif self.missing_count == 0:
             self.search_button.setDisabled(True)
             self.delete_button.setDisabled(True)
-            self.missing_count.setText(f"Unlinked Entries: {count}")
+            self.missing_count_label.setText(f"Unlinked Entries: {count}")
         else:
             self.search_button.setDisabled(False)
             self.delete_button.setDisabled(False)
-            self.missing_count.setText(f"Unlinked Entries: {count}")
+            self.missing_count_label.setText(f"Unlinked Entries: {count}")
+
+    def set_dupe_count(self, count: int):
+        self.dupe_count = count
+        if self.dupe_count < 0:
+            self.dupe_count_label.setText("Duplicate Entries: N/A")
+            self.merge_dupe_button.setDisabled(True)
+        elif self.dupe_count == 0:
+            self.dupe_count_label.setText(f"Duplicate Entries: {count}")
+            self.merge_dupe_button.setDisabled(True)
+        else:
+            self.dupe_count_label.setText(f"Duplicate Entries: {count}")
+            self.merge_dupe_button.setDisabled(False)