From 021690744108563f00ea6272a42dcb0fb821539b Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Mon, 21 Jul 2025 07:46:27 +0530 Subject: [PATCH 01/47] INTEGRITY: Only skip processing for entries with the same status when matching key is present. --- db_functions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/db_functions.py b/db_functions.py index a41c409d..4dea844c 100644 --- a/db_functions.py +++ b/db_functions.py @@ -131,16 +131,18 @@ def insert_fileset( # Check if key/megakey already exists, if so, skip insertion (no quotes on purpose) if detection: with conn.cursor() as cursor: - cursor.execute("SELECT id FROM fileset WHERE megakey = %s", (megakey,)) + cursor.execute( + "SELECT id, status FROM fileset WHERE megakey = %s", (megakey,) + ) existing_entry = cursor.fetchone() else: with conn.cursor() as cursor: - cursor.execute("SELECT id FROM fileset WHERE `key` = %s", (key,)) + cursor.execute("SELECT id, status FROM fileset WHERE `key` = %s", (key,)) existing_entry = cursor.fetchone() - if existing_entry is not None: + if (existing_entry is not None) and (status == existing_entry["status"]): existing_entry = existing_entry["id"] with conn.cursor() as cursor: cursor.execute("SET @fileset_last = %s", (existing_entry,)) From 90be689471f2f9bdf1901b64bd839674d60be85c Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Mon, 21 Jul 2025 07:53:46 +0530 Subject: [PATCH 02/47] INTEGRITY: Logging dropped candidates missed earlier. --- db_functions.py | 51 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/db_functions.py b/db_functions.py index 4dea844c..f9430827 100644 --- a/db_functions.py +++ b/db_functions.py @@ -1707,6 +1707,8 @@ def set_process( set_to_candidate_dict = defaultdict(list) id_to_fileset_dict = defaultdict(dict) + no_candidate_logs = [] + # Deep copy to avoid changes in game_data in the loop affecting the lookup map. game_data_lookup = {fs["name"]: copy.deepcopy(fs) for fs in game_data} @@ -1755,7 +1757,6 @@ def set_process( # Separating out the matching logic for glk engine engine_name = fileset["sourcefile"].split("-")[0] - (candidate_filesets, fileset_count) = set_filter_candidate_filesets( fileset_id, fileset, fileset_count, transaction_id, engine_name, conn ) @@ -1768,20 +1769,27 @@ def set_process( fileset["description"] if "description" in fileset else "" ) log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + console_log_text = f"Early fileset drop as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + no_candidate_logs.append(console_log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn ) dropped_early_no_candidate += 1 delete_original_fileset(fileset_id, conn) + continue id_to_fileset_dict[fileset_id] = fileset set_to_candidate_dict[fileset_id].extend(candidate_filesets) - console_message = "Candidate filtering finished." - console_log(console_message) + for console_log_text in no_candidate_logs: + console_log(console_log_text) + no_candidate_logs = [] + console_message = ( - f"{dropped_early_no_candidate} Filesets Dropped for No candidates." + f"{dropped_early_no_candidate} Filesets Dropped Early for having no candidates." ) console_log(console_message) + console_message = "Candidate filtering finished." + console_log(console_message) console_message = "Looking for duplicates..." console_log(console_message) @@ -1848,6 +1856,7 @@ def set_process( auto_merged_filesets, manual_merged_filesets, mismatch_filesets, + dropped_early_no_candidate, ) = set_perform_match( fileset, src, @@ -1861,13 +1870,18 @@ def set_process( mismatch_filesets, manual_merge_map, set_to_candidate_dict, + dropped_early_no_candidate, + no_candidate_logs, conn, skiplog, ) - match_count += 1 + console_log("Matching performed.") + for console_log_text in no_candidate_logs: + console_log(console_log_text) + with conn.cursor() as cursor: for fileset_id, candidates in manual_merge_map.items(): if len(candidates) == 0: @@ -1878,15 +1892,17 @@ def set_process( fileset["description"] if "description" in fileset else "" ) log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name}, Description: {fileset_description}." + console_log(console_log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn ) dropped_early_no_candidate += 1 + manual_merged_filesets -= 1 delete_original_fileset(fileset_id, conn) else: category_text = "Manual Merge Required" log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidates)}." - manual_merged_filesets += 1 add_manual_merge( candidates, fileset_id, @@ -1962,14 +1978,30 @@ def set_perform_match( mismatch_filesets, manual_merge_map, set_to_candidate_dict, + dropped_early_no_candidate, + no_candidate_logs, conn, skiplog, ): """ - "Performs matching for set.dat" + Performs matching for set.dat """ with conn.cursor() as cursor: - if len(candidate_filesets) == 1: + if len(candidate_filesets) == 0: + category_text = "Drop fileset - No Candidates" + fileset_name = fileset["name"] if "name" in fileset else "" + fileset_description = ( + fileset["description"] if "description" in fileset else "" + ) + log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name}, Description: {fileset_description}." + no_candidate_logs.append(console_log_text) + create_log( + escape_string(category_text), user, escape_string(log_text), conn + ) + dropped_early_no_candidate += 1 + delete_original_fileset(fileset_id, conn) + elif len(candidate_filesets) == 1: matched_fileset_id = candidate_filesets[0] cursor.execute( "SELECT status FROM fileset WHERE id = %s", (matched_fileset_id,) @@ -2032,12 +2064,14 @@ def set_perform_match( elif len(candidate_filesets) > 1: manual_merge_map[fileset_id] = candidate_filesets + manual_merged_filesets += 1 return ( fully_matched_filesets, auto_merged_filesets, manual_merged_filesets, mismatch_filesets, + dropped_early_no_candidate, ) @@ -2247,6 +2281,7 @@ def set_filter_candidate_filesets( filesize = f["size"] if is_glk and (filesize in set_glk_file_size or filesize == 0): count += 1 + continue if (filename, filesize) in set_file_name_size: if filesize == -1: count += 1 From d6f17b577d42c4751da408d235cef6cff6ab386c Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Tue, 22 Jul 2025 02:04:25 +0530 Subject: [PATCH 03/47] INTEGRITY: Add fileset creation details in log when new fileset is not deleted. --- db_functions.py | 77 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/db_functions.py b/db_functions.py index f9430827..18367886 100644 --- a/db_functions.py +++ b/db_functions.py @@ -183,10 +183,10 @@ def insert_fileset( log_text = f"Created Fileset:{fileset_last}, {log_text}" if src == "user": - log_text = f"Created Fileset:{fileset_last}, from user: IP {ip}, {log_text}" + log_text = f"Created Fileset:{fileset_last}, from user: IP {ip}." user = f"cli:{getpass.getuser()}" if username is None else username - if not skiplog: + if not skiplog and detection: log_last = create_log( escape_string(category_text), user, escape_string(log_text), conn ) @@ -1033,7 +1033,7 @@ def scan_process( fileset_description = ( fileset["description"] if "description" in fileset else "" ) - log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." create_log( escape_string(category_text), user, escape_string(log_text), conn ) @@ -1169,6 +1169,8 @@ def scan_perform_match( Put them for manual merge. """ with conn.cursor() as cursor: + fileset_name = fileset["name"] if "name" in fileset else "" + fileset_description = fileset["description"] if "description" in fileset else "" if len(candidate_filesets) == 1: matched_fileset_id = candidate_filesets[0] cursor.execute( @@ -1180,6 +1182,15 @@ def scan_perform_match( if status == "partial": # Partial filesets contain all the files, so does the scanned filesets, so this case should not ideally happen. if total_files(matched_fileset_id, conn) > total_fileset_files(fileset): + log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" + category_text = "Uploaded from scan." + create_log( + escape_string(category_text), + user, + escape_string(log_text), + conn, + ) + console_log(log_text) category_text = "Missing files" log_text = f"Missing files in Fileset:{fileset_id}. Try manual merge with Fileset:{matched_fileset_id}." add_manual_merge( @@ -1229,6 +1240,15 @@ def scan_perform_match( automatic_merged_filesets += 1 else: + log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" + category_text = "Uploaded from scan." + create_log( + escape_string(category_text), + user, + escape_string(log_text), + conn, + ) + console_log(log_text) category_text = "Manual Merge - Detection found" log_text = f"Matched with detection. Merge Fileset:{fileset_id} manually with Fileset:{matched_fileset_id}." add_manual_merge( @@ -1269,6 +1289,12 @@ def scan_perform_match( delete_original_fileset(fileset_id, conn) elif len(candidate_filesets) > 1: + log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" + category_text = "Uploaded from scan." + create_log( + escape_string(category_text), user, escape_string(log_text), conn + ) + console_log(log_text) category_text = "Manual Merge - Multiple Candidates" log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidate_filesets)}." manual_merged_filesets += 1 @@ -1768,8 +1794,8 @@ def set_process( fileset_description = ( fileset["description"] if "description" in fileset else "" ) - log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." - console_log_text = f"Early fileset drop as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." + log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." + console_log_text = f"Early fileset drop as no matching candidates. Name: {fileset_name} Description: {fileset_description}." no_candidate_logs.append(console_log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn @@ -1829,7 +1855,7 @@ def set_process( fileset_description = ( fileset["description"] if "description" in fileset else "" ) - log_text = f"Drop fileset, multiple filesets mapping to single detection. Name: {fileset_name}, Description: {fileset_description}. Clashed with Fileset:{candidate} ({engine}:{gameid}-{platform}-{language})" + log_text = f"Drop fileset, multiple filesets mapping to single detection. Name: {fileset_name} Description: {fileset_description}. Clashed with Fileset:{candidate} ({engine}:{gameid}-{platform}-{language})" console_log(log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn @@ -1884,15 +1910,15 @@ def set_process( with conn.cursor() as cursor: for fileset_id, candidates in manual_merge_map.items(): + fileset = id_to_fileset_dict[fileset_id] + fileset_name = fileset["name"] if "name" in fileset else "" + fileset_description = ( + fileset["description"] if "description" in fileset else "" + ) if len(candidates) == 0: category_text = "Drop fileset - No Candidates" - fileset = id_to_fileset_dict[fileset_id] - fileset_name = fileset["name"] if "name" in fileset else "" - fileset_description = ( - fileset["description"] if "description" in fileset else "" - ) - log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." - console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name}, Description: {fileset_description}." + log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." + console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name} Description: {fileset_description}." console_log(console_log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn @@ -1901,6 +1927,12 @@ def set_process( manual_merged_filesets -= 1 delete_original_fileset(fileset_id, conn) else: + log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" + category_text = "Uploaded from dat." + create_log( + escape_string(category_text), user, escape_string(log_text), conn + ) + console_log(log_text) category_text = "Manual Merge Required" log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidates)}." add_manual_merge( @@ -1987,14 +2019,12 @@ def set_perform_match( Performs matching for set.dat """ with conn.cursor() as cursor: + fileset_name = fileset["name"] if "name" in fileset else "" + fileset_description = fileset["description"] if "description" in fileset else "" if len(candidate_filesets) == 0: category_text = "Drop fileset - No Candidates" - fileset_name = fileset["name"] if "name" in fileset else "" - fileset_description = ( - fileset["description"] if "description" in fileset else "" - ) - log_text = f"Drop fileset as no matching candidates. Name: {fileset_name}, Description: {fileset_description}." - console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name}, Description: {fileset_description}." + log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." + console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name} Description: {fileset_description}." no_candidate_logs.append(console_log_text) create_log( escape_string(category_text), user, escape_string(log_text), conn @@ -2048,6 +2078,15 @@ def set_perform_match( delete_original_fileset(fileset_id, conn) else: + log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" + category_text = "Uploaded from dat." + create_log( + escape_string(category_text), + user, + escape_string(log_text), + conn, + ) + console_log(log_text) category_text = "Mismatch" log_text = f"Fileset:{fileset_id} mismatched with Fileset:{matched_fileset_id} with status:{status}. Try manual merge. Unmatched Files in set.dat fileset = {len(unmatched_dat_files)} Unmatched Files in candidate fileset = {len(unmatched_candidate_files)}. List of unmatched files scan.dat : {', '.join(scan_file for scan_file in unmatched_dat_files)}, List of unmatched files full fileset : {', '.join(scan_file for scan_file in unmatched_candidate_files)}" console_log(log_text) From 404898ea7e498b5a2be4b7d48d39151a049bdeb5 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Tue, 22 Jul 2025 12:52:41 +0530 Subject: [PATCH 04/47] INTEGRITY: Fix placeholder for widetable query. --- clear.py | 2 +- fileset.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/clear.py b/clear.py index acdae141..ccc5588c 100644 --- a/clear.py +++ b/clear.py @@ -19,7 +19,7 @@ def truncate_all_tables(conn): for table in tables: try: - cursor.execute("TRUNCATE TABLE %s", (table,)) + cursor.execute(f"TRUNCATE TABLE `{table}`") print(f"Table '{table}' truncated successfully") except pymysql.Error as err: print(f"Error truncating table '{table}': {err}") diff --git a/fileset.py b/fileset.py index a45556eb..d43df8f8 100644 --- a/fileset.py +++ b/fileset.py @@ -266,8 +266,7 @@ def fileset(): if widetable == "full": file_ids = [file["id"] for file in result] cursor.execute( - "SELECT file, checksum, checksize, checktype FROM filechecksum WHERE file IN (%s)", - (",".join(map(str, file_ids)),), + f"SELECT file, checksum, checksize, checktype FROM filechecksum WHERE file IN ({','.join(map(str, file_ids))})" ) checksums = cursor.fetchall() From 97961a79d147f8fa57c4a35ed972f00590386f99 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Wed, 23 Jul 2025 20:22:44 +0530 Subject: [PATCH 05/47] INTEGRITY: Redirect fileset url if id exceeds the bounds. --- fileset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fileset.py b/fileset.py index d43df8f8..fdc7e2ab 100644 --- a/fileset.py +++ b/fileset.py @@ -118,6 +118,11 @@ def fileset(): cursor.execute("SELECT MAX(id) FROM fileset") max_id = cursor.fetchone()["MAX(id)"] + if id > max_id: + return redirect(f"/fileset?id={max_id}") + if id < min_id: + return redirect(f"/fileset?id={min_id}") + # Ensure the id is between the minimum and maximum id id = max(min_id, min(id, max_id)) From ab293d97cebdcc67ec7998e94f222787a69201b4 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Wed, 23 Jul 2025 23:14:50 +0530 Subject: [PATCH 06/47] INTEGRITY: Fix the issue for filesets with null field not being filtered. --- fileset.py | 2 +- pagination.py | 68 +++++++++++++++++++++++---------------------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/fileset.py b/fileset.py index fdc7e2ab..9906e25b 100644 --- a/fileset.py +++ b/fileset.py @@ -1229,10 +1229,10 @@ def fileset_search(): order = "ORDER BY fileset.id" filters = { "fileset": "fileset", - "gameid": "game", "extra": "game", "platform": "game", "language": "game", + "gameid": "game", "megakey": "fileset", "status": "fileset", "transaction": "transactions", diff --git a/pagination.py b/pagination.py index 091384ce..5b124828 100644 --- a/pagination.py +++ b/pagination.py @@ -101,8 +101,6 @@ def create_page( num_of_pages = (num_of_results + results_per_page - 1) // results_per_page print(f"Num of results: {num_of_results}, Num of pages: {num_of_pages}") - if num_of_results == 0: - return "No results for given filters" page = int(request.args.get("page", 1)) page = max(1, min(page, num_of_pages)) @@ -118,11 +116,12 @@ def create_page( value = pymysql.converters.escape_string(value) if value == "": value = ".*" - condition += ( - f" AND {filters[key]}.{'id' if key == 'fileset' else key} REGEXP '{value}'" - if condition != "WHERE " - else f"{filters[key]}.{'id' if key == 'fileset' else key} REGEXP '{value}'" - ) + field = f"{filters[key]}.{'id' if key == 'fileset' else key}" + if value == ".*": + clause = f"({field} IS NULL OR {field} REGEXP '{value}')" + else: + clause = f"{field} REGEXP '{value}'" + condition += f" AND {clause}" if condition != "WHERE " else clause if condition == "WHERE ": condition = "" @@ -149,39 +148,32 @@ def create_page(
""" - if not results: - return "No results for given filters" - if results: - if filters: - if records_table != "log": - html += "" - else: - html += "" + if filters: + if records_table != "log": + html += "" + else: + html += "" - for key in results[0].keys(): - if key not in filters: - html += "" - continue - filter_value = request.args.get(key, "") - html += f"" - html += "" + for key in filters.keys(): + filter_value = request.args.get(key, "") + html += f"" + html += "" - html += "" - if records_table != "log": - html += "" - for key in results[0].keys(): - if key in ["fileset", "fileset_id"]: - continue - vars = "&".join( - [f"{k}={v}" for k, v in request.args.items() if k != "sort"] - ) - sort = request.args.get("sort", "") - if sort == key: - vars += f"&sort={key}-desc" - else: - vars += f"&sort={key}" - html += f"" + html += "" + if records_table != "log": + html += "" + for key in filters.keys(): + if key in ["fileset", "fileset_id"]: + continue + vars = "&".join([f"{k}={v}" for k, v in request.args.items() if k != "sort"]) + sort = request.args.get("sort", "") + if sort == key: + vars += f"&sort={key}-desc" + else: + vars += f"&sort={key}" + html += f"" + if results: counter = offset + 1 for row in results: if counter == offset + 1: # If it is the first run of the loop @@ -232,6 +224,8 @@ def create_page( counter += 1 html += "
#Fileset ID{key}#Fileset ID{key}
" + if not results: + html += "

No results for given filters

" # Pagination vars = "&".join([f"{k}={v}" for k, v in request.args.items() if k != "page"]) From 6d8c4bf7aabd936af88e0b42d3dd318b8166a4b4 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Wed, 23 Jul 2025 23:44:09 +0530 Subject: [PATCH 07/47] INTEGRITY: Join game table before engine table. --- pagination.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pagination.py b/pagination.py index 5b124828..899203a2 100644 --- a/pagination.py +++ b/pagination.py @@ -74,7 +74,11 @@ def create_page( # Handle multiple tables from_query = records_table - tables_list = list(tables) + join_order = ["game", "engine"] + tables_list = sorted( + list(tables), + key=lambda t: join_order.index(t) if t in join_order else 99, + ) if records_table not in tables_list or len(tables_list) > 1: for table in tables_list: if table == records_table: From 78cbabcb19bf65dc8e7d716111c6e9880bee0f72 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Wed, 23 Jul 2025 23:44:37 +0530 Subject: [PATCH 08/47] INTEGRITY: Add max and min pages in dashboard. --- pagination.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pagination.py b/pagination.py index 899203a2..8497ec4d 100644 --- a/pagination.py +++ b/pagination.py @@ -241,8 +241,8 @@ def create_page( html += f"" html += "" From 28d83ba0c9cfac22a7f7aaafb60cf3d8b4f2f9fa Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Fri, 25 Jul 2025 13:59:25 +0530 Subject: [PATCH 09/47] INTEGRITY: Improve merge workflow. --- fileset.py | 541 +++++++++++++----------- static/js/confirm_merge_form_handler.js | 85 ++++ 2 files changed, 389 insertions(+), 237 deletions(-) create mode 100644 static/js/confirm_merge_form_handler.js diff --git a/fileset.py b/fileset.py index 9906e25b..7ee5dd85 100644 --- a/fileset.py +++ b/fileset.py @@ -8,6 +8,7 @@ ) import pymysql.cursors import json +import html as html_lib import os from user_fileset_functions import ( user_insert_fileset, @@ -16,13 +17,14 @@ from pagination import create_page import difflib from db_functions import ( - find_matching_filesets, get_all_related_filesets, convert_log_text_to_links, user_integrity_check, db_connect, create_log, db_connect_root, + get_checksum_props, + delete_original_fileset, ) from collections import defaultdict from schema import init_database @@ -159,8 +161,7 @@ def fileset(): """ html += f"" - html += f"" - html += f"" + # html += f"" html += f""" @@ -334,7 +335,6 @@ def fileset(): # Generate the HTML for the developer actions html += "

Developer Actions

" html += f"" - html += f"" if "delete" in request.form: cursor.execute( @@ -419,121 +419,46 @@ def fileset(): html += "\n" html += "
\n" - return render_template_string(html) - finally: - connection.close() - - -@app.route("/fileset//match", methods=["GET"]) -def match_fileset_route(id): - base_dir = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join(base_dir, "mysql_config.json") - with open(config_path) as f: - mysql_cred = json.load(f) - - connection = pymysql.connect( - host=mysql_cred["servername"], - user=mysql_cred["username"], - password=mysql_cred["password"], - db=mysql_cred["dbname"], - charset="utf8mb4", - cursorclass=pymysql.cursors.DictCursor, - ) - try: - with connection.cursor() as cursor: - cursor.execute("SELECT * FROM fileset WHERE id = %s", (id,)) - fileset = cursor.fetchone() - fileset["rom"] = [] - if not fileset: - return f"No fileset found with id {id}", 404 - - cursor.execute( - "SELECT file.id, name, size, checksum, detection, detection_type FROM file WHERE fileset = %s", - (id,), - ) - result = cursor.fetchall() - file_ids = {} - for file in result: - file_ids[file["id"]] = (file["name"], file["size"]) - cursor.execute( - "SELECT file, checksum, checksize, checktype FROM filechecksum WHERE file IN (%s)", - (",".join(map(str, file_ids.keys())),), - ) - - files = cursor.fetchall() - checksum_dict = defaultdict( - lambda: {"name": "", "size": 0, "checksums": {}} - ) - - for i in files: - file_id = i["file"] - file_name, file_size = file_ids[file_id] - checksum_dict[file_name]["name"] = file_name - checksum_dict[file_name]["size"] = file_size - checksum_key = ( - f"{i['checktype']}-{i['checksize']}" - if i["checksize"] != 0 - else i["checktype"] - ) - checksum_dict[file_name]["checksums"][checksum_key] = i["checksum"] - - fileset["rom"] = [ - {"name": value["name"], "size": value["size"], **value["checksums"]} - for value in checksum_dict.values() - ] - - matched_map = find_matching_filesets(fileset, connection, fileset["status"]) - - html = f""" - - - - - - - -

Matched Filesets for Fileset: {id}

- - - - - - + # Manual merge final candidates + query = """ + SELECT + fs.*, + g.name AS game_name, + g.engine AS game_engine, + g.platform AS game_platform, + g.language AS game_language, + g.extra AS extra + FROM + fileset fs + LEFT JOIN + game g ON fs.game = g.id + JOIN + possible_merges pm ON pm.child_fileset = fs.id + WHERE pm.parent_fileset = %s """ - - for fileset_id, match_count in matched_map.items(): - if fileset_id == id: - continue - cursor.execute( - "SELECT COUNT(file.id) FROM file WHERE fileset = %s", (fileset_id,) - ) - count = cursor.fetchone()["COUNT(file.id)"] - html += f""" - - - - - - - + cursor.execute(query, (id,)) + results = cursor.fetchall() + if results: + html += """ +

Possible Merges

+
Fileset IDMatch CountActions
{fileset_id}{len(match_count)} / {count}View Details - - - - - - -
- -
-
+ """ + for result in results: + html += f""" + + + + + + + + + + """ + html += "
IDGame NamePlatformLanguageExtraDetailsAction
{result["id"]}{result["game_name"]}{result["game_platform"]}{result["game_language"]}{result["extra"]}View DetailsMerge
\n" - html += "" return render_template_string(html) finally: connection.close() @@ -755,7 +680,18 @@ def confirm_merge(id): (id,), ) source_fileset = cursor.fetchone() - print(source_fileset) + + # Select all files + file_query = """ + SELECT f.name, f.size, f.`size-r`, f.`size-rd`, + fc.checksum, fc.checksize, fc.checktype, f.detection + FROM file f + JOIN filechecksum fc ON fc.file = f.id + WHERE f.fileset = %s + """ + cursor.execute(file_query, (id,)) + source_files = cursor.fetchall() + cursor.execute( """ SELECT @@ -774,6 +710,9 @@ def confirm_merge(id): """, (target_id,), ) + target_fileset = cursor.fetchone() + cursor.execute(file_query, (target_id,)) + target_files = cursor.fetchall() def highlight_differences(source, target): diff = difflib.ndiff(source, target) @@ -806,12 +745,11 @@ def highlight_differences(source, target):

Confirm Merge

+
- + """ - target_fileset = cursor.fetchone() - for column in source_fileset.keys(): source_value = str(source_fileset[column]) target_value = str(target_fileset[column]) @@ -826,16 +764,141 @@ def highlight_differences(source, target): else: html += f"" + # Files + source_files_map = defaultdict(dict) + target_files_map = defaultdict(dict) + detection_files_set = set() + + if source_files: + for file in source_files: + checksize = file["checksize"] + if checksize != "1048576" and file["checksize"] == "1M": + checksize = "1048576" + if checksize != "1048576" and int(file["checksize"]) == 0: + checksize = "full" + check = file["checktype"] + "-" + checksize + source_files_map[file["name"].lower()][check] = file["checksum"] + source_files_map[file["name"].lower()]["size"] = file["size"] + source_files_map[file["name"].lower()]["size-r"] = file["size-r"] + source_files_map[file["name"].lower()]["size-rd"] = file["size-rd"] + + if target_files: + for file in target_files: + checksize = file["checksize"] + if checksize != "1048576" and file["checksize"] == "1M": + checksize = "1048576" + if checksize != "1048576" and int(file["checksize"]) == 0: + checksize = "full" + check = file["checktype"] + "-" + checksize + target_files_map[file["name"].lower()][check] = file["checksum"] + target_files_map[file["name"].lower()]["size"] = file["size"] + target_files_map[file["name"].lower()]["size-r"] = file["size-r"] + target_files_map[file["name"].lower()]["size-rd"] = file["size-rd"] + print(file) + if file["detection"] == 1: + detection_files_set.add(file["name"].lower()) + + print(detection_files_set) + + all_filenames = sorted( + set(source_files_map.keys()) | set(target_files_map.keys()) + ) + html += "" + for filename in all_filenames: + source_dict = source_files_map.get(filename, {}) + target_dict = target_files_map.get(filename, {}) + + html += f"" + + keys = sorted(set(source_dict.keys()) | set(target_dict.keys())) + + for key in keys: + source_value = str(source_dict.get(key, "")) + target_value = str(target_dict.get(key, "")) + + source_checked = "checked" if key in source_dict else "" + source_checksum = source_files_map[filename.lower()].get(key, "") + target_checksum = target_files_map[filename.lower()].get(key, "") + + source_val = html_lib.escape( + json.dumps( + { + "side": "source", + "filename": filename, + "prop": key, + "value": source_checksum, + "detection": "0", + } + ) + ) + if filename in detection_files_set: + target_val = html_lib.escape( + json.dumps( + { + "side": "target", + "filename": filename, + "prop": key, + "value": target_checksum, + "detection": "1", + } + ) + ) + else: + target_val = html_lib.escape( + json.dumps( + { + "side": "target", + "filename": filename, + "prop": key, + "value": target_checksum, + "detection": "0", + } + ) + ) + + if source_value != target_value: + source_highlighted, target_highlighted = highlight_differences( + source_value, target_value + ) + + html += f""" + + + + + + """ + else: + html += f""" + + + + + + """ + html += """
FieldSource FilesetTarget Fileset
FieldSource FilesetTarget Fileset
{column}{source_value}{target_value}
Files
{filename}Source FileTarget File
{key} + + {source_highlighted} + + + {target_highlighted} +
{key} + + {source_value} + + + {target_value} +
- - +
+ """ @@ -851,9 +914,11 @@ def highlight_differences(source, target): @app.route("/fileset//merge/execute", methods=["POST"]) -def execute_merge(id, source=None, target=None): - source_id = request.form["source_id"] if not source else source - target_id = request.form["target_id"] if not target else target +def execute_merge(id): + data = request.get_json() + source_id = data.get("source_id") + target_id = data.get("target_id") + options = data.get("options") base_dir = os.path.dirname(os.path.abspath(__file__)) config_path = os.path.join(base_dir, "mysql_config.json") @@ -875,145 +940,136 @@ def execute_merge(id, source=None, target=None): source_fileset = cursor.fetchone() cursor.execute("SELECT * FROM fileset WHERE id = %s", (target_id,)) - if source_fileset["status"] == "detection": + if source_fileset["status"] == "dat": cursor.execute( """ - UPDATE fileset SET - game = %s + UPDATE fileset SET status = %s, `key` = %s, - megakey = %s, `timestamp` = %s - WHERE id = %s + WHERE id = %s """, ( - source_fileset["game"], - source_fileset["status"], + "partial", source_fileset["key"], - source_fileset["megakey"], source_fileset["timestamp"], target_id, ), ) - cursor.execute("DELETE FROM file WHERE fileset = %s", (target_id,)) - - cursor.execute("SELECT * FROM file WHERE fileset = %s", (source_id,)) - source_files = cursor.fetchall() + source_filenames = set() + change_fileset_id = set() + file_details_map = defaultdict(dict) + + for file in options: + filename = file["filename"].lower() + if "detection" not in file_details_map[filename]: + file_details_map[filename]["detection"] = file["detection"] + file_details_map[filename]["detection_type"] = file["prop"] + elif ( + "detection" in file_details_map[filename] + and file_details_map[filename]["detection"] != "1" + ): + file_details_map[filename]["detection"] = file["detection"] + file_details_map[filename]["detection_type"] = file["prop"] + if file["prop"].startswith("md5"): + if "checksums" not in file_details_map[filename]: + file_details_map[filename]["checksums"] = [] + file_details_map[filename]["checksums"].append( + {"check": file["prop"], "value": file["value"]} + ) + if file["side"] == "source": + source_filenames.add(filename) - for file in source_files: - cursor.execute( + # Delete older checksums + for file in options: + filename = file["filename"].lower() + if file["side"] == "source": + cursor.execute( + """SELECT f.id as file_id FROM file f + JOIN fileset fs ON fs.id = f.fileset + WHERE f.name = %s + AND fs.id = %s""", + (filename, source_id), + ) + file_id = cursor.fetchone()["file_id"] + query = """ + DELETE FROM filechecksum + WHERE file = %s """ - INSERT INTO file (name, size, checksum, fileset, detection, `timestamp`) - VALUES (%s, %s, %s, %s, %s, NOW()) - """, - ( - file["name"].lower(), - file["size"], - file["checksum"], - target_id, - file["detection"], - ), - ) - - cursor.execute("SELECT LAST_INSERT_ID() as file_id") - new_file_id = cursor.fetchone()["file_id"] + cursor.execute(query, (file_id,)) + else: + if filename not in source_filenames: + cursor.execute( + """SELECT f.id as file_id FROM file f + JOIN fileset fs ON fs.id = f.fileset + WHERE f.name = %s + AND fs.id = %s""", + (filename, target_id), + ) + target_file_id = cursor.fetchone()["file_id"] + change_fileset_id.add(target_file_id) + for filename, details in file_details_map.items(): cursor.execute( - "SELECT * FROM filechecksum WHERE file = %s", (file["id"],) + """SELECT f.id as file_id FROM file f + JOIN fileset fs ON fs.id = f.fileset + WHERE f.name = %s + AND fs.id = %s""", + (filename, source_id), ) - file_checksums = cursor.fetchall() - - for checksum in file_checksums: + source_file_id = cursor.fetchone()["file_id"] + detection = ( + details["detection"] == "1" if "detection" in details else False + ) + if detection: + query = """ + UPDATE file + SET detection = 1, + detection_type = %s + WHERE id = %s + """ cursor.execute( - """ - INSERT INTO filechecksum (file, checksize, checktype, checksum) - VALUES (%s, %s, %s, %s) - """, + query, ( - new_file_id, - checksum["checksize"], - checksum["checktype"], - checksum["checksum"], + details["detection_type"], + source_file_id, ), ) - elif source_fileset["status"] in ["scan", "dat"]: - cursor.execute( - """ - UPDATE fileset SET - status = %s, - `key` = %s, - `timestamp` = %s - WHERE id = %s - """, - ( - source_fileset["status"] - if source_fileset["status"] != "dat" - else "partial", - source_fileset["key"], - source_fileset["timestamp"], - target_id, - ), - ) - cursor.execute("SELECT * FROM file WHERE fileset = %s", (source_id,)) - source_files = cursor.fetchall() - - cursor.execute("SELECT * FROM file WHERE fileset = %s", (target_id,)) - target_files = cursor.fetchall() - - target_files_dict = {} - for target_file in target_files: - cursor.execute( - "SELECT * FROM filechecksum WHERE file = %s", - (target_file["id"],), - ) - target_checksums = cursor.fetchall() - for checksum in target_checksums: - target_files_dict[checksum["checksum"]] = target_file - - for source_file in source_files: - cursor.execute( - "SELECT * FROM filechecksum WHERE file = %s", - (source_file["id"],), - ) - source_checksums = cursor.fetchall() - file_exists = False - for checksum in source_checksums: - print(checksum["checksum"]) - if checksum["checksum"] in target_files_dict.keys(): - target_file = target_files_dict[checksum["checksum"]] - source_file["detection"] = target_file["detection"] + cursor.execute( + """SELECT f.id as file_id FROM file f + JOIN fileset fs ON fs.id = f.fileset + WHERE f.name = %s + AND fs.id = %s""", + (filename, target_id), + ) + target_file_id = cursor.fetchone()["file_id"] + cursor.execute( + "DELETE FROM file WHERE id = %s", (target_file_id,) + ) + for c in details["checksums"]: + checksum = c["value"] + check = c["check"] + checksize, checktype, checksum = get_checksum_props( + check, checksum + ) + query = "INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (%s, %s, %s, %s)" + cursor.execute( + query, (source_file_id, checksize, checktype, checksum) + ) - cursor.execute( - "DELETE FROM file WHERE id = %s", (target_file["id"],) - ) - file_exists = True - break - print(file_exists) cursor.execute( - """INSERT INTO file (name, size, checksum, fileset, detection, `timestamp`) VALUES ( - %s, %s, %s, %s, %s, NOW())""", - ( - source_file["name"], - source_file["size"], - source_file["checksum"], - target_id, - source_file["detection"], - ), + "UPDATE file SET fileset = %s WHERE id = %s", + (target_id, source_file_id), ) - new_file_id = cursor.lastrowid - for checksum in source_checksums: - # TODO: Handle the string - cursor.execute( - "INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (%s, %s, %s, %s)", - ( - new_file_id, - checksum["checksize"], - f"{checksum['checktype']}-{checksum['checksize']}", - checksum["checksum"], - ), - ) + # for target_file_id in change_fileset_id: + # query = """ + # UPDATE file + # SET fileset = %s + # WHERE id = %s + # """ + # cursor.execute(query, (source_id, target_file_id)) cursor.execute( """ @@ -1023,6 +1079,17 @@ def execute_merge(id, source=None, target=None): (target_id, source_id), ) + delete_original_fileset(source_id, connection) + category_text = "Manually Merged" + log_text = f"Manually merged Fileset:{source_id} with Fileset:{target_id}." + create_log(category_text, "Moderator", log_text, connection) + + query = """ + DELETE FROM possible_merges + WHERE parent_fileset = %s + """ + cursor.execute(query, (source_id,)) + connection.commit() return redirect(url_for("fileset", id=target_id)) diff --git a/static/js/confirm_merge_form_handler.js b/static/js/confirm_merge_form_handler.js new file mode 100644 index 00000000..d514091b --- /dev/null +++ b/static/js/confirm_merge_form_handler.js @@ -0,0 +1,85 @@ +document.getElementById("confirm_merge_form").addEventListener("submit", async function (e) { + e.preventDefault(); + + const form = e.target; + + source_id = form.querySelector('input[name="source_id"]').value + + const jsonData = { + source_id: source_id, + target_id: form.querySelector('input[name="target_id"]').value, + options: [] + }; + + const checkedBoxes = form.querySelectorAll('input[name="options[]"]:checked'); + jsonData.options = Array.from(checkedBoxes).map(cb => { + const optionData = JSON.parse(cb.value); + optionData.tick = "on"; + return optionData; + }); + + console.log("Data being sent:", jsonData); + + const response = await fetch(`/fileset/${source_id}/merge/execute`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(jsonData), + }); + + if (response.redirected) { + window.location.href = response.url; + } +}); + + +function checkForConflicts() { + const checkedBoxes = document.querySelectorAll('input[name="options[]"]:checked'); + const conflicts = new Map(); + + Array.from(checkedBoxes).forEach(cb => { + const option = JSON.parse(cb.value); + const key = `${option.filename}|${option.prop}`; + if (!conflicts.has(key)) { + conflicts.set(key, []); + } + conflicts.get(key).push({side: option.side, checkbox: cb}); + }); + + document.querySelectorAll('input[name="options[]"]').forEach(cb => { + cb.style.backgroundColor = ''; + cb.parentElement.style.backgroundColor = ''; + }); + + let hasConflicts = false; + + conflicts.forEach((items, key) => { + if (items.length > 1) { + + hasConflicts = true; + + items.forEach(item => { + item.checkbox.style.backgroundColor = '#ffcccc'; + item.checkbox.parentElement.style.backgroundColor = '#ffe6e6'; + }); + } + }); + + const submitButton = document.querySelector('button[type="submit"]'); + if (hasConflicts) { + submitButton.disabled = true; + submitButton.textContent = 'Resolve Conflicts First'; + submitButton.style.backgroundColor = '#ccc'; + } else { + submitButton.disabled = false; + submitButton.textContent = 'Confirm Merge'; + submitButton.style.backgroundColor = ''; + } +} + + +document.querySelectorAll('input[name="options[]"]').forEach(checkbox => { + checkbox.addEventListener('change', checkForConflicts); +}); + From 1e791850d6a70a157754a0f78ccf03005dbb2a18 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 00:13:31 +0530 Subject: [PATCH 10/47] INTEGRITY: Add detection type for full checksums in detection entries --- db_functions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db_functions.py b/db_functions.py index 18367886..a0b68bfd 100644 --- a/db_functions.py +++ b/db_functions.py @@ -218,6 +218,11 @@ def insert_file(file, detection, src, conn): if "md5" in file: checksum = file["md5"] checksum = checksum.split(":")[1] if ":" in checksum else checksum + tag = checksum.split(":")[0] if ":" in checksum else "" + checktype = "md5" + if tag != "": + checktype += "-" + tag + checksize = 0 else: for key, value in file.items(): if "md5" in key: From 2bc1e6f003c43914b247bf349fa499e53e3c2ce5 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 00:26:52 +0530 Subject: [PATCH 11/47] INTEGRITY: Update timestamp for detection files in partial fileset conversion for set.dat --- db_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db_functions.py b/db_functions.py index a0b68bfd..b3320238 100644 --- a/db_functions.py +++ b/db_functions.py @@ -2846,7 +2846,8 @@ def set_populate_file(fileset, fileset_id, conn, detection): query = """ UPDATE file SET size = %s, - name = %s + name = %s, + `timestamp` = NOW() WHERE id = %s """ From 749b5a955b28a19bfff8a3772f5415ea1ae8e2ae Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 13:01:47 +0530 Subject: [PATCH 12/47] INTEGRITY: Add extra error handling for parsing. --- dat_parser.py | 157 +++++++++++++++++++++++++++++++++--------------- db_functions.py | 74 ++++++++++++----------- 2 files changed, 147 insertions(+), 84 deletions(-) diff --git a/dat_parser.py b/dat_parser.py index a76480b2..9655d5c5 100644 --- a/dat_parser.py +++ b/dat_parser.py @@ -1,5 +1,6 @@ import re import os +import sys from db_functions import db_insert, match_fileset import argparse @@ -79,21 +80,40 @@ def match_outermost_brackets(input): depth = 0 inside_quotes = False cur_index = 0 + line_number = 1 + index_line = 1 - for i in range(len(input)): - char = input[i] + for i, char in enumerate(input): + if char == "\n": + line_number += 1 + inside_quotes = False - if char == "(" and not inside_quotes: + if char == '"' and input[i - 1] != "\\": + inside_quotes = not inside_quotes + + elif char == "(" and not inside_quotes: if depth == 0: + if "rom" in input[i - 4 : i]: + raise ValueError( + f"Missing an opening '(' for the game. Look near line {line_number}." + ) + index_line = line_number cur_index = i depth += 1 + elif char == ")" and not inside_quotes: + if depth == 0: + print(f"Warning: unmatched ')' at line {line_number}") + continue depth -= 1 if depth == 0: match = input[cur_index : i + 1] matches.append((match, cur_index)) - elif char == '"' and input[i - 1] != "\\": - inside_quotes = not inside_quotes + + if depth != 0: + raise ValueError( + f"Unmatched '(' starting at line {index_line}: possibly an unclosed block." + ) return matches @@ -104,61 +124,102 @@ def parse_dat(dat_filepath): associated arrays """ if not os.path.isfile(dat_filepath): - print("File not readable") - return + print(f"Error: File does not exist or is unreadable: {dat_filepath}.") + return None - with open(dat_filepath, "r", encoding="utf-8") as dat_file: - content = dat_file.read() + try: + with open(dat_filepath, "r", encoding="utf-8") as dat_file: + content = dat_file.read() + except (IOError, UnicodeDecodeError) as e: + print(f"Error: Failed to read file {dat_filepath}: {e}") + return None header = {} game_data = [] resources = {} - matches = match_outermost_brackets(content) - # print(matches) + try: + matches = match_outermost_brackets(content) + except Exception as e: + print(f"Error: Failed to parse outer brackets in {dat_filepath}: {e}") + return None if matches: for data_segment in matches: - if ( - "clrmamepro" in content[data_segment[1] - 11 : data_segment[1]] - or "scummvm" in content[data_segment[1] - 8 : data_segment[1]] - ): - header = map_key_values(data_segment[0], header) - elif "game" in content[data_segment[1] - 5 : data_segment[1]]: - temp = {} - temp = map_key_values(data_segment[0], temp) - game_data.append(temp) - elif "resource" in content[data_segment[1] - 9 : data_segment[1]]: - temp = {} - temp = map_key_values(data_segment[0], temp) - resources[temp["name"]] = temp - # print(header, game_data, resources, dat_filepath) + try: + if ( + "clrmamepro" in content[data_segment[1] - 11 : data_segment[1]] + or "scummvm" in content[data_segment[1] - 8 : data_segment[1]] + ): + header = map_key_values(data_segment[0], header) + elif "game" in content[data_segment[1] - 5 : data_segment[1]]: + temp = {} + temp = map_key_values(data_segment[0], temp) + game_data.append(temp) + elif "resource" in content[data_segment[1] - 9 : data_segment[1]]: + temp = {} + temp = map_key_values(data_segment[0], temp) + resources[temp["name"]] = temp + except Exception as e: + print(f"Error: Failed to parse a data_segment: {e}") + return None + return header, game_data, resources, dat_filepath def main(): - parser = argparse.ArgumentParser( - description="Process DAT files and interact with the database." - ) - parser.add_argument( - "--upload", nargs="+", help="Upload DAT file(s) to the database" - ) - parser.add_argument( - "--match", nargs="+", help="Populate matching games in the database" - ) - parser.add_argument("--user", help="Username for database") - parser.add_argument("-r", help="Recurse through directories", action="store_true") - parser.add_argument("--skiplog", help="Skip logging dups", action="store_true") - - args = parser.parse_args() - - if args.upload: - for filepath in args.upload: - db_insert(parse_dat(filepath), args.user, args.skiplog) - - if args.match: - for filepath in args.match: - # print(parse_dat(filepath)[2]) - match_fileset(parse_dat(filepath), args.user, args.skiplog) + try: + parser = argparse.ArgumentParser( + description="Process DAT files and interact with the database." + ) + parser.add_argument( + "--upload", nargs="+", help="Upload DAT file(s) to the database" + ) + parser.add_argument( + "--match", nargs="+", help="Populate matching games in the database" + ) + parser.add_argument("--user", help="Username for database") + parser.add_argument( + "-r", help="Recurse through directories", action="store_true" + ) + parser.add_argument("--skiplog", help="Skip logging dups", action="store_true") + + args = parser.parse_args() + + if not args.upload and not args.match: + print("Error: No action specified. Use --upload or --match") + parser.print_help() + sys.exit(1) + + if args.upload: + for filepath in args.upload: + try: + parsed_data = parse_dat(filepath) + if parsed_data is not None: + db_insert(parsed_data, args.user, args.skiplog) + else: + print(f"Error: Failed to parse file for upload: {filepath}") + except Exception as e: + print(f"Error uploading {filepath}: {e}") + continue + + if args.match: + for filepath in args.match: + try: + parsed_data = parse_dat(filepath) + if parsed_data[0] is not None: + match_fileset(parsed_data, args.user, args.skiplog) + else: + print(f"Error: Failed to parse file for matching: {filepath}") + except Exception as e: + print(f"Error matching {filepath}: {e}") + continue + + except KeyboardInterrupt: + print("Operation cancelled by user") + sys.exit(0) + except Exception as e: + print(f"Error: Unexpected error in main: {e}") + sys.exit(1) if __name__ == "__main__": diff --git a/db_functions.py b/db_functions.py index b3320238..ff5d7f8f 100644 --- a/db_functions.py +++ b/db_functions.py @@ -519,14 +519,19 @@ def db_insert(data_arr, username=None, skiplog=False): try: author = header["author"] version = header["version"] + if author != "scummvm": + raise ValueError( + f"Author needs to be scummvm for seeding. Incorrect author: {author}" + ) + except ValueError as ve: + raise ve except KeyError as e: print(f"Missing key in header: {e}") return - src = "dat" if author not in ["scan", "scummvm"] else author - - detection = src == "scummvm" - status = "detection" if detection else src + src = author + detection = True + status = "detection" conn.cursor().execute("SET @fileset_time_last = %s", (int(time.time()),)) @@ -552,38 +557,35 @@ def db_insert(data_arr, username=None, skiplog=False): key = calc_key(fileset) megakey = calc_megakey(fileset) - if detection: - try: - engine_name = fileset.get("engine", "") - engineid = fileset["sourcefile"] - gameid = fileset["name"] - title = fileset.get("title", "") - extra = fileset.get("extra", "") - platform = fileset.get("platform", "") - lang = fileset.get("language", "") - except KeyError as e: - print( - f"Missing key in header: {e} for {fileset.get('name', '')}-{fileset.get('language', '')}-{fileset.get('platform', '')}" - ) - return + try: + engine_name = fileset.get("engine", "") + engineid = fileset["sourcefile"] + gameid = fileset["name"] + title = fileset.get("title", "") + extra = fileset.get("extra", "") + platform = fileset.get("platform", "") + lang = fileset.get("language", "") + except KeyError as e: + print( + f"Missing key in header: {e} for {fileset.get('name', '')}-{fileset.get('language', '')}-{fileset.get('platform', '')}" + ) + return - with conn.cursor() as cursor: - query = """ - SELECT id - FROM fileset - WHERE `key` = %s - """ - cursor.execute(query, (key,)) - existing_entry = cursor.fetchone() - if existing_entry is not None: - log_text = f"Skipping Entry as similar entry already exsits - Fileset:{existing_entry['id']}. Skpped entry details - engineid = {engineid}, gameid = {gameid}, platform = {platform}, language = {lang}" - create_log("Warning", user, escape_string(log_text), conn) - console_log(log_text) - continue + with conn.cursor() as cursor: + query = """ + SELECT id + FROM fileset + WHERE `key` = %s + """ + cursor.execute(query, (key,)) + existing_entry = cursor.fetchone() + if existing_entry is not None: + log_text = f"Skipping Entry as similar entry already exsits - Fileset:{existing_entry['id']}. Skpped entry details - engineid = {engineid}, gameid = {gameid}, platform = {platform}, language = {lang}" + create_log("Warning", user, escape_string(log_text), conn) + console_log(log_text) + continue - insert_game( - engine_name, engineid, title, gameid, extra, platform, lang, conn - ) + insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn) log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}." @@ -894,8 +896,8 @@ def match_fileset(data_arr, username=None, skiplog=False): return src = "dat" if author not in ["scan", "scummvm"] else author - detection = src == "scummvm" - source_status = "detection" if detection else src + detection = False + source_status = src conn.cursor().execute("SET @fileset_time_last = %s", (int(time.time()),)) From 546dfe3a447108a152712ba9a11c949761de8c63 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 13:29:40 +0530 Subject: [PATCH 13/47] INTEGRITY: Add commit/rollback transaction support to match_fileset and db_insert. --- db_functions.py | 346 +++++++++++++++++++++++++----------------------- 1 file changed, 184 insertions(+), 162 deletions(-) diff --git a/db_functions.py b/db_functions.py index ff5d7f8f..b2cbc558 100644 --- a/db_functions.py +++ b/db_functions.py @@ -382,19 +382,14 @@ def punycode_need_encode(orig): def create_log(category, user, text, conn): - query = f"INSERT INTO log (`timestamp`, category, user, `text`) VALUES (FROM_UNIXTIME({int(time.time())}), '{escape_string(category)}', '{escape_string(user)}', '{escape_string(text)}')" - query = "INSERT INTO log (`timestamp`, category, user, `text`) VALUES (FROM_UNIXTIME(%s), %s, %s, %s)" with conn.cursor() as cursor: try: + query = "INSERT INTO log (`timestamp`, category, user, `text`) VALUES (FROM_UNIXTIME(%s), %s, %s, %s)" cursor.execute(query, (int(time.time()), category, user, text)) - conn.commit() - except Exception as e: - conn.rollback() - print(f"Creating log failed: {e}") - log_last = None - else: cursor.execute("SELECT LAST_INSERT_ID()") log_last = cursor.fetchone()["LAST_INSERT_ID()"] + except Exception as e: + raise RuntimeError("Log creation failed") from e return log_last @@ -405,9 +400,7 @@ def update_history(source_id, target_id, conn, log_last=None): cursor.execute( query, (target_id, source_id, log_last if log_last is not None else 0) ) - conn.commit() except Exception as e: - conn.rollback() print(f"Creating log failed: {e}") log_last = None else: @@ -523,120 +516,137 @@ def db_insert(data_arr, username=None, skiplog=False): raise ValueError( f"Author needs to be scummvm for seeding. Incorrect author: {author}" ) - except ValueError as ve: - raise ve except KeyError as e: print(f"Missing key in header: {e}") return - src = author - detection = True - status = "detection" - - conn.cursor().execute("SET @fileset_time_last = %s", (int(time.time()),)) - - with conn.cursor() as cursor: - cursor.execute("SELECT MAX(`transaction`) FROM transactions") - temp = cursor.fetchone()["MAX(`transaction`)"] - if temp is None: - temp = 0 - transaction_id = temp + 1 + try: + src = author + detection = True + status = "detection" - category_text = f"Uploaded from {src}" - log_text = f"Started loading DAT file {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Transaction: {transaction_id}" + with conn.cursor() as cursor: + cursor.execute("SET @fileset_time_last = %s", (int(time.time()),)) - user = f"cli:{getpass.getuser()}" if username is None else username - create_log(escape_string(category_text), user, escape_string(log_text), conn) + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + temp = cursor.fetchone()["MAX(`transaction`)"] + if temp is None: + temp = 0 + transaction_id = temp + 1 - console_log(log_text) - console_log_total_filesets(filepath) + category_text = f"Uploaded from {src}" + log_text = f"Started loading DAT file {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Transaction: {transaction_id}" - fileset_count = 1 - for fileset in game_data: - console_log_detection(fileset_count) - key = calc_key(fileset) - megakey = calc_megakey(fileset) + user = f"cli:{getpass.getuser()}" if username is None else username + create_log(escape_string(category_text), user, escape_string(log_text), conn) - try: - engine_name = fileset.get("engine", "") - engineid = fileset["sourcefile"] - gameid = fileset["name"] - title = fileset.get("title", "") - extra = fileset.get("extra", "") - platform = fileset.get("platform", "") - lang = fileset.get("language", "") - except KeyError as e: - print( - f"Missing key in header: {e} for {fileset.get('name', '')}-{fileset.get('language', '')}-{fileset.get('platform', '')}" - ) - return + console_log(log_text) + console_log_total_filesets(filepath) - with conn.cursor() as cursor: - query = """ - SELECT id - FROM fileset - WHERE `key` = %s - """ - cursor.execute(query, (key,)) - existing_entry = cursor.fetchone() - if existing_entry is not None: - log_text = f"Skipping Entry as similar entry already exsits - Fileset:{existing_entry['id']}. Skpped entry details - engineid = {engineid}, gameid = {gameid}, platform = {platform}, language = {lang}" - create_log("Warning", user, escape_string(log_text), conn) - console_log(log_text) - continue + fileset_count = 1 + for fileset in game_data: + console_log_detection(fileset_count) + key = calc_key(fileset) + megakey = calc_megakey(fileset) + + try: + engine_name = fileset.get("engine", "") + engineid = fileset["sourcefile"] + gameid = fileset["name"] + title = fileset.get("title", "") + extra = fileset.get("extra", "") + platform = fileset.get("platform", "") + lang = fileset.get("language", "") + except KeyError as e: + raise RuntimeError( + f"Missing key in header: {e} for {fileset.get('name', '')}-{fileset.get('language', '')}-{fileset.get('platform', '')}" + ) - insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn) + with conn.cursor() as cursor: + query = """ + SELECT id + FROM fileset + WHERE `key` = %s + """ + cursor.execute(query, (key,)) + existing_entry = cursor.fetchone() + if existing_entry is not None: + log_text = f"Skipping Entry as similar entry already exsits - Fileset:{existing_entry['id']}. Skpped entry details - engineid = {engineid}, gameid = {gameid}, platform = {platform}, language = {lang}" + create_log("Warning", user, escape_string(log_text), conn) + console_log(log_text) + continue - log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}." + insert_game( + engine_name, engineid, title, gameid, extra, platform, lang, conn + ) - if insert_fileset( - src, - detection, - key, - megakey, - transaction_id, - log_text, - conn, - username=username, - skiplog=skiplog, - ): - # Some detection entries contain duplicate files. - unique_files = [] - seen = set() - for file_dict in fileset["rom"]: - dict_tuple = tuple(sorted(file_dict.items())) - if dict_tuple not in seen: - seen.add(dict_tuple) - unique_files.append(file_dict) - - for file in unique_files: - insert_file(file, detection, src, conn) - file_id = None - with conn.cursor() as cursor: - cursor.execute("SELECT @file_last AS file_id") - file_id = cursor.fetchone()["file_id"] - for key, value in file.items(): - if key not in ["name", "size", "size-r", "size-rd", "sha1", "crc"]: - insert_filechecksum(file, key, file_id, conn) + log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}." - fileset_count += 1 + if insert_fileset( + src, + detection, + key, + megakey, + transaction_id, + log_text, + conn, + username=username, + skiplog=skiplog, + ): + # Some detection entries contain duplicate files. + unique_files = [] + seen = set() + for file_dict in fileset["rom"]: + dict_tuple = tuple(sorted(file_dict.items())) + if dict_tuple not in seen: + seen.add(dict_tuple) + unique_files.append(file_dict) + + for file in unique_files: + insert_file(file, detection, src, conn) + file_id = None + with conn.cursor() as cursor: + cursor.execute("SELECT @file_last AS file_id") + file_id = cursor.fetchone()["file_id"] + for key, value in file.items(): + if key not in [ + "name", + "size", + "size-r", + "size-rd", + "sha1", + "crc", + ]: + insert_filechecksum(file, key, file_id, conn) + + fileset_count += 1 + + cur = conn.cursor() - cur = conn.cursor() + try: + cur.execute( + "SELECT COUNT(fileset) from transactions WHERE `transaction` = %s", + (transaction_id,), + ) + fileset_insertion_count = cur.fetchone()["COUNT(fileset)"] + category_text = f"Uploaded from {src}" + log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" + console_log(log_text) + except Exception as e: + print("Inserting failed:", e) + else: + user = f"cli:{getpass.getuser()}" if username is None else username + create_log( + escape_string(category_text), user, escape_string(log_text), conn + ) - try: - cur.execute( - "SELECT COUNT(fileset) from transactions WHERE `transaction` = %s", - (transaction_id,), - ) - fileset_insertion_count = cur.fetchone()["COUNT(fileset)"] - category_text = f"Uploaded from {src}" - log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" - console_log(log_text) + conn.commit() except Exception as e: - print("Inserting failed:", e) - else: - user = f"cli:{getpass.getuser()}" if username is None else username - create_log(escape_string(category_text), user, escape_string(log_text), conn) + conn.rollback() + print(f"Transaction failed: {e}") + finally: + conn.close() def compare_filesets(id1, id2, conn): @@ -895,59 +905,27 @@ def match_fileset(data_arr, username=None, skiplog=False): print(f"Missing key in header: {e}") return - src = "dat" if author not in ["scan", "scummvm"] else author - detection = False - source_status = src - - conn.cursor().execute("SET @fileset_time_last = %s", (int(time.time()),)) + try: + src = "dat" if author not in ["scan", "scummvm"] else author + detection = False + source_status = src - with conn.cursor() as cursor: - cursor.execute("SELECT MAX(`transaction`) FROM transactions") - transaction_id = cursor.fetchone()["MAX(`transaction`)"] - transaction_id = transaction_id + 1 if transaction_id else 1 + with conn.cursor() as cursor: + cursor.execute("SET @fileset_time_last = %s", (int(time.time()),)) + cursor.execute("SELECT MAX(`transaction`) FROM transactions") + transaction_id = cursor.fetchone()["MAX(`transaction`)"] + transaction_id = transaction_id + 1 if transaction_id else 1 - category_text = f"Uploaded from {src}" - log_text = f"Started loading DAT file {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}. Transaction: {transaction_id}" - console_log(log_text) - console_log_total_filesets(filepath) - user = f"cli:{getpass.getuser()}" if username is None else username - create_log(escape_string(category_text), user, escape_string(log_text), conn) + category_text = f"Uploaded from {src}" + log_text = f"Started loading DAT file {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}. Transaction: {transaction_id}" + console_log(log_text) + console_log_total_filesets(filepath) + user = f"cli:{getpass.getuser()}" if username is None else username + create_log(escape_string(category_text), user, escape_string(log_text), conn) - if src == "dat": - set_process( - game_data, - resources, - detection, - src, - conn, - transaction_id, - filepath, - author, - version, - source_status, - user, - skiplog, - ) - elif src == "scan": - scan_process( - game_data, - resources, - detection, - src, - conn, - transaction_id, - filepath, - author, - version, - source_status, - user, - skiplog, - ) - else: - game_data_lookup = {fs["name"]: fs for fs in game_data} - for fileset in game_data: - process_fileset( - fileset, + if src == "dat": + set_process( + game_data, resources, detection, src, @@ -958,11 +936,56 @@ def match_fileset(data_arr, username=None, skiplog=False): version, source_status, user, - game_data_lookup, + skiplog, ) - finalize_fileset_insertion( - conn, transaction_id, src, filepath, author, version, source_status, user - ) + elif src == "scan": + scan_process( + game_data, + resources, + detection, + src, + conn, + transaction_id, + filepath, + author, + version, + source_status, + user, + skiplog, + ) + else: + game_data_lookup = {fs["name"]: fs for fs in game_data} + for fileset in game_data: + process_fileset( + fileset, + resources, + detection, + src, + conn, + transaction_id, + filepath, + author, + version, + source_status, + user, + game_data_lookup, + ) + finalize_fileset_insertion( + conn, + transaction_id, + src, + filepath, + author, + version, + source_status, + user, + ) + conn.commit() + except Exception as e: + conn.rollback() + print(f"Transaction failed: {e}") + finally: + conn.close() def scan_process( @@ -2639,7 +2662,6 @@ def delete_original_fileset(fileset_id, conn): with conn.cursor() as cursor: cursor.execute("DELETE FROM file WHERE fileset = %s", (fileset_id,)) cursor.execute("DELETE FROM fileset WHERE id = %s", (fileset_id,)) - conn.commit() def update_fileset_status(cursor, fileset_id, status): From 686d06e3970f9d00ad50751b57f1a8378458cd74 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 13:36:45 +0530 Subject: [PATCH 14/47] INTEGRITY: Remove early string escaping for database logs as queries have been parametrised --- db_functions.py | 96 +++++++++++++++++-------------------------------- 1 file changed, 33 insertions(+), 63 deletions(-) diff --git a/db_functions.py b/db_functions.py index b2cbc558..272935b6 100644 --- a/db_functions.py +++ b/db_functions.py @@ -161,9 +161,7 @@ def insert_fileset( log_text = f"Updated Fileset:{existing_entry}, {log_text}" user = f"cli:{getpass.getuser()}" if username is None else username if not skiplog: - log_last = create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + log_last = create_log(category_text, user, log_text, conn) update_history(existing_entry, existing_entry, conn, log_last) return (existing_entry, True) @@ -187,9 +185,7 @@ def insert_fileset( user = f"cli:{getpass.getuser()}" if username is None else username if not skiplog and detection: - log_last = create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + log_last = create_log(category_text, user, log_text, conn) update_history(fileset_last, fileset_last, conn, log_last) else: update_history(0, fileset_last, conn) @@ -539,7 +535,7 @@ def db_insert(data_arr, username=None, skiplog=False): log_text = f"Started loading DAT file {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {status}. Transaction: {transaction_id}" user = f"cli:{getpass.getuser()}" if username is None else username - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) console_log(log_text) console_log_total_filesets(filepath) @@ -573,7 +569,7 @@ def db_insert(data_arr, username=None, skiplog=False): existing_entry = cursor.fetchone() if existing_entry is not None: log_text = f"Skipping Entry as similar entry already exsits - Fileset:{existing_entry['id']}. Skpped entry details - engineid = {engineid}, gameid = {gameid}, platform = {platform}, language = {lang}" - create_log("Warning", user, escape_string(log_text), conn) + create_log("Warning", user, log_text, conn) console_log(log_text) continue @@ -637,9 +633,7 @@ def db_insert(data_arr, username=None, skiplog=False): print("Inserting failed:", e) else: user = f"cli:{getpass.getuser()}" if username is None else username - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) conn.commit() except Exception as e: @@ -858,16 +852,12 @@ def populate_matching_games(): create_log( "Fileset merge", user, - escape_string( - f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}" - ), + f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}", conn, ) # Matching log - log_last = create_log( - escape_string(conn, category_text), user, escape_string(conn, log_text) - ) + log_last = create_log(conn, category_text, user, conn, log_text) # Add log id to the history table cursor.execute( @@ -921,7 +911,7 @@ def match_fileset(data_arr, username=None, skiplog=False): console_log(log_text) console_log_total_filesets(filepath) user = f"cli:{getpass.getuser()}" if username is None else username - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) if src == "dat": set_process( @@ -1064,9 +1054,7 @@ def scan_process( fileset["description"] if "description" in fileset else "" ) log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) dropped_early_no_candidate += 1 delete_original_fileset(fileset_id, conn) continue @@ -1105,11 +1093,11 @@ def scan_process( fileset_insertion_count = cursor.fetchone()["COUNT(fileset)"] category_text = f"Uploaded from {src}" log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}. State {source_status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) category_text = "Upload information" log_text = f"Number of filesets: {fileset_insertion_count}. Filesets automatically merged: {automatic_merged_filesets}. Filesets requiring manual merge (multiple candidates): {manual_merged_filesets}. Filesets requiring manual merge (matched with detection): {manual_merged_with_detection}. Filesets dropped, no candidate: {dropped_early_no_candidate}. Filesets matched with existing Full fileset: {match_with_full_fileset}. Filesets with mismatched files with Full fileset: {mismatch_with_full_fileset}. Filesets missing files compared to partial fileset candidate: {filesets_with_missing_files}." console_log(log_text) - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) def pre_update_files(rom, filesets_check_for_full, transaction_id, conn): @@ -1215,9 +1203,9 @@ def scan_perform_match( log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" category_text = "Uploaded from scan." create_log( - escape_string(category_text), + category_text, user, - escape_string(log_text), + log_text, conn, ) console_log(log_text) @@ -1273,9 +1261,9 @@ def scan_perform_match( log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" category_text = "Uploaded from scan." create_log( - escape_string(category_text), + category_text, user, - escape_string(log_text), + log_text, conn, ) console_log(log_text) @@ -1321,9 +1309,7 @@ def scan_perform_match( elif len(candidate_filesets) > 1: log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" category_text = "Uploaded from scan." - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) console_log(log_text) category_text = "Manual Merge - Multiple Candidates" log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidate_filesets)}." @@ -1827,9 +1813,7 @@ def set_process( log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." console_log_text = f"Early fileset drop as no matching candidates. Name: {fileset_name} Description: {fileset_description}." no_candidate_logs.append(console_log_text) - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) dropped_early_no_candidate += 1 delete_original_fileset(fileset_id, conn) continue @@ -1887,9 +1871,7 @@ def set_process( ) log_text = f"Drop fileset, multiple filesets mapping to single detection. Name: {fileset_name} Description: {fileset_description}. Clashed with Fileset:{candidate} ({engine}:{gameid}-{platform}-{language})" console_log(log_text) - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) dropped_early_single_candidate_multiple_sets += 1 delete_original_fileset(set_fileset, conn) del set_to_candidate_dict[set_fileset] @@ -1950,18 +1932,14 @@ def set_process( log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name} Description: {fileset_description}." console_log(console_log_text) - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) dropped_early_no_candidate += 1 manual_merged_filesets -= 1 delete_original_fileset(fileset_id, conn) else: log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" category_text = "Uploaded from dat." - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) console_log(log_text) category_text = "Manual Merge Required" log_text = f"Merge Fileset:{fileset_id} manually. Possible matches are: {', '.join(f'Fileset:{id}' for id in candidates)}." @@ -1982,11 +1960,11 @@ def set_process( fileset_insertion_count = cursor.fetchone()["COUNT(fileset)"] category_text = f"Uploaded from {src}" log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}. State {source_status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) category_text = "Upload information" log_text = f"Number of filesets: {fileset_insertion_count}. Filesets automatically merged: {auto_merged_filesets}. Filesets dropped early (no candidate) - {dropped_early_no_candidate}. Filesets dropped early (mapping to single detection) - {dropped_early_single_candidate_multiple_sets}. Filesets requiring manual merge: {manual_merged_filesets}. Partial/Full filesets already present: {fully_matched_filesets}. Partial/Full filesets with mismatch {mismatch_filesets}." console_log(log_text) - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) def set_filter_by_platform(gameid, candidate_filesets, conn): @@ -2056,9 +2034,7 @@ def set_perform_match( log_text = f"Drop fileset as no matching candidates. Name: {fileset_name} Description: {fileset_description}." console_log_text = f"Fileset dropped as no candidates anymore. Name: {fileset_name} Description: {fileset_description}." no_candidate_logs.append(console_log_text) - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) dropped_early_no_candidate += 1 delete_original_fileset(fileset_id, conn) elif len(candidate_filesets) == 1: @@ -2098,9 +2074,9 @@ def set_perform_match( category_text = "Already present" log_text = f"Already present as - Fileset:{matched_fileset_id}. Deleting Fileset:{fileset_id}" log_last = create_log( - escape_string(category_text), + category_text, user, - escape_string(log_text), + log_text, conn, ) update_history(fileset_id, matched_fileset_id, conn, log_last) @@ -2111,9 +2087,9 @@ def set_perform_match( log_text = f"Created Fileset:{fileset_id}. Name: {fileset_name} Description: {fileset_description}" category_text = "Uploaded from dat." create_log( - escape_string(category_text), + category_text, user, - escape_string(log_text), + log_text, conn, ) console_log(log_text) @@ -2226,7 +2202,7 @@ def add_manual_merge( """ cursor.execute(query, (child_fileset, parent_fileset)) - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) if print_text: print(print_text) @@ -2968,9 +2944,7 @@ def log_matched_fileset(src, fileset_last, fileset_id, state, user, conn): log_text = ( f"Matched Fileset:{fileset_last} with Fileset:{fileset_id}. State {state}." ) - log_last = create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + log_last = create_log(category_text, user, log_text, conn) update_history(fileset_last, fileset_id, conn, log_last) @@ -2992,7 +2966,7 @@ def log_scan_match_with_full( f"Fileset matched completely with Full Fileset:{candidate_id}. Dropping." ) print(log_text) - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) def finalize_fileset_insertion( @@ -3007,9 +2981,7 @@ def finalize_fileset_insertion( category_text = f"Uploaded from {src}" if src != "user": log_text = f"Completed loading DAT file, filename {filepath}, size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}. Number of filesets: {fileset_insertion_count}. Transaction: {transaction_id}" - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) def user_integrity_check(data, ip, game_metadata=None): @@ -3051,9 +3023,7 @@ def user_integrity_check(data, ip, game_metadata=None): user = f"cli:{getpass.getuser()}" - create_log( - escape_string(category_text), user, escape_string(log_text), conn - ) + create_log(category_text, user, log_text, conn) matched_map = find_matching_filesets(data, conn, src) @@ -3186,7 +3156,7 @@ def user_integrity_check(data, ip, game_metadata=None): finally: category_text = f"Uploaded from {src}" log_text = f"Completed loading file, State {source_status}. Transaction: {transaction_id}" - create_log(escape_string(category_text), user, escape_string(log_text), conn) + create_log(category_text, user, log_text, conn) # conn.close() return matched_map, missing_map, extra_map From 5e4627b48bae406db54007465fab0f92f6665eb8 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sat, 26 Jul 2025 13:48:57 +0530 Subject: [PATCH 15/47] INTEGRITY: Remove depracated/redundant code. --- db_functions.py | 617 +------------------------------------- fileset.py | 48 ++- user_fileset_functions.py | 205 ------------- 3 files changed, 23 insertions(+), 847 deletions(-) delete mode 100644 user_fileset_functions.py diff --git a/db_functions.py b/db_functions.py index 272935b6..9b385ebb 100644 --- a/db_functions.py +++ b/db_functions.py @@ -1,18 +1,14 @@ import pymysql import json -from collections import Counter import getpass import time import hashlib import os -from pymysql.converters import escape_string from collections import defaultdict import re import copy import sys -SPECIAL_SYMBOLS = '/":*|\\?%<>\x7f' - def db_connect(): console_log("Connecting to the Database.") @@ -324,59 +320,6 @@ def delete_filesets(conn): cursor.execute(query) -def my_escape_string(s: str) -> str: - """ - Escape strings - - Escape the following: - - escape char: \x81 - - unallowed filename chars: https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words - - control chars < 0x20 - """ - new_name = "" - for char in s: - if char == "\x81": - new_name += "\x81\x79" - elif char in SPECIAL_SYMBOLS or ord(char) < 0x20: - new_name += "\x81" + chr(0x80 + ord(char)) - else: - new_name += char - return new_name - - -def encode_punycode(orig): - """ - Punyencode strings - - - escape special characters and - - ensure filenames can't end in a space or dotif temp == None: - """ - s = my_escape_string(orig) - encoded = s.encode("punycode").decode("ascii") - # punyencoding adds an '-' at the end when there are no special chars - # don't use it for comparing - compare = encoded - if encoded.endswith("-"): - compare = encoded[:-1] - if orig != compare or compare[-1] in " .": - return "xn--" + encoded - return orig - - -def punycode_need_encode(orig): - """ - A filename needs to be punyencoded when it: - - - contains a char that should be escaped or - - ends with a dot or a space. - """ - if not all((0x20 <= ord(c) < 0x80) and c not in SPECIAL_SYMBOLS for c in orig): - return True - if orig[-1] in " .": - return True - return False - - def create_log(category, user, text, conn): with conn.cursor() as cursor: try: @@ -643,233 +586,6 @@ def db_insert(data_arr, username=None, skiplog=False): conn.close() -def compare_filesets(id1, id2, conn): - with conn.cursor() as cursor: - cursor.execute( - "SELECT name, size, `size-r`, `size-rd`, checksum FROM file WHERE fileset = %s", - (id1,), - ) - fileset1 = cursor.fetchall() - cursor.execute( - "SELECT name, size, `size-r`, `size-rd`, checksum FROM file WHERE fileset = %s", - (id2,), - ) - fileset2 = cursor.fetchall() - - # Sort filesets on checksum - fileset1.sort(key=lambda x: x[2]) - fileset2.sort(key=lambda x: x[2]) - - if len(fileset1) != len(fileset2): - return False - - for i in range(len(fileset1)): - # If checksums do not match - if fileset1[i][2] != fileset2[i][2]: - return False - - return True - - -def status_to_match(status): - order = ["detection", "dat", "scan", "partialmatch", "fullmatch", "user"] - return order[: order.index(status)] - - -def find_matching_game(game_files): - matching_games = [] # All matching games - matching_filesets = [] # All filesets containing one file from game_files - matches_count = 0 # Number of files with a matching detection entry - - conn = db_connect() - - for file in game_files: - checksum = file[1] - - query = "SELECT file.fileset as file_fileset FROM filechecksum JOIN file ON filechecksum.file = file.id WHERE filechecksum.checksum = %s AND file.detection = TRUE" - with conn.cursor() as cursor: - cursor.execute(query, (checksum,)) - records = cursor.fetchall() - - # If file is not part of detection entries, skip it - if len(records) == 0: - continue - - matches_count += 1 - for record in records: - matching_filesets.append(record[0]) - - # Check if there is a fileset_id that is present in all results - for key, value in Counter(matching_filesets).items(): - with conn.cursor() as cursor: - cursor.execute( - "SELECT COUNT(file.id) FROM file JOIN fileset ON file.fileset = fileset.id WHERE fileset.id = %s", - (key,), - ) - count_files_in_fileset = cursor.fetchone()["COUNT(file.id)"] - - # We use < instead of != since one file may have more than one entry in the fileset - # We see this in Drascula English version, where one entry is duplicated - if value < matches_count or value < count_files_in_fileset: - continue - - with conn.cursor() as cursor: - cursor.execute( - "SELECT engineid, game.id, gameid, platform, language, `key`, src, fileset.id as fileset FROM game JOIN fileset ON fileset.game = game.id JOIN engine ON engine.id = game.engine WHERE fileset.id = %s", - (key,), - ) - records = cursor.fetchall() - - matching_games.append(records[0]) - - if len(matching_games) != 1: - return matching_games - - # Check the current fileset priority with that of the match - with conn.cursor() as cursor: - cursor.execute( - f"SELECT id FROM fileset, ({query}) AS res WHERE id = file_fileset AND status IN ({', '.join(['%s'] * len(game_files[3]))})", - status_to_match(game_files[3]), - ) - records = cursor.fetchall() - - # If priority order is correct - if len(records) != 0: - return matching_games - - if compare_filesets(matching_games[0]["fileset"], game_files[0][0], conn): - with conn.cursor() as cursor: - cursor.execute( - "UPDATE fileset SET `delete` = TRUE WHERE id = %s", (game_files[0][0],) - ) - return [] - - return matching_games - - -def merge_filesets(detection_id, dat_id): - conn = db_connect() - - try: - with conn.cursor() as cursor: - cursor.execute( - "SELECT DISTINCT(filechecksum.checksum), checksize, checktype FROM filechecksum JOIN file on file.id = filechecksum.file WHERE fileset = %s'", - (detection_id,), - ) - detection_files = cursor.fetchall() - - for file in detection_files: - checksum = file[0] - checksize = file[1] - checktype = file[2] - - cursor.execute( - "DELETE FROM file WHERE checksum = %s AND fileset = %s LIMIT 1", - (checksum, detection_id), - ) - cursor.execute( - "UPDATE file JOIN filechecksum ON filechecksum.file = file.id SET detection = TRUE, checksize = %s, checktype = %s WHERE fileset = %s AND filechecksum.checksum = %s", - (checksize, checktype, dat_id, checksum), - ) - - cursor.execute( - "INSERT INTO history (`timestamp`, fileset, oldfileset) VALUES (FROM_UNIXTIME(%s), %s, %s)", - (int(time.time()), dat_id, detection_id), - ) - cursor.execute("SELECT LAST_INSERT_ID()") - history_last = cursor.fetchone()["LAST_INSERT_ID()"] - - cursor.execute( - "UPDATE history SET fileset = %s WHERE fileset = %s", - (dat_id, detection_id), - ) - cursor.execute("DELETE FROM fileset WHERE id = %s", (detection_id,)) - - conn.commit() - except Exception as e: - conn.rollback() - print(f"Error merging filesets: {e}") - finally: - # conn.close() - pass - - return history_last - - -def populate_matching_games(): - conn = db_connect() - - # Getting unmatched filesets - unmatched_filesets = [] - - with conn.cursor() as cursor: - cursor.execute( - "SELECT fileset.id, filechecksum.checksum, src, status FROM fileset JOIN file ON file.fileset = fileset.id JOIN filechecksum ON file.id = filechecksum.file WHERE fileset.game IS NULL AND status != 'user'" - ) - unmatched_files = cursor.fetchall() - - # Splitting them into different filesets - i = 0 - while i < len(unmatched_files): - cur_fileset = unmatched_files[i][0] - temp = [] - while i < len(unmatched_files) and cur_fileset == unmatched_files[i][0]: - temp.append(unmatched_files[i]) - i += 1 - unmatched_filesets.append(temp) - - for fileset in unmatched_filesets: - matching_games = find_matching_game(fileset) - - if len(matching_games) != 1: # If there is no match/non-unique match - continue - - matched_game = matching_games[0] - - # Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch) - status = fileset[0][2] - if fileset[0][2] == "dat": - status = "partialmatch" - elif fileset[0][2] == "scan": - status = "fullmatch" - - # Convert NULL values to string with value NULL for printing - matched_game = {k: "NULL" if v is None else v for k, v in matched_game.items()} - - category_text = f"Matched from {fileset[0][2]}" - log_text = f"Matched game {matched_game['engineid']}:\n{matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']}\nvariant {matched_game['key']}. State {status}. Fileset:{fileset[0][0]}." - - # Updating the fileset.game value to be $matched_game["id"] - query = "UPDATE fileset SET game = %s, status = %s, `key` = %s WHERE id = %s" - - history_last = merge_filesets(matched_game["fileset"], fileset[0][0]) - - if cursor.execute( - query, (matched_game["id"], status, matched_game["key"], fileset[0][0]) - ): - user = f"cli:{getpass.getuser()}" - - create_log( - "Fileset merge", - user, - f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}", - conn, - ) - - # Matching log - log_last = create_log(conn, category_text, user, conn, log_text) - - # Add log id to the history table - cursor.execute( - "UPDATE history SET log = %s WHERE id = %s", (log_last, history_last) - ) - - try: - conn.commit() - except Exception: - print("Updating matched games failed") - - def match_fileset(data_arr, username=None, skiplog=False): """ data_arr -> tuple : (header, game_data, resources, filepath). @@ -943,33 +659,6 @@ def match_fileset(data_arr, username=None, skiplog=False): user, skiplog, ) - else: - game_data_lookup = {fs["name"]: fs for fs in game_data} - for fileset in game_data: - process_fileset( - fileset, - resources, - detection, - src, - conn, - transaction_id, - filepath, - author, - version, - source_status, - user, - game_data_lookup, - ) - finalize_fileset_insertion( - conn, - transaction_id, - src, - filepath, - author, - version, - source_status, - user, - ) conn.commit() except Exception as e: conn.rollback() @@ -2429,78 +2118,6 @@ def is_candidate_by_checksize(candidate, fileset, conn): return False -def process_fileset( - fileset, - resources, - detection, - src, - conn, - transaction_id, - filepath, - author, - version, - source_status, - user, - game_data_lookup, -): - if detection: - insert_game_data(fileset, conn) - - # Ideally romof should be enough, but adding in case of an edge case - current_name = fileset.get("romof") or fileset.get("cloneof") - - # Iteratively check for extra files if linked to multiple filesets - while current_name: - if current_name in resources: - fileset["rom"] += resources[current_name]["rom"] - break - - elif current_name in game_data_lookup: - linked = game_data_lookup[current_name] - fileset["rom"] += linked.get("rom", []) - current_name = linked.get("romof") or linked.get("cloneof") - else: - break - - key = calc_key(fileset) if not detection else "" - megakey = calc_megakey(fileset) if detection else "" - log_text = f"size {os.path.getsize(filepath)}, author {author}, version {version}. State {source_status}." - if src != "dat": - matched_map = find_matching_filesets(fileset, conn, src) - else: - matched_map = matching_set(fileset, conn) - - (fileset_id, _) = insert_new_fileset( - fileset, conn, detection, src, key, megakey, transaction_id, log_text, user - ) - - if matched_map: - handle_matched_filesets( - fileset_id, - matched_map, - fileset, - conn, - detection, - src, - key, - megakey, - transaction_id, - log_text, - user, - ) - - -def insert_game_data(fileset, conn): - engine_name = fileset["engine"] - engineid = fileset["sourcefile"] - gameid = fileset["name"] - title = fileset["title"] - extra = fileset["extra"] - platform = fileset["platform"] - lang = fileset["language"] - insert_game(engine_name, engineid, title, gameid, extra, platform, lang, conn) - - def find_matching_filesets(fileset, conn, status): matched_map = defaultdict(list) if status != "user": @@ -2535,105 +2152,6 @@ def find_matching_filesets(fileset, conn, status): return matched_map -def matching_set(fileset, conn): - matched_map = defaultdict(list) - with conn.cursor() as cursor: - for file in fileset["rom"]: - matched_set = set() - if "md5" in file: - checksum = file["md5"] - if ":" in checksum: - checksum = checksum.split(":")[1] - size = file["size"] - - query = """ - SELECT DISTINCT fs.id AS fileset_id - FROM fileset fs - JOIN file f ON fs.id = f.fileset - JOIN filechecksum fc ON f.id = fc.file - WHERE fc.checksum = %s AND fc.checktype LIKE 'md5%' - AND fc.checksize > %s - AND fs.status = 'detection' - """ - cursor.execute(query, (checksum, size)) - records = cursor.fetchall() - if records: - for record in records: - matched_set.add(record["fileset_id"]) - for id in matched_set: - matched_map[id].append(file) - return matched_map - - -def handle_matched_filesets( - fileset_last, - matched_map, - fileset, - conn, - detection, - src, - key, - megakey, - transaction_id, - log_text, - user, -): - matched_list = sorted(matched_map.items(), key=lambda x: len(x[1]), reverse=True) - is_full_matched = False - with conn.cursor() as cursor: - for matched_fileset_id, matched_count in matched_list: - if is_full_matched: - break - cursor.execute( - "SELECT status FROM fileset WHERE id = %s", (matched_fileset_id,) - ) - status = cursor.fetchone()["status"] - cursor.execute( - "SELECT COUNT(file.id) FROM file WHERE fileset = %s", - (matched_fileset_id,), - ) - count = cursor.fetchone()["COUNT(file.id)"] - - if status in ["detection", "obsolete"] and count == len(matched_count): - is_full_matched = True - update_fileset_status( - cursor, matched_fileset_id, "full" if src != "dat" else "partial" - ) - populate_file(fileset, matched_fileset_id, conn, detection) - log_matched_fileset( - src, - fileset_last, - matched_fileset_id, - "full" if src != "dat" else "partial", - user, - conn, - ) - delete_original_fileset(fileset_last, conn) - elif status == "full" and len(fileset["rom"]) == count: - is_full_matched = True - log_matched_fileset( - src, fileset_last, matched_fileset_id, "full", user, conn - ) - delete_original_fileset(fileset_last, conn) - return - elif (status == "partial") and count == len(matched_count): - is_full_matched = True - update_fileset_status(cursor, matched_fileset_id, "full") - populate_file(fileset, matched_fileset_id, conn, detection) - log_matched_fileset( - src, fileset_last, matched_fileset_id, "full", user, conn - ) - delete_original_fileset(fileset_last, conn) - elif status == "scan" and count == len(matched_count): - log_matched_fileset( - src, fileset_last, matched_fileset_id, "full", user, conn - ) - elif src == "dat": - log_matched_fileset( - src, fileset_last, matched_fileset_id, "partial matched", user, conn - ) - - def delete_original_fileset(fileset_id, conn): with conn.cursor() as cursor: cursor.execute("DELETE FROM file WHERE fileset = %s", (fileset_id,)) @@ -2652,131 +2170,6 @@ def update_fileset_status(cursor, fileset_id, status): ) -def populate_file(fileset, fileset_id, conn, detection): - with conn.cursor() as cursor: - cursor.execute("SELECT * FROM file WHERE fileset = %s", (fileset_id,)) - target_files = cursor.fetchall() - target_files_dict = {} - for target_file in target_files: - cursor.execute( - "SELECT * FROM filechecksum WHERE file = %s", (target_file["id"],) - ) - target_checksums = cursor.fetchall() - for checksum in target_checksums: - target_files_dict[checksum["checksum"]] = target_file - target_files_dict[target_file["id"]] = ( - f"{checksum['checktype']}-{checksum['checksize']}" - ) - for file in fileset["rom"]: - file_exists = False - checksum = "" - checksize = 5000 - checktype = "None" - if "md5" in file: - checksum = file["md5"] - else: - for key, value in file.items(): - if "md5" in key: - checksize, checktype, checksum = get_checksum_props(key, value) - break - - if not detection: - checktype = "None" - detection = 0 - detection_type = ( - f"{checktype}-{checksize}" if checktype != "None" else f"{checktype}" - ) - - extended_file_size = True if "size-r" in file else False - - name = normalised_path(file["name"]) - escaped_name = escape_string(name) - - columns = ["name", "size"] - values = [f"'{escaped_name}'", f"'{file['size']}'"] - - if extended_file_size: - columns.extend(["`size-r`", "`size-rd`"]) - values.extend([f"'{file['size-r']}'", f"'{file['size-rd']}'"]) - - columns.extend( - ["checksum", "fileset", "detection", "detection_type", "`timestamp`"] - ) - values.extend( - [ - f"'{checksum}'", - str(fileset_id), - str(detection), - f"'{detection_type}'", - "NOW()", - ] - ) - - query = ( - f"INSERT INTO file ({', '.join(columns)}) VALUES ({', '.join(values)})" - ) - cursor.execute(query) - cursor.execute("SET @file_last = LAST_INSERT_ID()") - cursor.execute("SELECT @file_last AS file_id") - - file_id = cursor.fetchone()["file_id"] - d_type = 0 - previous_checksums = {} - - for key, value in file.items(): - if key not in ["name", "size", "size-r", "size-rd", "sha1", "crc"]: - insert_filechecksum(file, key, file_id, conn) - if value in target_files_dict and not file_exists: - cursor.execute( - f"SELECT detection_type FROM file WHERE id = {target_files_dict[value]['id']}" - ) - d_type = cursor.fetchone()["detection_type"] - file_exists = True - cursor.execute( - f"SELECT * FROM file WHERE fileset = {fileset_id}" - ) - target_files = cursor.fetchall() - for target_file in target_files: - cursor.execute( - f"SELECT * FROM filechecksum WHERE file = {target_file['id']}" - ) - target_checksums = cursor.fetchall() - for checksum in target_checksums: - previous_checksums[ - f"{checksum['checktype']}-{checksum['checksize']}" - ] = checksum["checksum"] - cursor.execute( - f"DELETE FROM file WHERE id = {target_files_dict[value]['id']}" - ) - - if file_exists: - cursor.execute( - f"SELECT checktype, checksize FROM filechecksum WHERE file = {file_id}" - ) - existing_checks = cursor.fetchall() - existing_checksum = [] - for existing_check in existing_checks: - existing_checksum.append( - existing_check["checktype"] + "-" + existing_check["checksize"] - ) - for key, value in previous_checksums.items(): - if key not in existing_checksum: - checksize, checktype, checksum = get_checksum_props(key, value) - cursor.execute( - "INSERT INTO filechecksum (file, checksize, checktype, checksum) VALUES (%s, %s, %s, %s)", - (file_id, checksize, checktype, checksum), - ) - - cursor.execute(f"UPDATE file SET detection = 1 WHERE id = {file_id}") - cursor.execute( - f"UPDATE file SET detection_type = '{d_type}' WHERE id = {file_id}" - ) - else: - cursor.execute( - f"UPDATE file SET detection_type = 'None' WHERE id = {file_id}" - ) - - def set_populate_file(fileset, fileset_id, conn, detection): """ Updates the old fileset in case of a match. Further deletes the newly created fileset which is not needed anymore. @@ -3133,11 +2526,11 @@ def user_integrity_check(data, ip, game_metadata=None): log_matched_fileset( src, matched_fileset_id, matched_fileset_id, "full", user, conn ) - elif status == "partial" and count == matched_count: - populate_file(data, matched_fileset_id, conn, None, src) - log_matched_fileset( - src, matched_fileset_id, matched_fileset_id, "partial", user, conn - ) + # elif status == "partial" and count == matched_count: + # populate_file(data, matched_fileset_id, conn, None, src) + # log_matched_fileset( + # src, matched_fileset_id, matched_fileset_id, "partial", user, conn + # ) elif status == "user" and count == matched_count: add_usercount(matched_fileset_id, conn) log_matched_fileset( diff --git a/fileset.py b/fileset.py index 7ee5dd85..4377d102 100644 --- a/fileset.py +++ b/fileset.py @@ -10,10 +10,6 @@ import json import html as html_lib import os -from user_fileset_functions import ( - user_insert_fileset, - match_and_merge_user_filesets, -) from pagination import create_page import difflib from db_functions import ( @@ -344,10 +340,6 @@ def fileset(): connection.commit() html += "

Fileset marked for deletion

" - if "match" in request.form: - match_and_merge_user_filesets(request.form["match"]) - return redirect(url_for("fileset", id=request.form["match"])) - # Generate the HTML for the fileset history cursor.execute( "SELECT `timestamp`, category, `text`, id FROM log WHERE `text` REGEXP 'Fileset:%s' ORDER BY `timestamp` DESC, id DESC", @@ -1134,28 +1126,24 @@ def validate(): json_response = {"error": error_codes["success"], "files": []} - if not game_metadata: - if not json_object.get("files"): - json_response["error"] = error_codes["empty"] - del json_response["files"] - json_response["status"] = "empty_fileset" - return jsonify(json_response) - - json_response["error"] = error_codes["no_metadata"] - del json_response["files"] - json_response["status"] = "no_metadata" - - conn = db_connect() - try: - fileset_id = user_insert_fileset(json_object, ip, conn) - finally: - conn.close() - json_response["fileset"] = fileset_id - return jsonify(json_response) - - matched_map = {} - missing_map = {} - extra_map = {} + # if not game_metadata: + # if not json_object.get("files"): + # json_response["error"] = error_codes["empty"] + # del json_response["files"] + # json_response["status"] = "empty_fileset" + # return jsonify(json_response) + + # json_response["error"] = error_codes["no_metadata"] + # del json_response["files"] + # json_response["status"] = "no_metadata" + + # conn = db_connect() + # try: + # fileset_id = user_insert_fileset(json_object, ip, conn) + # finally: + # conn.close() + # json_response["fileset"] = fileset_id + # return jsonify(json_response) file_object = json_object["files"] if not file_object: diff --git a/user_fileset_functions.py b/user_fileset_functions.py deleted file mode 100644 index 6ca1c1f0..00000000 --- a/user_fileset_functions.py +++ /dev/null @@ -1,205 +0,0 @@ -import hashlib -import time -from db_functions import ( - db_connect, - insert_fileset, - insert_file, - insert_filechecksum, - find_matching_game, - merge_filesets, - create_log, - calc_megakey, -) -import getpass -import pymysql - - -def user_calc_key(user_fileset): - key_string = "" - for file in user_fileset: - for key, value in file.items(): - if key != "checksums": - key_string += ":" + str(value) - continue - for checksum_pair in value: - key_string += ":" + checksum_pair["checksum"] - key_string = key_string.strip(":") - return hashlib.md5(key_string.encode()).hexdigest() - - -def file_json_to_array(file_json_object): - res = {} - for key, value in file_json_object.items(): - if key != "checksums": - res[key] = value - continue - for checksum_pair in value: - res[checksum_pair["type"]] = checksum_pair["checksum"] - return res - - -def user_insert_queue(user_fileset, conn): - query = "INSERT INTO queue (time, notes, fileset, ticketid, userid, commit) VALUES (%s, NULL, @fileset_last, NULL, NULL, NULL)" - - with conn.cursor() as cursor: - cursor.execute(query, (int(time.time()),)) - conn.commit() - - -def user_insert_fileset(user_fileset, ip, conn): - src = "user" - detection = False - key = "" - megakey = calc_megakey(user_fileset) - with conn.cursor() as cursor: - cursor.execute("SELECT MAX(`transaction`) FROM transactions") - transaction_id = cursor.fetchone()["MAX(`transaction`)"] + 1 - log_text = "from user submitted files" - cursor.execute("SET @fileset_time_last = %s", (int(time.time()),)) - if insert_fileset( - src, detection, key, megakey, transaction_id, log_text, conn, ip - ): - for file in user_fileset["files"]: - file = file_json_to_array(file) - insert_file(file, detection, src, conn) - for key, value in file.items(): - if key not in ["name", "size"]: - insert_filechecksum(file, key, conn) - cursor.execute("SELECT @fileset_last") - fileset_id = cursor.fetchone()["@fileset_last"] - conn.commit() - return fileset_id - - -def match_and_merge_user_filesets(id): - conn = db_connect() - - # Getting unmatched filesets - unmatched_filesets = [] - - with conn.cursor() as cursor: - cursor.execute( - "SELECT fileset.id, filechecksum.checksum, src, status FROM fileset JOIN file ON file.fileset = fileset.id JOIN filechecksum ON file.id = filechecksum.file WHERE status = 'user' AND fileset.id = %s", - (id,), - ) - unmatched_files = cursor.fetchall() - - # Splitting them into different filesets - i = 0 - while i < len(unmatched_files): - cur_fileset = unmatched_files[i][0] - temp = [] - while i < len(unmatched_files) and cur_fileset == unmatched_files[i][0]: - temp.append(unmatched_files[i]) - i += 1 - unmatched_filesets.append(temp) - - for fileset in unmatched_filesets: - matching_games = find_matching_game(fileset) - - if len(matching_games) != 1: # If there is no match/non-unique match - continue - - matched_game = matching_games[0] - - status = "full" - - # Convert NULL values to string with value NULL for printing - matched_game = {k: "NULL" if v is None else v for k, v in matched_game.items()} - - category_text = f"Matched from {fileset[0][2]}" - log_text = f"Matched game {matched_game['engineid']}:\n{matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']}\nvariant {matched_game['key']}. State {status}. Fileset:{fileset[0][0]}." - - # Updating the fileset.game value to be $matched_game["id"] - query = "UPDATE fileset SET game = %s, status = %s, `key` = %s WHERE id = %s" - - history_last = merge_filesets(matched_game["fileset"], fileset[0][0]) - - if cursor.execute( - query, (matched_game["id"], status, matched_game["key"], fileset[0][0]) - ): - user = f"cli:{getpass.getuser()}" - - # Merge log - create_log( - "Fileset merge", - user, - pymysql.escape_string( - conn, - f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0][0]}", - ), - ) - - # Matching log - log_last = create_log( - pymysql.escape_string(conn, category_text), - user, - pymysql.escape_string(conn, log_text), - ) - - # Add log id to the history table - cursor.execute( - "UPDATE history SET log = %s WHERE id = %s", (log_last, history_last) - ) - - if not conn.commit(): - print("Updating matched games failed") - with conn.cursor() as cursor: - cursor.execute( - """ - SELECT fileset.id, filechecksum.checksum, src, status - FROM fileset - JOIN file ON file.fileset = fileset.id - JOIN filechecksum ON file.id = filechecksum.file - WHERE status = 'user' AND fileset.id = %s - """, - (id,), - ) - unmatched_files = cursor.fetchall() - - unmatched_filesets = [] - cur_fileset = None - temp = [] - for file in unmatched_files: - if cur_fileset is None or cur_fileset != file["id"]: - if temp: - unmatched_filesets.append(temp) - cur_fileset = file["id"] - temp = [] - temp.append(file) - if temp: - unmatched_filesets.append(temp) - - for fileset in unmatched_filesets: - matching_games = find_matching_game(fileset) - if len(matching_games) != 1: - continue - matched_game = matching_games[0] - status = "full" - matched_game = { - k: ("NULL" if v is None else v) for k, v in matched_game.items() - } - category_text = f"Matched from {fileset[0]['src']}" - log_text = f"Matched game {matched_game['engineid']}: {matched_game['gameid']}-{matched_game['platform']}-{matched_game['language']} variant {matched_game['key']}. State {status}. Fileset:{fileset[0]['id']}." - query = """ - UPDATE fileset - SET game = %s, status = %s, `key` = %s - WHERE id = %s - """ - history_last = merge_filesets(matched_game["fileset"], fileset[0]["id"]) - with conn.cursor() as cursor: - cursor.execute( - query, - (matched_game["id"], status, matched_game["key"], fileset[0]["id"]), - ) - user = "cli:" + getpass.getuser() - create_log( - "Fileset merge", - user, - f"Merged Fileset:{matched_game['fileset']} and Fileset:{fileset[0]['id']}", - ) - log_last = create_log(category_text, user, log_text) - cursor.execute( - "UPDATE history SET log = %s WHERE id = %s", (log_last, history_last) - ) - conn.commit() From 4222dde49e5924e74fff20afac46c1acad47b20a Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Sun, 27 Jul 2025 01:53:39 +0530 Subject: [PATCH 16/47] INTEGRITY: Improve homepage navbar. --- fileset.py | 118 +++++++++++++++++++++++++------------------- index.html | 2 - pagination.py | 16 ++++-- static/style.css | 76 +++++++++++++++++++++++----- templates/home.html | 105 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 249 insertions(+), 68 deletions(-) delete mode 100644 index.html create mode 100644 templates/home.html diff --git a/fileset.py b/fileset.py index 4377d102..cdcbe0d4 100644 --- a/fileset.py +++ b/fileset.py @@ -5,6 +5,7 @@ url_for, render_template_string, jsonify, + render_template, ) import pymysql.cursors import json @@ -32,37 +33,12 @@ @app.route("/") def index(): - html = """ - - - - - - - -

Fileset Database

-

Fileset Actions

- -

Logs

- -
- -
- - - """ - return render_template_string(html) + return redirect(url_for("logs")) + + +@app.route("/home") +def home(): + return render_template("home.html") @app.route("/clear_database", methods=["POST"]) @@ -148,10 +124,18 @@ def fileset(): - @@ -505,7 +505,7 @@ def merge_fileset(id): User Games List Ready for review Fileset Search - Logs + Logs Config @@ -553,7 +553,7 @@ def merge_fileset(id): User Games List Ready for review Fileset Search - Logs + Logs Config @@ -621,7 +621,7 @@ def possible_merge_filesets(id): User Games List Ready for review Fileset Search - Logs + Logs Config @@ -824,7 +824,7 @@ def highlight_differences(source, target): User Games List Ready for review Fileset Search - Logs + Logs Config diff --git a/pagination.py b/pagination.py index 22f7930c..339fc819 100644 --- a/pagination.py +++ b/pagination.py @@ -22,6 +22,30 @@ def get_join_columns(table1, table2, mapping): return "No primary-foreign key mapping provided. Filter is invalid" +def build_search_condition(value, column): + phrases = re.findall(r'"([^"]+)"', value) + if phrases: + conditions = [f"{column} REGEXP '{re.escape(p)}'" for p in phrases] + return " AND ".join(conditions) + + if "+" in value: + and_terms = value.split("+") + and_conditions = [] + for term in and_terms: + or_terms = term.strip().split() + if len(or_terms) > 1: + or_cond = " OR ".join( + [f"{column} REGEXP '{re.escape(t)}'" for t in or_terms if t] + ) + and_conditions.append(f"({or_cond})") + else: + and_conditions.append(f"{column} REGEXP '{re.escape(term.strip())}'") + return " AND ".join(and_conditions) + else: + or_terms = value.split() + return " OR ".join([f"{column} REGEXP '{re.escape(t)}'" for t in or_terms if t]) + + def create_page( filename, results_per_page, @@ -46,62 +70,47 @@ def create_page( ) with conn.cursor() as cursor: - # Handle sorting - sort = request.args.get("sort") - if sort: - column = sort.split("-") - order = f"ORDER BY {column[0]}" - if "desc" in sort: - order += " DESC" + tables = set() + where_clauses = [] - if set(request.args.keys()).difference({"page", "sort"}): - condition = "WHERE " - tables = set() - for key, value in request.args.items(): - if key in ["page", "sort"] or value == "": - continue - tables.add(filters[key]) - if value == "": - value = ".*" - condition += ( - f" AND {filters[key]}.{'id' if key == 'fileset' else key} REGEXP '{value}'" - if condition != "WHERE " - else f"{filters[key]}.{'id' if key == 'fileset' else key} REGEXP '{value}'" - ) + for key, value in request.args.items(): + if key in ("page", "sort") or value == "": + continue + tables.add(filters[key]) + col = f"{filters[key]}.{'id' if key == 'fileset' else key}" + parsed = build_search_condition(value, col) + if parsed: + where_clauses.append(parsed) - if condition == "WHERE ": - condition = "" + condition = "" + if where_clauses: + condition = "WHERE " + " AND ".join(where_clauses) - # Handle multiple tables - from_query = records_table - join_order = ["game", "engine"] - tables_list = sorted( - list(tables), - key=lambda t: join_order.index(t) if t in join_order else 99, - ) - if records_table not in tables_list or len(tables_list) > 1: - for table in tables_list: - if table == records_table: - continue - if table == "engine": - if "game" in tables: - from_query += " JOIN engine ON engine.id = game.engine" - else: - from_query += " JOIN game ON game.id = fileset.game JOIN engine ON engine.id = game.engine" + from_query = records_table + join_order = ["game", "engine"] + tables_list = sorted( + list(tables), key=lambda t: join_order.index(t) if t in join_order else 99 + ) + + if records_table not in tables_list or len(tables_list) > 1: + for t in tables_list: + if t == records_table: + continue + if t == "engine": + if "game" in tables: + from_query += " JOIN engine ON engine.id = game.engine" else: - from_query += f" JOIN {table} ON {get_join_columns(records_table, table, mapping)}" - cursor.execute( - f"SELECT COUNT({records_table}.id) AS count FROM {from_query} {condition}" - ) - num_of_results = cursor.fetchone()["count"] + from_query += " JOIN game ON game.id = fileset.game JOIN engine ON engine.id = game.engine" + else: + from_query += ( + f" JOIN {t} ON {get_join_columns(records_table, t, mapping)}" + ) - elif "JOIN" in records_table: - first_table = records_table.split(" ")[0] - cursor.execute(f"SELECT COUNT({first_table}.id) FROM {records_table}") - num_of_results = cursor.fetchone()[f"COUNT({first_table}.id)"] - else: - cursor.execute(f"SELECT COUNT(id) FROM {records_table}") - num_of_results = cursor.fetchone()["COUNT(id)"] + base_table = records_table.split(" ")[0] + cursor.execute( + f"SELECT COUNT({base_table}.id) AS count FROM {from_query} {condition}" + ) + num_of_results = cursor.fetchone()["count"] num_of_pages = (num_of_results + results_per_page - 1) // results_per_page print(f"Num of results: {num_of_results}, Num of pages: {num_of_pages}") @@ -110,29 +119,21 @@ def create_page( page = max(1, min(page, num_of_pages)) offset = (page - 1) * results_per_page - # Fetch results - if set(request.args.keys()).difference({"page"}): - condition = "WHERE " - for key, value in request.args.items(): - if key not in filters: - continue - - value = pymysql.converters.escape_string(value) - if value == "": - value = ".*" - field = f"{filters[key]}.{'id' if key == 'fileset' else key}" - if value == ".*": - clause = f"({field} IS NULL OR {field} REGEXP '{value}')" - else: - clause = f"{field} REGEXP '{value}'" - condition += f" AND {clause}" if condition != "WHERE " else clause - - if condition == "WHERE ": - condition = "" - - query = f"{select_query} {condition} {order} LIMIT {results_per_page} OFFSET {offset}" + # Sort + order = "" + sort_param = request.args.get("sort") + if sort_param: + sort_parts = sort_param.split("-") + sort_col = sort_parts[0] + order = f"ORDER BY {sort_col}" + if "desc" in sort_param: + order += " DESC" else: - query = f"{select_query} {order} LIMIT {results_per_page} OFFSET {offset}" + if records_table == "log": + order = "ORDER BY `id` DESC" + + # Fetch results + query = f"{select_query} {condition} {order} LIMIT {results_per_page} OFFSET {offset}" cursor.execute(query) results = cursor.fetchall() @@ -154,7 +155,7 @@ def create_page( User Games List Ready for review Fileset Search - Logs + Logs Config diff --git a/templates/config.html b/templates/config.html index a578779d..d730f153 100644 --- a/templates/config.html +++ b/templates/config.html @@ -117,7 +117,7 @@ User Games List Ready for review Fileset Search - Logs + Logs Config diff --git a/templates/home.html b/templates/home.html index c41cda05..ec093df7 100644 --- a/templates/home.html +++ b/templates/home.html @@ -89,7 +89,7 @@ User Games List Ready for review Fileset Search - Logs + Logs Config
From 45cbd16ea347fc54bac2d8ed18ea6a6bc4caa9a8 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Thu, 31 Jul 2025 13:23:33 +0530 Subject: [PATCH 26/47] INTEGRITY: Add separate logs per page and filesets per page in config. --- fileset.py | 84 +++++++++++++++---------------------------- templates/config.html | 81 +++++++++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 85 deletions(-) diff --git a/fileset.py b/fileset.py index 4eae1dd0..3ba6f995 100644 --- a/fileset.py +++ b/fileset.py @@ -1383,26 +1383,41 @@ def config(): Stores the user configurations in the cookies """ if request.method == "POST": - items_per_page = request.form.get("items_per_page", "25") + filesets_per_page = request.form.get("filesets_per_page", "25") + logs_per_page = request.form.get("logs_per_page", "25") try: - items_per_page_int = int(items_per_page) - if items_per_page_int < 1: - items_per_page = "1" + filesets_per_page_int = int(filesets_per_page) + logs_per_page_int = int(logs_per_page) + if filesets_per_page_int < 1: + filesets_per_page = "1" + if logs_per_page_int < 1: + logs_per_page_int = "1" except ValueError: - items_per_page = "25" + filesets_per_page = "25" + logs_per_page = "25" resp = make_response(redirect(url_for("config"))) - resp.set_cookie("items_per_page", items_per_page, max_age=365 * 24 * 60 * 60) + resp.set_cookie( + "filesets_per_page", filesets_per_page, max_age=365 * 24 * 60 * 60 + ) + resp.set_cookie("logs_per_page", logs_per_page, max_age=365 * 24 * 60 * 60) return resp - items_per_page = int(request.cookies.get("items_per_page", "25")) + filesets_per_page = int(request.cookies.get("filesets_per_page", "25")) + logs_per_page = int(request.cookies.get("logs_per_page", "25")) + + return render_template( + "config.html", filesets_per_page=filesets_per_page, logs_per_page=logs_per_page + ) + - return render_template("config.html", items_per_page=items_per_page) +def get_filesets_per_page(): + return int(request.cookies.get("filesets_per_page", "25")) -def get_items_per_page(): - return int(request.cookies.get("items_per_page", "25")) +def get_logs_per_page(): + return int(request.cookies.get("logs_per_page", "25")) @app.route("/validate", methods=["POST"]) @@ -1497,47 +1512,6 @@ def ready_for_review(): return redirect(url) -@app.route("/games_list") -def games_list(): - filename = "games_list" - records_table = "game" - select_query = """ - SELECT engineid, gameid, extra, platform, language, game.name, - status, fileset.id as fileset - FROM game - JOIN engine ON engine.id = game.engine - JOIN fileset ON game.id = fileset.game - """ - order = "ORDER BY gameid" - filters = { - "engineid": "engine", - "gameid": "game", - "extra": "game", - "platform": "game", - "language": "game", - "name": "game", - "status": "fileset", - } - mapping = { - "engine.id": "game.engine", - "game.id": "fileset.game", - } - - items_per_page = get_items_per_page() - - return render_template_string( - create_page( - filename, - items_per_page, - records_table, - select_query, - order, - filters, - mapping, - ) - ) - - @app.route("/logs") def logs(): filename = "logs" @@ -1551,10 +1525,10 @@ def logs(): "user": "log", "text": "log", } - items_per_page = get_items_per_page() + logs_per_page = get_logs_per_page() return render_template_string( create_page( - filename, items_per_page, records_table, select_query, order, filters + filename, logs_per_page, records_table, select_query, order, filters ) ) @@ -1588,11 +1562,11 @@ def fileset_search(): "engine.id": "game.engine", "fileset.id": "transactions.fileset", } - items_per_page = get_items_per_page() + filesets_per_page = get_filesets_per_page() return render_template_string( create_page( filename, - items_per_page, + filesets_per_page, records_table, select_query, order, diff --git a/templates/config.html b/templates/config.html index d730f153..a524ac3c 100644 --- a/templates/config.html +++ b/templates/config.html @@ -23,7 +23,6 @@ background-color: #ffffff; color: #000000; padding: 10px; - font-size: 50px; align-self: flex-start; margin-left: 2vh; } @@ -35,51 +34,58 @@ .config-section { margin-bottom: 30px; - padding: 20px; + padding: 25px; + padding-left: 50px; + padding-right: 50px; border: 1px solid #ddd; border-radius: 8px; background-color: #f9f9f9; - max-width: 600px; - margin-left: auto; - margin-right: auto; + width: 100%; + box-sizing: border-box; } .config-item { display: flex; - align-items: center; - margin-bottom: 15px; + flex-direction: column; + margin-bottom: 20px; + gap: 10px; } .config-item label { - flex: 1; - margin-right: 15px; font-weight: 500; + font-size: 14px; + color: #333; } .current-value { font-style: italic; color: #666; font-size: 12px; + margin-top: 4px; + } + + .input-container { + display: flex; + align-items: center; + gap: 15px; + margin-bottom: 20px; + } + + .submit-section { + padding: 20px 0; + border-top: 1px solid #ddd; + margin-top: 20px; } .config-item input, .config-item select { - padding: 8px 12px; + padding: 10px 12px; border: 1px solid #ccc; - border-radius: 4px; + border-radius: 6px; font-size: 14px; - min-width: 120px; - margin-right: 2vw; - } - - .success-message { - background-color: #d4edda; - color: #155724; - padding: 10px; - border: 1px solid #c3e6cb; - border-radius: 4px; - margin-bottom: 20px; - text-align: center; + width: 200px; + box-sizing: border-box; + transition: border-color 0.3s ease; } @media (max-width: 768px) { @@ -124,24 +130,39 @@

User Configurations

- +
- - -
Current: {{ items_per_page }}
+ +
+ +
Default: 25
+
+ +
+ +
Default: 25
+
-
+
+
From 6b44b812e3506696f5c7298ee8f6fcd54760b497 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Thu, 31 Jul 2025 14:04:14 +0530 Subject: [PATCH 27/47] INTEGRITY: Add/update deployment related files. --- apache2-config/gamesdb.sev.zone.conf | 11 +++++---- app.wsgi | 12 ++++++++++ fileset.py | 2 +- requirements.txt | 36 ++++++++++++++-------------- 4 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 app.wsgi diff --git a/apache2-config/gamesdb.sev.zone.conf b/apache2-config/gamesdb.sev.zone.conf index 8b37f5be..43563729 100644 --- a/apache2-config/gamesdb.sev.zone.conf +++ b/apache2-config/gamesdb.sev.zone.conf @@ -4,12 +4,15 @@ ServerAdmin webmaster@localhost CustomLog ${APACHE_LOG_DIR}/integrity-access.log combined ErrorLog ${APACHE_LOG_DIR}/integrity-error.log - DocumentRoot /home/ubuntu/projects/python/scummvm-sites + DocumentRoot /home/ubuntu/projects/python/scummvm_sites_2025/scummvm-sites WSGIDaemonProcess scummvm-sites user=www-data group=www-data threads=5 - WSGIScriptAlias / /home/ubuntu/projects/python/scummvm-sites/app.wsgi + WSGIScriptAlias / /home/ubuntu/projects/python/scummvm_sites_2025/scummvm-sites/app.wsgi - - Require all granted + + AuthType Basic + AuthName "nope" + AuthUserFile /home/ubuntu/projects/python/scummvm_sites_2025/.htpasswd + Require valid-user diff --git a/app.wsgi b/app.wsgi new file mode 100644 index 00000000..a52d3aba --- /dev/null +++ b/app.wsgi @@ -0,0 +1,12 @@ +import sys +import logging + +sys.path.insert(0, "/home/ubuntu/projects/python/scummvm_sites_2025/scummvm-sites") + +from fileset import app as application + +logging.basicConfig(stream=sys.stderr) +sys.stderr = sys.stdout + +if __name__ == "__main__": + application.run() diff --git a/fileset.py b/fileset.py index 3ba6f995..42d07d31 100644 --- a/fileset.py +++ b/fileset.py @@ -1597,4 +1597,4 @@ def delete_files(id): if __name__ == "__main__": app.secret_key = secret_key - app.run(port=5001, debug=True, host="0.0.0.0") + app.run(debug=True, host="0.0.0.0") diff --git a/requirements.txt b/requirements.txt index 8486da70..8340e269 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ -blinker==1.9.0 -cffi==1.17.1 -click==8.1.8 -cryptography==45.0.3 -Flask==3.1.0 -iniconfig==2.1.0 -itsdangerous==2.2.0 -Jinja2==3.1.5 -MarkupSafe==3.0.2 -packaging==25.0 -pluggy==1.6.0 -pycparser==2.22 -Pygments==2.19.1 -PyMySQL==1.1.1 -pytest==8.4.0 -setuptools==75.8.0 -Werkzeug==3.1.3 -wheel==0.45.1 +blinker +cffi +click +cryptography +Flask +iniconfig +itsdangerous +Jinja2 +MarkupSafe +packaging +pluggy +pycparser +Pygments +PyMySQL +pytest +setuptools +Werkzeug +wheel From 7aabe392e9e982633b76061d3753e733959f6906 Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Thu, 31 Jul 2025 14:29:05 +0530 Subject: [PATCH 28/47] INTEGRITY: Add underline on hover for navbar links. --- static/style.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/static/style.css b/static/style.css index 527824b7..b93da503 100644 --- a/static/style.css +++ b/static/style.css @@ -67,9 +67,9 @@ nav { margin-left: 10px; } -/* .nav-buttons a:hover { - box-shadow: 0 4px 12px rgba(39, 145, 232, 0.4); -} */ +.nav-buttons a:hover { + text-decoration: underline; +} .logo img { height: 75px; From 9ac4cf7b14776c1991971619ab6f3c3a5a3931ed Mon Sep 17 00:00:00 2001 From: ShivangNagta Date: Thu, 31 Jul 2025 14:47:19 +0530 Subject: [PATCH 29/47] INTEGRITY: Add favicon. --- fileset.py | 10 ++++++++++ pagination.py | 2 ++ templates/config.html | 2 ++ templates/home.html | 2 ++ 4 files changed, 16 insertions(+) diff --git a/fileset.py b/fileset.py index 42d07d31..db0c89c7 100644 --- a/fileset.py +++ b/fileset.py @@ -129,6 +129,8 @@ def fileset(): + +