rmsitz/offlineu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add delete and ignore actions to Duplicate Courses

Browse Source 
Delete permanently removes a course from disk (confirmed with the full
path first) and cleans up any Hidden/Next Up/Recently Viewed
references to it. Ignore marks a specific pair as confirmed-not-
duplicates so it stops resurfacing in scans, without suppressing
either course's other matches; ignored pairs are listed and reversible
under "Ignored matches," and now travel with backup/restore.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
rmsitz
2026-08-24 11:11:41 -04:00
co-authored by Claude Sonnet 5
parent 9f0f49a399
commit 49b68b8e51
3 changed files with 296 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
offlineu_core.py
+138 -2
View File
@@ -12,6 +12,7 @@ import re
import sys
import argparse
import io
import itertools
import shutil
import subprocess
import threading
@@ -1450,6 +1451,47 @@ def _propose_destination(item_name: str, categories: List[Dict[str, Any]],
}
IGNORED_DUPLICATES_FILE = os.path.join(DATA_DIR, 'ignored_duplicates.json')
def get_ignored_duplicate_pairs() -> Set[Tuple[str, str]]:
"""Pairs of course paths the user has confirmed are NOT duplicates of
each other, despite matching the tokenizer - excluded from future
duplicate scans (see _find_duplicate_courses) so a wrong call doesn't
keep resurfacing. Each pair is stored path-order-independent."""
try:
if os.path.exists(IGNORED_DUPLICATES_FILE):
with open(IGNORED_DUPLICATES_FILE, 'r') as f:
data = json.load(f)
if isinstance(data, list):
return {tuple(sorted(pair)) for pair in data if isinstance(pair, list) and len(pair) == 2}
except (json.JSONDecodeError, OSError) as e:
print(f"Could not load ignored duplicates: {e}")
return set()
def _save_ignored_duplicate_pairs(pairs: Set[Tuple[str, str]]) -> None:
os.makedirs(DATA_DIR, exist_ok=True)
with open(IGNORED_DUPLICATES_FILE, 'w') as f:
json.dump([list(p) for p in sorted(pairs)], f, indent=2)
def add_ignored_duplicate_pairs(pairs: List[Tuple[str, str]]) -> None:
"""Mark one or more course-path pairs as confirmed-not-duplicates."""
existing = get_ignored_duplicate_pairs()
existing |= {tuple(sorted(p)) for p in pairs}
_save_ignored_duplicate_pairs(existing)
def remove_ignored_duplicate_pair(path_a: str, path_b: str) -> None:
"""Undo add_ignored_duplicate_pairs for one pair - lets a wrong ignore
call be reversed, matching the reversible-curation pattern hidden
paths/Next Up already follow elsewhere in the app."""
existing = get_ignored_duplicate_pairs()
existing.discard(tuple(sorted((path_a, path_b))))
_save_ignored_duplicate_pairs(existing)
def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]:
"""
Group courses across the whole library (Unsorted included, so a
@@ -1464,9 +1506,12 @@ def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]
Pairwise Jaccard similarity (shared tokens / all tokens) at or above
min_similarity gets union-found into groups, so if A matches B and B
matches C, all three land in one group instead of two overlapping
pairs. Nothing here touches disk - it's read-only, for the user to
review and hide/rename/move manually.
pairs - except any pair the user has explicitly marked as not-actually-
duplicates (get_ignored_duplicate_pairs), which never unions regardless
of how similar the names look. Nothing here touches disk - it's
read-only, for the user to review and hide/delete/rename/move manually.
"""
ignored_pairs = get_ignored_duplicate_pairs()
entries = []
for course in get_all_course_dirs():
tokens = _tokenize(course.name)
@@ -1490,6 +1535,8 @@ def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]
pair_similarity: Dict[Tuple[int, int], float] = {}
for i in range(n):
for j in range(i + 1, n):
if tuple(sorted((entries[i]['path'], entries[j]['path']))) in ignored_pairs:
continue
a, b = entries[i]['tokens'], entries[j]['tokens']
union_tokens = a | b
if not union_tokens:
@@ -3053,6 +3100,94 @@ def duplicate_courses_api():
return jsonify({'groups': groups, 'course_count': len(get_all_course_dirs())})
@app.route('/api/library/duplicates/ignore', methods=['POST'])
def ignore_duplicate_group_api():
"""Mark every pairwise combination within a reviewed duplicate group as
confirmed-not-duplicates, so the same match doesn't resurface on the
next scan."""
data = request.json or {}
paths = data.get('paths') or []
if not isinstance(paths, list) or len(paths) < 2:
return jsonify({'success': False, 'error': 'Need at least 2 paths'}), 400
pairs = list(itertools.combinations(sorted(set(paths)), 2))
add_ignored_duplicate_pairs(pairs)
return jsonify({'success': True, 'ignored_pairs': len(pairs)})
@app.route('/api/library/duplicates/ignored')
def list_ignored_duplicates_api():
"""Every pair currently excluded from duplicate scans, for a 'currently
ignored' review list - the same reversible-curation pattern hidden
paths already use."""
result = []
for path_a, path_b in sorted(get_ignored_duplicate_pairs()):
result.append({
'a': {'path': path_a, 'name': os.path.basename(path_a.rstrip(os.sep)) or path_a},
'b': {'path': path_b, 'name': os.path.basename(path_b.rstrip(os.sep)) or path_b},
})
return jsonify({'ignored': result})
@app.route('/api/library/duplicates/ignore', methods=['DELETE'])
def restore_ignored_duplicate_api():
"""Un-ignore one pair, so it's eligible to show up in duplicate scans again."""
data = request.json or {}
path_a = data.get('a', '')
path_b = data.get('b', '')
if not path_a or not path_b:
return jsonify({'success': False, 'error': 'Missing pair'}), 400
remove_ignored_duplicate_pair(path_a, path_b)
return jsonify({'success': True})
@app.route('/api/library/delete', methods=['POST'])
def delete_library_item_api():
"""
Permanently delete a course/folder from disk - the one truly
destructive action in this app, so it's scoped tightly: the path must
be inside the library root and can't be the root itself. The client is
expected to have already confirmed with the user (Duplicate Courses is
the only caller today). Also clears any hidden/Next-Up/Recently-Viewed
references to the deleted path, the same cleanup Clean Up Stale
References would otherwise have to do after the fact.
"""
data = request.json or {}
path = (data.get('path') or '').strip()
if not path:
return jsonify({'success': False, 'error': 'Missing path'}), 400
library_root = os.path.abspath(get_library_root())
target_abs = os.path.abspath(path)
if not (target_abs == library_root or target_abs.startswith(library_root + os.sep)):
return jsonify({'success': False, 'error': 'Path is outside the library'}), 403
if target_abs == library_root:
return jsonify({'success': False, 'error': 'Cannot delete the library root itself'}), 400
if not os.path.isdir(target_abs):
return jsonify({'success': False, 'error': 'No longer exists - already deleted?'}), 404
try:
shutil.rmtree(target_abs)
except OSError as e:
return jsonify({'success': False, 'error': str(e)}), 500
invalidate_cache()
set_path_hidden(target_abs, False)
set_path_queued(target_abs, False)
remove_recent_views_for_path(target_abs)
global current_course
active_course_reset = False
if current_course is not None:
course_abs = os.path.abspath(current_course.path)
if course_abs == target_abs or course_abs.startswith(target_abs + os.sep):
current_course = None
active_course_reset = True
return jsonify({'success': True, 'active_course_reset': active_course_reset})
@app.route('/api/library/stale-references')
def stale_references_api():
"""Entries in hidden_paths/next_up/recent_views pointing at paths no
@@ -3432,6 +3567,7 @@ BACKUP_ROOT_FILES = {
'next_up.json': NEXT_UP_FILE,
'recent_views.json': RECENT_VIEWS_FILE,
'outline_config.json': OUTLINE_CONFIG_FILE,
'ignored_duplicates.json': IGNORED_DUPLICATES_FILE,
}