diff --git a/README.md b/README.md
index eafe2c5..03f0170 100644
--- a/README.md
+++ b/README.md
@@ -128,7 +128,14 @@ silently stay blank instead of erroring.
same tokenizer as Sort Unsorted so a platform-name or release-tag
difference doesn't hide a real duplicate. Groups by similarity (union of
any two courses over the threshold, transitively) rather than showing
- raw pairs; each course in a group gets a one-click Hide.
+ raw pairs; each course in a group gets a one-click Hide, or a **Delete**
+ that permanently removes it from disk (the one destructive action in the
+ whole app - confirmed with the full path before it runs). A group can
+ also be marked **"Not a duplicate"** if the match is wrong, which
+ excludes that specific pair from future scans without touching anything
+ else that happens to match one of those courses; ignored pairs are
+ listed (and reversible) under "Ignored matches," and persist through
+ backup/restore alongside hidden paths and Next Up.
- *Clean Up Stale References*: finds entries in the hidden-paths list,
Next Up queue, or Recently Viewed history that point at a path no longer
on disk - normally from renaming/moving/deleting a course directly on
@@ -177,8 +184,9 @@ silently stay blank instead of erroring.
**Backup & integrations**
- One-click backup export and restore: settings, hidden-path choices, the
- Next Up queue, recent-view history, Outline config, and every course's
- progress/notes as a zip (not the course files themselves). Restore
+ Next Up queue, recent-view history, ignored-duplicate pairs, Outline
+ config, and every course's progress/notes as a zip (not the course files
+ themselves). Restore
overwrites current data and needs a matching course folder to already
exist for each course's progress to land - it's a "put my data back"
action, not a merge
@@ -197,6 +205,7 @@ Everything under `OFFLINEU_DATA_DIR` (app-wide, not tied to a course):
| `hidden_paths.json` | Courses/folders curated out of the browser |
| `next_up.json` | The Next Up queue, in order |
| `recent_views.json` | Cross-course "Recently Viewed" history |
+| `ignored_duplicates.json` | Course-path pairs confirmed not duplicates |
| `outline_config.json` | Outline API token/collection mapping |
Per-course, written inside the course's own folder on the library volume:
diff --git a/offlineu_core.py b/offlineu_core.py
index 5dba6aa..b948d63 100644
--- a/offlineu_core.py
+++ b/offlineu_core.py
@@ -12,6 +12,7 @@ import re
import sys
import argparse
import io
+import itertools
import shutil
import subprocess
import threading
@@ -1450,6 +1451,47 @@ def _propose_destination(item_name: str, categories: List[Dict[str, Any]],
}
+IGNORED_DUPLICATES_FILE = os.path.join(DATA_DIR, 'ignored_duplicates.json')
+
+
+def get_ignored_duplicate_pairs() -> Set[Tuple[str, str]]:
+ """Pairs of course paths the user has confirmed are NOT duplicates of
+ each other, despite matching the tokenizer - excluded from future
+ duplicate scans (see _find_duplicate_courses) so a wrong call doesn't
+ keep resurfacing. Each pair is stored path-order-independent."""
+ try:
+ if os.path.exists(IGNORED_DUPLICATES_FILE):
+ with open(IGNORED_DUPLICATES_FILE, 'r') as f:
+ data = json.load(f)
+ if isinstance(data, list):
+ return {tuple(sorted(pair)) for pair in data if isinstance(pair, list) and len(pair) == 2}
+ except (json.JSONDecodeError, OSError) as e:
+ print(f"Could not load ignored duplicates: {e}")
+ return set()
+
+
+def _save_ignored_duplicate_pairs(pairs: Set[Tuple[str, str]]) -> None:
+ os.makedirs(DATA_DIR, exist_ok=True)
+ with open(IGNORED_DUPLICATES_FILE, 'w') as f:
+ json.dump([list(p) for p in sorted(pairs)], f, indent=2)
+
+
+def add_ignored_duplicate_pairs(pairs: List[Tuple[str, str]]) -> None:
+ """Mark one or more course-path pairs as confirmed-not-duplicates."""
+ existing = get_ignored_duplicate_pairs()
+ existing |= {tuple(sorted(p)) for p in pairs}
+ _save_ignored_duplicate_pairs(existing)
+
+
+def remove_ignored_duplicate_pair(path_a: str, path_b: str) -> None:
+ """Undo add_ignored_duplicate_pairs for one pair - lets a wrong ignore
+ call be reversed, matching the reversible-curation pattern hidden
+ paths/Next Up already follow elsewhere in the app."""
+ existing = get_ignored_duplicate_pairs()
+ existing.discard(tuple(sorted((path_a, path_b))))
+ _save_ignored_duplicate_pairs(existing)
+
+
def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]:
"""
Group courses across the whole library (Unsorted included, so a
@@ -1464,9 +1506,12 @@ def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]
Pairwise Jaccard similarity (shared tokens / all tokens) at or above
min_similarity gets union-found into groups, so if A matches B and B
matches C, all three land in one group instead of two overlapping
- pairs. Nothing here touches disk - it's read-only, for the user to
- review and hide/rename/move manually.
+ pairs - except any pair the user has explicitly marked as not-actually-
+ duplicates (get_ignored_duplicate_pairs), which never unions regardless
+ of how similar the names look. Nothing here touches disk - it's
+ read-only, for the user to review and hide/delete/rename/move manually.
"""
+ ignored_pairs = get_ignored_duplicate_pairs()
entries = []
for course in get_all_course_dirs():
tokens = _tokenize(course.name)
@@ -1490,6 +1535,8 @@ def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]
pair_similarity: Dict[Tuple[int, int], float] = {}
for i in range(n):
for j in range(i + 1, n):
+ if tuple(sorted((entries[i]['path'], entries[j]['path']))) in ignored_pairs:
+ continue
a, b = entries[i]['tokens'], entries[j]['tokens']
union_tokens = a | b
if not union_tokens:
@@ -3053,6 +3100,94 @@ def duplicate_courses_api():
return jsonify({'groups': groups, 'course_count': len(get_all_course_dirs())})
+@app.route('/api/library/duplicates/ignore', methods=['POST'])
+def ignore_duplicate_group_api():
+ """Mark every pairwise combination within a reviewed duplicate group as
+ confirmed-not-duplicates, so the same match doesn't resurface on the
+ next scan."""
+ data = request.json or {}
+ paths = data.get('paths') or []
+ if not isinstance(paths, list) or len(paths) < 2:
+ return jsonify({'success': False, 'error': 'Need at least 2 paths'}), 400
+
+ pairs = list(itertools.combinations(sorted(set(paths)), 2))
+ add_ignored_duplicate_pairs(pairs)
+ return jsonify({'success': True, 'ignored_pairs': len(pairs)})
+
+
+@app.route('/api/library/duplicates/ignored')
+def list_ignored_duplicates_api():
+ """Every pair currently excluded from duplicate scans, for a 'currently
+ ignored' review list - the same reversible-curation pattern hidden
+ paths already use."""
+ result = []
+ for path_a, path_b in sorted(get_ignored_duplicate_pairs()):
+ result.append({
+ 'a': {'path': path_a, 'name': os.path.basename(path_a.rstrip(os.sep)) or path_a},
+ 'b': {'path': path_b, 'name': os.path.basename(path_b.rstrip(os.sep)) or path_b},
+ })
+ return jsonify({'ignored': result})
+
+
+@app.route('/api/library/duplicates/ignore', methods=['DELETE'])
+def restore_ignored_duplicate_api():
+ """Un-ignore one pair, so it's eligible to show up in duplicate scans again."""
+ data = request.json or {}
+ path_a = data.get('a', '')
+ path_b = data.get('b', '')
+ if not path_a or not path_b:
+ return jsonify({'success': False, 'error': 'Missing pair'}), 400
+ remove_ignored_duplicate_pair(path_a, path_b)
+ return jsonify({'success': True})
+
+
+@app.route('/api/library/delete', methods=['POST'])
+def delete_library_item_api():
+ """
+ Permanently delete a course/folder from disk - the one truly
+ destructive action in this app, so it's scoped tightly: the path must
+ be inside the library root and can't be the root itself. The client is
+ expected to have already confirmed with the user (Duplicate Courses is
+ the only caller today). Also clears any hidden/Next-Up/Recently-Viewed
+ references to the deleted path, the same cleanup Clean Up Stale
+ References would otherwise have to do after the fact.
+ """
+ data = request.json or {}
+ path = (data.get('path') or '').strip()
+ if not path:
+ return jsonify({'success': False, 'error': 'Missing path'}), 400
+
+ library_root = os.path.abspath(get_library_root())
+ target_abs = os.path.abspath(path)
+
+ if not (target_abs == library_root or target_abs.startswith(library_root + os.sep)):
+ return jsonify({'success': False, 'error': 'Path is outside the library'}), 403
+ if target_abs == library_root:
+ return jsonify({'success': False, 'error': 'Cannot delete the library root itself'}), 400
+ if not os.path.isdir(target_abs):
+ return jsonify({'success': False, 'error': 'No longer exists - already deleted?'}), 404
+
+ try:
+ shutil.rmtree(target_abs)
+ except OSError as e:
+ return jsonify({'success': False, 'error': str(e)}), 500
+
+ invalidate_cache()
+ set_path_hidden(target_abs, False)
+ set_path_queued(target_abs, False)
+ remove_recent_views_for_path(target_abs)
+
+ global current_course
+ active_course_reset = False
+ if current_course is not None:
+ course_abs = os.path.abspath(current_course.path)
+ if course_abs == target_abs or course_abs.startswith(target_abs + os.sep):
+ current_course = None
+ active_course_reset = True
+
+ return jsonify({'success': True, 'active_course_reset': active_course_reset})
+
+
@app.route('/api/library/stale-references')
def stale_references_api():
"""Entries in hidden_paths/next_up/recent_views pointing at paths no
@@ -3432,6 +3567,7 @@ BACKUP_ROOT_FILES = {
'next_up.json': NEXT_UP_FILE,
'recent_views.json': RECENT_VIEWS_FILE,
'outline_config.json': OUTLINE_CONFIG_FILE,
+ 'ignored_duplicates.json': IGNORED_DUPLICATES_FILE,
}
diff --git a/templates/unsorted.html b/templates/unsorted.html
index c1f5ad0..535d9a0 100644
--- a/templates/unsorted.html
+++ b/templates/unsorted.html
@@ -247,6 +247,10 @@
margin-bottom: 10px;
}
.dup-group-header {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 10px;
font-size: 0.78em;
font-weight: 600;
text-transform: uppercase;
@@ -304,6 +308,14 @@
color: var(--text-primary);
}
.btn-secondary:hover { background: var(--bg-tertiary-hover); }
+ .btn-danger {
+ background: transparent;
+ color: var(--error);
+ border: 1px solid var(--error);
+ }
+ .btn-danger:hover {
+ background: rgba(255, 107, 107, 0.15);
+ }
.btn-sm {
padding: 6px 12px;
font-size: 0.85em;
@@ -533,6 +545,11 @@
+
+
+
+
Confirmed not duplicates
+
@@ -1223,12 +1240,16 @@
status.textContent = `${groups.length} possible duplicate group${groups.length === 1 ? '' : 's'}, out of ${data.course_count} courses.`;
results.innerHTML = groups.map(group => `
-
${Math.round(group.similarity * 100)}% similar
+
+ ${Math.round(group.similarity * 100)}% similar
+
+
${group.courses.map(c => `
${escapeHtml(c.name)}${escapeHtml(c.path)}
+
`).join('')}
@@ -1259,6 +1280,130 @@
.catch(() => {});
}
+ function deleteDuplicateCourse(btnEl, path, name) {
+ if (!confirm(`Permanently delete "${name}" and everything inside it?\n\n${path}\n\nThis cannot be undone.`)) return;
+ btnEl.disabled = true;
+ fetch('/api/library/delete', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ path })
+ })
+ .then(r => r.json().then(data => ({ ok: r.ok, data })))
+ .then(({ ok, data }) => {
+ if (!ok || !data.success) {
+ btnEl.disabled = false;
+ alert(data.error || 'Delete failed');
+ return;
+ }
+ if (data.active_course_reset) {
+ alert('That was your active course - reload the main page to pick a different one.');
+ }
+ const row = btnEl.closest('.dup-course-row');
+ const group = row ? row.closest('.dup-group') : null;
+ if (row) row.remove();
+ if (group && group.querySelectorAll('.dup-course-row').length < 2) {
+ group.remove();
+ }
+ refreshDupStatusText();
+ })
+ .catch(() => {
+ btnEl.disabled = false;
+ alert('Could not reach the server.');
+ });
+ }
+
+ // Keeps the "N possible duplicate groups" line honest after a
+ // client-side removal (delete/ignore), without needing a full
+ // re-scan that would lose the user's place in a long results list.
+ function refreshDupStatusText() {
+ const status = document.getElementById('dup-status');
+ const remaining = document.querySelectorAll('#dup-results .dup-group').length;
+ if (remaining === 0) {
+ status.style.color = 'var(--success)';
+ status.textContent = 'No possible duplicates remaining.';
+ } else {
+ status.style.color = 'var(--text-muted)';
+ status.textContent = `${remaining} possible duplicate group${remaining === 1 ? '' : 's'} remaining.`;
+ }
+ }
+
+ function ignoreDuplicateGroup(btnEl) {
+ const group = btnEl.closest('.dup-group');
+ if (!group) return;
+ const paths = Array.from(group.querySelectorAll('.dup-course-path')).map(el => el.textContent);
+ if (paths.length < 2) return;
+ if (!confirm('Mark these as NOT duplicates of each other? They won\'t be flagged together again (reversible from "Ignored matches").')) return;
+ btnEl.disabled = true;
+ fetch('/api/library/duplicates/ignore', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ paths })
+ })
+ .then(r => r.json())
+ .then(data => {
+ if (data.success) {
+ group.remove();
+ refreshDupStatusText();
+ } else {
+ btnEl.disabled = false;
+ }
+ })
+ .catch(() => { btnEl.disabled = false; });
+ }
+
+ function toggleIgnoredDuplicates() {
+ const wrap = document.getElementById('dup-ignored-wrap');
+ if (wrap.style.display !== 'none') {
+ wrap.style.display = 'none';
+ return;
+ }
+ wrap.style.display = 'block';
+ loadIgnoredDuplicates();
+ }
+
+ function loadIgnoredDuplicates() {
+ const list = document.getElementById('dup-ignored-list');
+ list.innerHTML = '