rmsitz/offlineu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tighten Sort Unsorted matching; consolidate file management tools

Browse Source 
Fix Sort Unsorted picking a wrong destination on words that are common
clutter across many categories (a creator's name, generic marketing
filler) by ranking matches on a folder's own deliberate name over
incidental sibling-course words, and downweighting borrowed words that
recur across many categories. Add a folder-picker/create-new/rename UI
to override any proposal.

Turn /unsorted into a general File Management page: move Refresh
Library, Bulk Rename, and Manage Library (hide/rename) here from
Settings, add a Move action to Manage Library using the same picker,
and add Duplicate Courses and Clean Up Stale References tools. Add a
footer link to the page from the dashboard.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
rmsitz
2026-08-24 10:29:55 -04:00
co-authored by Claude Sonnet 5
parent 4eda493e00
commit 9ff8c70311
5 changed files with 1325 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
offlineu_core.py
+373 -37
View File
@@ -1009,10 +1009,11 @@ def rebase_library_path(old_abs: str, new_abs: str) -> None:
"""
After a directory in the library is renamed/moved on disk, rewrite any
stored absolute paths that pointed inside it (hidden_paths.json,
recent_views.json's course_path) so curation and Recently Viewed don't
silently go stale. Lesson-level progress needs no rebasing - it lives
inside the directory itself, keyed by paths relative to it, so it moves
with the rename automatically.
next_up.json, recent_views.json's course_path) so curation, the Next
Up queue, and Recently Viewed don't silently go stale. Lesson-level
progress needs no rebasing - it lives inside the directory itself,
keyed by paths relative to it, so it moves with the rename
automatically.
"""
hidden = get_hidden_paths()
rebased_hidden = [_rebase_prefix(p, old_abs, new_abs) for p in hidden]
@@ -1021,6 +1022,11 @@ def rebase_library_path(old_abs: str, new_abs: str) -> None:
with open(HIDDEN_PATHS_FILE, 'w') as f:
json.dump(sorted(set(rebased_hidden)), f, indent=2)
queued = get_next_up_paths()
rebased_queue = [_rebase_prefix(p, old_abs, new_abs) for p in queued]
if rebased_queue != queued:
_save_next_up_paths(rebased_queue)
views = get_recent_views()
changed = False
for entry in views:
@@ -1200,6 +1206,29 @@ _TOKEN_STOPWORDS = {
'a', 'an', 'and', 'for', 'from', 'in', 'is', 'of', 'on', 'or', 'the', 'to', 'with',
'course', 'courses', 'training', 'class', 'complete', 'full', 'guide', 'tutorial',
'part', 'vol', 'volume', 'edition',
# Skill-level/marketing filler - describes the packaging, not the
# subject, same spirit as 'complete'/'guide'/'tutorial' above.
'advanced', 'intermediate', 'beginner', 'beginners', 'basics', 'basic',
'essentials', 'fundamentals', 'mastering', 'master', 'crash', 'level', 'levels',
# E-learning platforms/marketplaces - whichever site a course was bought
# or ripped from says nothing about its subject, and it shows up in
# almost every course name from that source.
'udemy', 'pluralsight', 'lynda', 'linkedin', 'linkedinlearning', 'coursera',
'skillshare', 'packt', 'packtpub', 'oreilly', 'edx', 'educative', 'egghead',
'frontendmasters', 'codecademy', 'udacity', 'skillsoft', 'cbtnuggets', 'cbt',
'itprotv', 'cybrary', 'tutsplus', 'masterclass', 'treehouse', 'datacamp',
# Scene/distribution-group tags and rip metadata that piggyback on
# downloaded course names - never a subject, always noise.
'bookware', 'tutsnode', 'ftu', 'ftuforum', 'freecoursesonline', 'freetutorials',
'coursedrive', 'downtem', 'glodls', 'garr', 'ridware', 'illiterate', 'zh',
'nfo', 'readnfo', 'repack', 'reup', 'reupload', 'repost', 'release', 'released',
'rip', 'x264', 'x265', 'www', 'com', 'net', 'org', 'info',
# "Freshness" marketing words - describe the release, not the subject.
'update', 'updated', 'new', 'newest', 'latest', 'final', 'retail',
# Calendar words - dates say nothing about what a course is about.
'january', 'february', 'march', 'april', 'june', 'july', 'august', 'september',
'october', 'november', 'december', 'jan', 'feb', 'mar', 'apr', 'jun', 'jul',
'aug', 'sep', 'sept', 'oct', 'nov', 'dec',
}
@@ -1236,16 +1265,26 @@ def _tokenize_ordered(name: str) -> List[str]:
def _build_category_index() -> List[Dict[str, Any]]:
"""
Every category/subcategory folder currently in the library (e.g. IT,
IT/AWS) as a possible destination, each with keyword tokens from its
own path components plus the titles of any courses already filed
directly under it - that second part lets a folder match on subjects
its name alone doesn't mention (an "IT/AWS" folder full of courses
titled "...Solutions Architect..." now also matches on "solutions" and
"architect"), without hardcoding any subject list. Stops recursing
into a folder once it reads as a course itself (_looks_like_course,
the same rule the Library browser uses), so individual courses never
show up as if they were categories to file things under. The Unsorted
folder itself is excluded - it's the source, never a valid destination.
IT/AWS) as a possible destination, each with keyword tokens split into
two tiers of trust:
- 'path_tokens': from the folder's own path components - a deliberate
name the user chose, so a match here is always taken at face value.
- 'bonus_tokens': from the titles of courses already filed directly
under it - lets a folder match on subjects its name alone doesn't
mention (an "IT/AWS" folder full of courses titled "...Solutions
Architect..." now also matches on "solutions" and "architect"),
without hardcoding any subject list. Since these come from
whatever the courses happen to be named, they're incidental rather
than deliberate - _propose_destination downweights them by how
many *different* categories share the same bonus word, so a word
that turns out to be common clutter (recurring across many
categories' course titles) can't outweigh a specific match
elsewhere just by sharing more of it.
Stops recursing into a folder once it reads as a course itself
(_looks_like_course, the same rule the Library browser uses), so
individual courses never show up as if they were categories to file
things under. The Unsorted folder itself is excluded - it's the
source, never a valid destination.
"""
library_root = Path(get_library_root())
try:
@@ -1271,20 +1310,23 @@ def _build_category_index() -> List[Dict[str, Any]]:
if _looks_like_course(entry):
continue
new_parts = path_parts + [entry.name]
tokens: Set[str] = set()
path_tokens: Set[str] = set()
for part in new_parts:
tokens |= _tokenize(part)
path_tokens |= _tokenize(part)
bonus_tokens: Set[str] = set()
try:
for child in entry.iterdir():
if child.is_dir() and not child.name.startswith('.') and _looks_like_course(child):
tokens |= _tokenize(child.name)
bonus_tokens |= _tokenize(child.name)
except (PermissionError, OSError):
pass
bonus_tokens -= path_tokens
categories.append({
'path': str(entry),
'relative': '/'.join(new_parts),
'depth': len(new_parts),
'tokens': tokens,
'path_tokens': path_tokens,
'bonus_tokens': bonus_tokens,
})
walk(entry, new_parts)
@@ -1292,6 +1334,20 @@ def _build_category_index() -> List[Dict[str, Any]]:
return categories
def _bonus_token_frequency(categories: List[Dict[str, Any]]) -> Dict[str, int]:
"""How many distinct categories each bonus token shows up in, across
the whole library - used to tell a word that's actually specific to one
place ("cloudformation") from one that's just common clutter recurring
in course titles everywhere ("advanced", a creator's name, etc.), so
the latter can be downweighted instead of needing to be individually
named as a stopword ahead of time."""
freq: Dict[str, int] = {}
for cat in categories:
for token in cat['bonus_tokens']:
freq[token] = freq.get(token, 0) + 1
return freq
def _list_unsorted_items() -> List[Path]:
"""Direct children of the Unsorted folder - each one is a course
waiting to be filed into the real category tree."""
@@ -1304,7 +1360,8 @@ def _list_unsorted_items() -> List[Path]:
)
def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Dict[str, Any]:
def _propose_destination(item_name: str, categories: List[Dict[str, Any]],
bonus_df: Dict[str, int]) -> Dict[str, Any]:
"""
Best-effort destination for one Unsorted item, by keyword overlap
against the existing category tree. Three outcomes:
@@ -1314,11 +1371,26 @@ def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Di
named from whichever of the item's own keywords the category
didn't already cover.
- 'manual': nothing matches with any confidence - needs a human.
"Confident" here means either a fairly specific existing folder (2+
levels deep, e.g. IT/AWS) matched on even one shared keyword, or a
shallower folder matched on 2+ - a single shared word with a bare
top-level category ("IT") isn't enough to drop something straight
into it; that's exactly the case a new subfolder proposal is for.
Categories are ranked by evidence quality, not raw shared-word count -
a match on a *path* token (the folder's own deliberate name, e.g. the
"AWS" in IT/AWS) always outranks any number of *bonus* tokens (words
only borrowed from a sibling course's title - see
_build_category_index), since a folder's own name is deliberate and a
sibling course's title is incidental. Among bonus-only matches, the
single least-diluted word wins rather than the total - a bonus word
shared by only one category counts fully, one recurring across N
categories counts for 1/N - so several incidental, recurring words (a
prolific creator's name, a repeated marketing phrase, etc. showing up
across many categories' course titles) can't outvote one genuinely
specific word just by there being more of them, and don't need to be
individually named as stopwords ahead of time to be discounted.
"Confident" then means the best match is either a path-token hit or an
essentially-unique bonus word (not diluted across other categories),
AND it's either at a fairly specific existing folder (2+ levels deep,
e.g. IT/AWS) or corroborated by a second shared word - one weak,
common word alone isn't enough to drop something straight into a bare
top-level category ("IT"); that's exactly the case a new subfolder
proposal is for.
"""
item_tokens_ordered = _tokenize_ordered(item_name)
item_tokens = set(item_tokens_ordered)
@@ -1328,26 +1400,36 @@ def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Di
scored = []
for cat in categories:
overlap = item_tokens & cat['tokens']
if overlap:
scored.append((len(overlap), cat['depth'], cat, overlap))
overlap = item_tokens & (cat['path_tokens'] | cat['bonus_tokens'])
if not overlap:
continue
path_hits = overlap & cat['path_tokens']
bonus_overlap = overlap - cat['path_tokens']
bonus_best = max((1.0 / bonus_df.get(t, 1) for t in bonus_overlap), default=0.0)
scored.append({
'cat': cat, 'overlap': overlap,
'path_hits': len(path_hits), 'bonus_best': bonus_best, 'count': len(overlap),
})
if not scored:
return {'type': 'manual', 'destination': None,
'reason': "No existing folder shares any keywords with this name."}
scored.sort(key=lambda s: (s[0], s[1]), reverse=True)
best_count, best_depth, best_cat, best_overlap = scored[0]
overlap_words = ', '.join(w for w in item_tokens_ordered if w in best_overlap)
scored.sort(key=lambda s: (s['path_hits'], s['bonus_best'], s['count'], s['cat']['depth']), reverse=True)
best = scored[0]
best_cat = best['cat']
overlap_words = ', '.join(w for w in item_tokens_ordered if w in best['overlap'])
best_all_tokens = best_cat['path_tokens'] | best_cat['bonus_tokens']
if best_depth >= 2 or best_count >= 2:
is_strong_match = best['path_hits'] >= 1 or best['bonus_best'] >= 1.0
if is_strong_match and (best_cat['depth'] >= 2 or best['count'] >= 2):
return {
'type': 'existing',
'destination': best_cat['relative'],
'reason': f'Matches existing folder "{best_cat["relative"]}" on: {overlap_words}',
}
leftover = item_tokens - best_cat['tokens']
leftover = item_tokens - best_all_tokens
if leftover:
# preserve the item's own word order rather than the set's
# arbitrary one, so "Docker Compose Basics" suggests "Compose
@@ -1368,6 +1450,110 @@ def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Di
}
def _find_duplicate_courses(min_similarity: float = 0.6) -> List[Dict[str, Any]]:
"""
Group courses across the whole library (Unsorted included, so a
just-dropped item that duplicates something already filed gets caught
too) whose names look like the same thing filed twice - a re-download
that landed in a different category, or a course that's both sorted
and still sitting in Unsorted. Reuses Sort Unsorted's own
tokenizer/stopword list, so platform names, release-group tags, and
dates are already ignored - "AWS Certified Solutions Architect" and
"AWS.Certified.Solutions.Architect.2024.BOOKWARE-XYZ" tokenize down to
the same core words despite looking nothing alike as strings.
Pairwise Jaccard similarity (shared tokens / all tokens) at or above
min_similarity gets union-found into groups, so if A matches B and B
matches C, all three land in one group instead of two overlapping
pairs. Nothing here touches disk - it's read-only, for the user to
review and hide/rename/move manually.
"""
entries = []
for course in get_all_course_dirs():
tokens = _tokenize(course.name)
if tokens:
entries.append({'path': str(course), 'name': course.name, 'tokens': tokens})
n = len(entries)
parent = list(range(n))
def find(i: int) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
def union(i: int, j: int) -> None:
ri, rj = find(i), find(j)
if ri != rj:
parent[ri] = rj
pair_similarity: Dict[Tuple[int, int], float] = {}
for i in range(n):
for j in range(i + 1, n):
a, b = entries[i]['tokens'], entries[j]['tokens']
union_tokens = a | b
if not union_tokens:
continue
similarity = len(a & b) / len(union_tokens)
if similarity >= min_similarity:
pair_similarity[(i, j)] = similarity
union(i, j)
clusters: Dict[int, List[int]] = {}
for i in range(n):
clusters.setdefault(find(i), []).append(i)
groups = []
for members in clusters.values():
if len(members) < 2:
continue
best_similarity = max(
(sim for (i, j), sim in pair_similarity.items() if i in members and j in members),
default=0.0
)
groups.append({
'courses': sorted(
({'path': entries[i]['path'], 'name': entries[i]['name']} for i in members),
key=lambda c: c['name'].lower()
),
'similarity': round(best_similarity, 2),
})
groups.sort(key=lambda g: g['similarity'], reverse=True)
return groups
def _find_stale_references() -> Dict[str, List[Dict[str, str]]]:
"""
Entries in hidden_paths.json / next_up.json / recent_views.json that
point at a path no longer on disk - typically because a course was
renamed, moved, or deleted directly on the NAS rather than through the
app (a rename/move made through the app already rebases these, see
rebase_library_path). Read-only; nothing is removed until
/api/library/stale-references/clean is called with the exact items
reviewed here.
"""
stale: Dict[str, List[Dict[str, str]]] = {'hidden': [], 'next_up': [], 'recent_views': []}
for path in get_hidden_paths():
if not os.path.isdir(path):
stale['hidden'].append({'path': path, 'name': os.path.basename(path.rstrip(os.sep)) or path})
for path in get_next_up_paths():
if not os.path.isdir(path):
stale['next_up'].append({'path': path, 'name': os.path.basename(path.rstrip(os.sep)) or path})
seen: Set[str] = set()
for entry in get_recent_views():
path = entry.get('course_path', '')
if path and path not in seen and not os.path.isdir(path):
seen.add(path)
name = entry.get('course_name') or os.path.basename(path.rstrip(os.sep)) or path
stale['recent_views'].append({'path': path, 'name': name})
return stale
def _probe_media_duration_seconds(file_path: Path) -> Optional[float]:
"""Read a single media file's duration via ffprobe. None if ffprobe is
missing, the file isn't readable, or the output can't be parsed."""
@@ -2045,6 +2231,20 @@ def get_recent_views() -> List[Dict[str, Any]]:
return []
def remove_recent_views_for_path(path: str) -> List[Dict[str, Any]]:
"""Drop every Recently Viewed entry for one course path - there can be
several, one per lesson. Used to clear a stale reference left behind
when a course was deleted/moved outside the app."""
entries = get_recent_views()
normalized = os.path.abspath(path)
remaining = [e for e in entries if e.get('course_path') != normalized]
if remaining != entries:
os.makedirs(DATA_DIR, exist_ok=True)
with open(RECENT_VIEWS_FILE, 'w') as f:
json.dump(remaining, f, indent=2)
return remaining
def _read_lesson_progress_entry(course_path: str, lesson_path: str) -> Dict[str, Any]:
"""
Read a single lesson's progress entry directly from its course's own
@@ -2681,16 +2881,20 @@ def scan_unsorted_api():
items = _list_unsorted_items()
categories = _build_category_index()
bonus_df = _bonus_token_frequency(categories)
results = []
for item in items:
proposal = _propose_destination(item.name, categories)
proposal = _propose_destination(item.name, categories, bonus_df)
results.append({'name': item.name, 'path': str(item), **proposal})
category_paths = sorted((c['relative'] for c in categories), key=str.lower)
return jsonify({
'items': results,
'unsorted_exists': True,
'unsorted_path': unsorted_root,
'category_count': len(categories),
'categories': category_paths,
})
@@ -2698,8 +2902,10 @@ def scan_unsorted_api():
def apply_unsorted_api():
"""
Move a batch of Unsorted items to their (possibly user-edited)
destinations, creating new subfolders as needed. Every source must
actually be inside the Unsorted folder and every destination must
destinations, creating new subfolders as needed, and optionally renaming
each item's folder in the same step (move + rename is one filesystem
operation either way, so there's no reason to make it two). Every source
must actually be inside the Unsorted folder and every destination must
stay inside the library root and outside Unsorted itself - guards
against a hand-edited destination path escaping the library, the same
zip-slip-style check used for backup restore.
@@ -2738,9 +2944,16 @@ def apply_unsorted_api():
errors.append({'source': source, 'error': 'Destination cannot be inside Unsorted'})
continue
final_path = os.path.join(dest_dir, os.path.basename(source_abs))
final_name = ((move or {}).get('name') or '').strip()
if not final_name:
final_name = os.path.basename(source_abs)
if os.sep in final_name or (os.altsep and os.altsep in final_name) or final_name in ('.', '..'):
errors.append({'source': source, 'error': f'"{final_name}" is not a valid folder name'})
continue
final_path = os.path.join(dest_dir, final_name)
if os.path.exists(final_path):
errors.append({'source': source, 'error': f'"{os.path.basename(source_abs)}" already exists at that destination'})
errors.append({'source': source, 'error': f'"{final_name}" already exists at that destination'})
continue
try:
@@ -2754,6 +2967,129 @@ def apply_unsorted_api():
return jsonify({'success': True, 'moved': len(moved), 'errors': errors})
@app.route('/api/library/categories')
def library_categories_api():
"""Every category/subcategory folder in the library, for destination
pickers - Sort Unsorted's own picker gets this from /api/unsorted/scan
already; this is the same list for callers (like Manage Library's move
action) that need it without depending on an Unsorted-folder scan."""
categories = _build_category_index()
return jsonify({'categories': sorted((c['relative'] for c in categories), key=str.lower)})
@app.route('/api/library/move', methods=['POST'])
def move_library_item_api():
"""
Move (and optionally rename) any course or folder already in the
library to a different location - the general-purpose version of
/api/unsorted/apply, for reorganizing something already filed rather
than just-arrived. Same source/destination-inside-library-root safety
checks, plus one apply_unsorted_api doesn't need: the destination
can't be inside the source's own subtree, since here the source can
itself be a category folder with children, not just a leaf course.
"""
data = request.json or {}
source = (data.get('source') or '').strip()
destination_relative = (data.get('destination') or '').strip().strip('/')
final_name = (data.get('name') or '').strip()
if not source or not destination_relative:
return jsonify({'success': False, 'error': 'Missing source or destination'}), 400
library_root = os.path.abspath(get_library_root())
source_abs = os.path.abspath(source)
if not (source_abs == library_root or source_abs.startswith(library_root + os.sep)):
return jsonify({'success': False, 'error': 'Source is outside the library'}), 403
if source_abs == library_root:
return jsonify({'success': False, 'error': 'Cannot move the library root itself'}), 400
if not os.path.isdir(source_abs):
return jsonify({'success': False, 'error': 'Source no longer exists'}), 404
dest_dir = os.path.abspath(os.path.join(library_root, destination_relative))
if not (dest_dir == library_root or dest_dir.startswith(library_root + os.sep)):
return jsonify({'success': False, 'error': 'Destination is outside the library'}), 403
if dest_dir == source_abs or dest_dir.startswith(source_abs + os.sep):
return jsonify({'success': False, 'error': 'Cannot move a folder into itself'}), 400
if not final_name:
final_name = os.path.basename(source_abs)
if os.sep in final_name or (os.altsep and os.altsep in final_name) or final_name in ('.', '..'):
return jsonify({'success': False, 'error': f'"{final_name}" is not a valid folder name'}), 400
final_path = os.path.join(dest_dir, final_name)
if final_path == source_abs:
return jsonify({'success': False, 'error': 'Already there'}), 400
if os.path.exists(final_path):
return jsonify({'success': False, 'error': f'"{final_name}" already exists at that destination'}), 409
try:
os.makedirs(dest_dir, exist_ok=True)
shutil.move(source_abs, final_path)
except OSError as e:
return jsonify({'success': False, 'error': str(e)}), 500
invalidate_cache()
rebase_library_path(source_abs, final_path)
global current_course
active_course_reset = False
if current_course is not None:
course_abs = os.path.abspath(current_course.path)
if course_abs == source_abs or course_abs.startswith(source_abs + os.sep):
current_course = None
active_course_reset = True
return jsonify({'success': True, 'new_path': final_path, 'active_course_reset': active_course_reset})
@app.route('/api/library/duplicates')
def duplicate_courses_api():
"""Scan for courses that look like copies of each other by name (see
_find_duplicate_courses). Not run automatically like Sort Unsorted's
scan - it's O(n^2) over every course in the library, so it's a manual
trigger instead."""
groups = _find_duplicate_courses()
return jsonify({'groups': groups, 'course_count': len(get_all_course_dirs())})
@app.route('/api/library/stale-references')
def stale_references_api():
"""Entries in hidden_paths/next_up/recent_views pointing at paths no
longer on disk (see _find_stale_references). Read-only - nothing is
removed until /api/library/stale-references/clean is called."""
return jsonify(_find_stale_references())
@app.route('/api/library/stale-references/clean', methods=['POST'])
def clean_stale_references_api():
"""Remove specific stale entries the client reviewed via the scan
above. Re-checks each path still doesn't exist before removing it, in
case it became valid again between scan and apply."""
data = request.json or {}
items = data.get('items') or []
if not isinstance(items, list) or not items:
return jsonify({'success': False, 'error': 'No items provided'}), 400
removed = 0
for item in items:
category = (item or {}).get('category')
path = (item or {}).get('path')
if not path or os.path.isdir(path):
continue
if category == 'hidden':
set_path_hidden(path, False)
removed += 1
elif category == 'next_up':
set_path_queued(path, False)
removed += 1
elif category == 'recent_views':
remove_recent_views_for_path(path)
removed += 1
return jsonify({'success': True, 'removed': removed})
# Single-process personal app, so a plain dict + background thread is
# enough state for the duration prewarm - no job queue needed. Guarded by
# 'running' so a second click while one's in flight just reports progress