Add Unsorted folder auto-sort with keyword matching

Scans an Unsorted folder at the library root and proposes destinations
for new courses by matching keywords against the existing category
tree, suggesting new subfolders when nothing matches closely, and
flagging items for manual review when nothing matches at all. Review
and apply happen on a dedicated /unsorted page linked from Settings;
nothing moves until the user confirms.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

offlineu_core.py

@@ -12,6 +12,7 @@ import re
 import sys
 import argparse
 import io
+import shutil
 import subprocess
 import threading
 import time
@@ -22,7 +23,7 @@ import urllib.error
 from pathlib import Path
 from datetime import datetime, timedelta
 from dataclasses import dataclass, asdict
-from typing import List, Dict, Optional, Any, Tuple, Iterator
+from typing import List, Dict, Optional, Any, Tuple, Iterator, Set
 from flask import Flask, render_template, request, jsonify, send_file, redirect, url_for
 
 app = Flask(__name__)
@@ -1183,6 +1184,190 @@ def get_all_course_dirs() -> List[Path]:
     return cache_get_or_compute('course_dirs', lambda: list(iter_all_courses(get_library_root())))
 
 
+# ---- Unsorted intake: propose where a newly-dropped course belongs ----
+#
+# Workflow: new/incoming courses get dropped into a folder named Unsorted
+# directly under the library root, sitting alongside the real category
+# tree (e.g. IT/, IT/AWS/, Business/). Scanning proposes a destination for
+# each item by keyword overlap against that *existing* tree - read fresh
+# from disk on every scan, never hardcoded - so it adapts automatically to
+# whatever categories actually exist. Nothing gets moved until the moves
+# are explicitly applied (see /api/unsorted/apply).
+UNSORTED_FOLDER_NAME = 'Unsorted'
+
+_TOKEN_SPLIT_RE = re.compile(r'[^a-z0-9]+')
+_TOKEN_STOPWORDS = {
+    'a', 'an', 'and', 'for', 'from', 'in', 'is', 'of', 'on', 'or', 'the', 'to', 'with',
+    'course', 'courses', 'training', 'class', 'complete', 'full', 'guide', 'tutorial',
+    'part', 'vol', 'volume', 'edition',
+}
+
+
+def _tokenize(name: str) -> Set[str]:
+    """
+    Break a folder/course name into lowercase keyword tokens for matching.
+    Splits camelCase/PascalCase boundaries before the general split, since
+    course names are commonly dot- or dash-joined with no spaces at all
+    (e.g. "AWSCertifiedSolutionsArchitect" as much as "AWS.Certified...").
+    Keeps short tokens (unlike most tokenizers) because real category
+    names here are often exactly that short - "IT", "PM" - and drops pure
+    numbers (section/edition numbers) and common filler words instead,
+    which is what actually carries no signal in this context.
+    """
+    return set(_tokenize_ordered(name))
+
+
+def _tokenize_ordered(name: str) -> List[str]:
+    """Same tokens as _tokenize, but as a de-duplicated list in the order
+    they first appear - used when a token subset needs to read back as a
+    sensible phrase (a suggested new folder name) rather than just being
+    compared as a set."""
+    spaced = re.sub(r'(?<=[a-z0-9])(?=[A-Z])', ' ', name)
+    raw = _TOKEN_SPLIT_RE.split(spaced.lower())
+    seen: Set[str] = set()
+    ordered = []
+    for t in raw:
+        if len(t) >= 2 and not t.isdigit() and t not in _TOKEN_STOPWORDS and t not in seen:
+            seen.add(t)
+            ordered.append(t)
+    return ordered
+
+
+def _build_category_index() -> List[Dict[str, Any]]:
+    """
+    Every category/subcategory folder currently in the library (e.g. IT,
+    IT/AWS) as a possible destination, each with keyword tokens from its
+    own path components plus the titles of any courses already filed
+    directly under it - that second part lets a folder match on subjects
+    its name alone doesn't mention (an "IT/AWS" folder full of courses
+    titled "...Solutions Architect..." now also matches on "solutions" and
+    "architect"), without hardcoding any subject list. Stops recursing
+    into a folder once it reads as a course itself (_looks_like_course,
+    the same rule the Library browser uses), so individual courses never
+    show up as if they were categories to file things under. The Unsorted
+    folder itself is excluded - it's the source, never a valid destination.
+    """
+    library_root = Path(get_library_root())
+    try:
+        unsorted_root = (library_root / UNSORTED_FOLDER_NAME).resolve()
+    except OSError:
+        unsorted_root = None
+    categories: List[Dict[str, Any]] = []
+
+    def walk(directory: Path, path_parts: List[str]):
+        try:
+            entries = sorted(
+                (p for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')),
+                key=lambda p: p.name.lower()
+            )
+        except (PermissionError, OSError):
+            return
+        for entry in entries:
+            try:
+                if unsorted_root is not None and entry.resolve() == unsorted_root:
+                    continue
+            except OSError:
+                pass
+            if _looks_like_course(entry):
+                continue
+            new_parts = path_parts + [entry.name]
+            tokens: Set[str] = set()
+            for part in new_parts:
+                tokens |= _tokenize(part)
+            try:
+                for child in entry.iterdir():
+                    if child.is_dir() and not child.name.startswith('.') and _looks_like_course(child):
+                        tokens |= _tokenize(child.name)
+            except (PermissionError, OSError):
+                pass
+            categories.append({
+                'path': str(entry),
+                'relative': '/'.join(new_parts),
+                'depth': len(new_parts),
+                'tokens': tokens,
+            })
+            walk(entry, new_parts)
+
+    walk(library_root, [])
+    return categories
+
+
+def _list_unsorted_items() -> List[Path]:
+    """Direct children of the Unsorted folder - each one is a course
+    waiting to be filed into the real category tree."""
+    unsorted_root = Path(get_library_root()) / UNSORTED_FOLDER_NAME
+    if not unsorted_root.is_dir():
+        return []
+    return sorted(
+        (p for p in unsorted_root.iterdir() if p.is_dir() and not p.name.startswith('.')),
+        key=lambda p: p.name.lower()
+    )
+
+
+def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Best-effort destination for one Unsorted item, by keyword overlap
+    against the existing category tree. Three outcomes:
+      - 'existing': a real folder already matches confidently - move there.
+      - 'new_folder': a broader category matches, but nothing that specific
+        already exists under it - propose creating a new subfolder there,
+        named from whichever of the item's own keywords the category
+        didn't already cover.
+      - 'manual': nothing matches with any confidence - needs a human.
+    "Confident" here means either a fairly specific existing folder (2+
+    levels deep, e.g. IT/AWS) matched on even one shared keyword, or a
+    shallower folder matched on 2+ - a single shared word with a bare
+    top-level category ("IT") isn't enough to drop something straight
+    into it; that's exactly the case a new subfolder proposal is for.
+    """
+    item_tokens_ordered = _tokenize_ordered(item_name)
+    item_tokens = set(item_tokens_ordered)
+    if not item_tokens:
+        return {'type': 'manual', 'destination': None,
+                'reason': "Couldn't extract any usable keywords from this name."}
+
+    scored = []
+    for cat in categories:
+        overlap = item_tokens & cat['tokens']
+        if overlap:
+            scored.append((len(overlap), cat['depth'], cat, overlap))
+
+    if not scored:
+        return {'type': 'manual', 'destination': None,
+                'reason': "No existing folder shares any keywords with this name."}
+
+    scored.sort(key=lambda s: (s[0], s[1]), reverse=True)
+    best_count, best_depth, best_cat, best_overlap = scored[0]
+    overlap_words = ', '.join(w for w in item_tokens_ordered if w in best_overlap)
+
+    if best_depth >= 2 or best_count >= 2:
+        return {
+            'type': 'existing',
+            'destination': best_cat['relative'],
+            'reason': f'Matches existing folder "{best_cat["relative"]}" on: {overlap_words}',
+        }
+
+    leftover = item_tokens - best_cat['tokens']
+    if leftover:
+        # preserve the item's own word order rather than the set's
+        # arbitrary one, so "Docker Compose Basics" suggests "Compose
+        # Basics" (dropping the already-covered "docker"), not a
+        # scrambled "Basics Compose"
+        leftover_ordered = [w for w in item_tokens_ordered if w in leftover]
+        new_name = ' '.join(w.upper() if len(w) <= 3 else w.capitalize() for w in leftover_ordered)
+        return {
+            'type': 'new_folder',
+            'destination': f"{best_cat['relative']}/{new_name}",
+            'reason': f'"{best_cat["relative"]}" matches on {overlap_words}, but no existing subfolder covers this - suggesting a new one.',
+        }
+
+    return {
+        'type': 'existing',
+        'destination': best_cat['relative'],
+        'reason': f'Matches existing folder "{best_cat["relative"]}" on: {overlap_words}',
+    }
+
+
 def _probe_media_duration_seconds(file_path: Path) -> Optional[float]:
     """Read a single media file's duration via ffprobe. None if ffprobe is
     missing, the file isn't readable, or the output can't be parsed."""
@@ -2486,6 +2671,89 @@ def refresh_library_api():
     return jsonify({'success': True})
 
 
+@app.route('/api/unsorted/scan')
+def scan_unsorted_api():
+    """Propose a destination for every item currently sitting in the Unsorted folder. Read-only - nothing moves until /api/unsorted/apply is called."""
+    library_root = get_library_root()
+    unsorted_root = os.path.join(library_root, UNSORTED_FOLDER_NAME)
+    if not os.path.isdir(unsorted_root):
+        return jsonify({'items': [], 'unsorted_exists': False, 'unsorted_path': unsorted_root})
+
+    items = _list_unsorted_items()
+    categories = _build_category_index()
+    results = []
+    for item in items:
+        proposal = _propose_destination(item.name, categories)
+        results.append({'name': item.name, 'path': str(item), **proposal})
+
+    return jsonify({
+        'items': results,
+        'unsorted_exists': True,
+        'unsorted_path': unsorted_root,
+        'category_count': len(categories),
+    })
+
+
+@app.route('/api/unsorted/apply', methods=['POST'])
+def apply_unsorted_api():
+    """
+    Move a batch of Unsorted items to their (possibly user-edited)
+    destinations, creating new subfolders as needed. Every source must
+    actually be inside the Unsorted folder and every destination must
+    stay inside the library root and outside Unsorted itself - guards
+    against a hand-edited destination path escaping the library, the same
+    zip-slip-style check used for backup restore.
+    """
+    data = request.json or {}
+    moves = data.get('moves')
+    if not isinstance(moves, list) or not moves:
+        return jsonify({'success': False, 'error': 'No moves provided'}), 400
+
+    library_root = os.path.abspath(get_library_root())
+    unsorted_root = os.path.abspath(os.path.join(library_root, UNSORTED_FOLDER_NAME))
+
+    moved = []
+    errors = []
+    for move in moves:
+        source = (move or {}).get('source', '')
+        destination_relative = ((move or {}).get('destination') or '').strip().strip('/')
+
+        if not source or not destination_relative:
+            errors.append({'source': source, 'error': 'Missing source or destination'})
+            continue
+
+        source_abs = os.path.abspath(source)
+        if not (source_abs == unsorted_root or source_abs.startswith(unsorted_root + os.sep)):
+            errors.append({'source': source, 'error': 'Source is not inside the Unsorted folder'})
+            continue
+        if not os.path.isdir(source_abs):
+            errors.append({'source': source, 'error': 'No longer exists - already moved?'})
+            continue
+
+        dest_dir = os.path.abspath(os.path.join(library_root, destination_relative))
+        if not (dest_dir == library_root or dest_dir.startswith(library_root + os.sep)):
+            errors.append({'source': source, 'error': 'Destination is outside the library'})
+            continue
+        if dest_dir == unsorted_root or dest_dir.startswith(unsorted_root + os.sep):
+            errors.append({'source': source, 'error': 'Destination cannot be inside Unsorted'})
+            continue
+
+        final_path = os.path.join(dest_dir, os.path.basename(source_abs))
+        if os.path.exists(final_path):
+            errors.append({'source': source, 'error': f'"{os.path.basename(source_abs)}" already exists at that destination'})
+            continue
+
+        try:
+            os.makedirs(dest_dir, exist_ok=True)
+            shutil.move(source_abs, final_path)
+            moved.append({'source': source, 'destination': final_path})
+        except OSError as e:
+            errors.append({'source': source, 'error': str(e)})
+
+    invalidate_cache()
+    return jsonify({'success': True, 'moved': len(moved), 'errors': errors})
+
+
 # Single-process personal app, so a plain dict + background thread is
 # enough state for the duration prewarm - no job queue needed. Guarded by
 # 'running' so a second click while one's in flight just reports progress
@@ -2981,6 +3249,12 @@ def notes_hub():
     return render_template('notes_hub.html', notes=get_all_notes(), active_tab='notes')
 
 
+@app.route('/unsorted')
+def unsorted_page():
+    """Review page for filing new courses out of the Unsorted folder - see /api/unsorted/scan for the actual proposals."""
+    return render_template('unsorted.html', active_tab=None)
+
+
 @app.route('/library/search-transcripts')
 def search_transcripts_api():
     """Search inside lesson subtitle files across the library."""