diff --git a/README.md b/README.md index 7bbb246..3af3711 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,16 @@ silently stay blank instead of erroring. - Hide courses/folders from the browser without touching anything on disk; bulk-select to hide or queue several at once - Bulk rename across course/folder names (find & replace) +- **Sort Unsorted** ([/unsorted](templates/unsorted.html), linked from + Settings): drop new/incoming courses into an `Unsorted` folder at the + library root, then scan proposes a destination for each one by keyword + overlap against the *existing* category tree - read fresh from disk every + scan, so it adapts to whatever folders you actually have rather than any + hardcoded subject list. Three outcomes per item: a confident match to an + existing folder, a suggested new subfolder under a broader category match, + or "needs review" (unchecked by default) when nothing overlaps at all. + Every destination is editable before applying, and nothing on disk moves + until you review and hit Apply. **Dashboard** - Library-wide stats (courses, lessons completed, time watched, time @@ -185,6 +195,10 @@ MyCourse/ No metadata files needed — course/section/lesson names come straight from folder and file names. +An `Unsorted/` folder at the library root (alongside the real category +folders) is the drop point for new/incoming courses — see Sort Unsorted +above. + ## Supported file types | Type | Extensions | diff --git a/offlineu_core.py b/offlineu_core.py index e09b06c..822c49d 100644 --- a/offlineu_core.py +++ b/offlineu_core.py @@ -12,6 +12,7 @@ import re import sys import argparse import io +import shutil import subprocess import threading import time @@ -22,7 +23,7 @@ import urllib.error from pathlib import Path from datetime import datetime, timedelta from dataclasses import dataclass, asdict -from typing import List, Dict, Optional, Any, Tuple, Iterator +from typing import List, Dict, Optional, Any, Tuple, Iterator, Set from flask import Flask, render_template, request, jsonify, send_file, redirect, url_for app = Flask(__name__) @@ -1183,6 +1184,190 @@ def get_all_course_dirs() -> List[Path]: return cache_get_or_compute('course_dirs', lambda: list(iter_all_courses(get_library_root()))) +# ---- Unsorted intake: propose where a newly-dropped course belongs ---- +# +# Workflow: new/incoming courses get dropped into a folder named Unsorted +# directly under the library root, sitting alongside the real category +# tree (e.g. IT/, IT/AWS/, Business/). Scanning proposes a destination for +# each item by keyword overlap against that *existing* tree - read fresh +# from disk on every scan, never hardcoded - so it adapts automatically to +# whatever categories actually exist. Nothing gets moved until the moves +# are explicitly applied (see /api/unsorted/apply). +UNSORTED_FOLDER_NAME = 'Unsorted' + +_TOKEN_SPLIT_RE = re.compile(r'[^a-z0-9]+') +_TOKEN_STOPWORDS = { + 'a', 'an', 'and', 'for', 'from', 'in', 'is', 'of', 'on', 'or', 'the', 'to', 'with', + 'course', 'courses', 'training', 'class', 'complete', 'full', 'guide', 'tutorial', + 'part', 'vol', 'volume', 'edition', +} + + +def _tokenize(name: str) -> Set[str]: + """ + Break a folder/course name into lowercase keyword tokens for matching. + Splits camelCase/PascalCase boundaries before the general split, since + course names are commonly dot- or dash-joined with no spaces at all + (e.g. "AWSCertifiedSolutionsArchitect" as much as "AWS.Certified..."). + Keeps short tokens (unlike most tokenizers) because real category + names here are often exactly that short - "IT", "PM" - and drops pure + numbers (section/edition numbers) and common filler words instead, + which is what actually carries no signal in this context. + """ + return set(_tokenize_ordered(name)) + + +def _tokenize_ordered(name: str) -> List[str]: + """Same tokens as _tokenize, but as a de-duplicated list in the order + they first appear - used when a token subset needs to read back as a + sensible phrase (a suggested new folder name) rather than just being + compared as a set.""" + spaced = re.sub(r'(?<=[a-z0-9])(?=[A-Z])', ' ', name) + raw = _TOKEN_SPLIT_RE.split(spaced.lower()) + seen: Set[str] = set() + ordered = [] + for t in raw: + if len(t) >= 2 and not t.isdigit() and t not in _TOKEN_STOPWORDS and t not in seen: + seen.add(t) + ordered.append(t) + return ordered + + +def _build_category_index() -> List[Dict[str, Any]]: + """ + Every category/subcategory folder currently in the library (e.g. IT, + IT/AWS) as a possible destination, each with keyword tokens from its + own path components plus the titles of any courses already filed + directly under it - that second part lets a folder match on subjects + its name alone doesn't mention (an "IT/AWS" folder full of courses + titled "...Solutions Architect..." now also matches on "solutions" and + "architect"), without hardcoding any subject list. Stops recursing + into a folder once it reads as a course itself (_looks_like_course, + the same rule the Library browser uses), so individual courses never + show up as if they were categories to file things under. The Unsorted + folder itself is excluded - it's the source, never a valid destination. + """ + library_root = Path(get_library_root()) + try: + unsorted_root = (library_root / UNSORTED_FOLDER_NAME).resolve() + except OSError: + unsorted_root = None + categories: List[Dict[str, Any]] = [] + + def walk(directory: Path, path_parts: List[str]): + try: + entries = sorted( + (p for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')), + key=lambda p: p.name.lower() + ) + except (PermissionError, OSError): + return + for entry in entries: + try: + if unsorted_root is not None and entry.resolve() == unsorted_root: + continue + except OSError: + pass + if _looks_like_course(entry): + continue + new_parts = path_parts + [entry.name] + tokens: Set[str] = set() + for part in new_parts: + tokens |= _tokenize(part) + try: + for child in entry.iterdir(): + if child.is_dir() and not child.name.startswith('.') and _looks_like_course(child): + tokens |= _tokenize(child.name) + except (PermissionError, OSError): + pass + categories.append({ + 'path': str(entry), + 'relative': '/'.join(new_parts), + 'depth': len(new_parts), + 'tokens': tokens, + }) + walk(entry, new_parts) + + walk(library_root, []) + return categories + + +def _list_unsorted_items() -> List[Path]: + """Direct children of the Unsorted folder - each one is a course + waiting to be filed into the real category tree.""" + unsorted_root = Path(get_library_root()) / UNSORTED_FOLDER_NAME + if not unsorted_root.is_dir(): + return [] + return sorted( + (p for p in unsorted_root.iterdir() if p.is_dir() and not p.name.startswith('.')), + key=lambda p: p.name.lower() + ) + + +def _propose_destination(item_name: str, categories: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Best-effort destination for one Unsorted item, by keyword overlap + against the existing category tree. Three outcomes: + - 'existing': a real folder already matches confidently - move there. + - 'new_folder': a broader category matches, but nothing that specific + already exists under it - propose creating a new subfolder there, + named from whichever of the item's own keywords the category + didn't already cover. + - 'manual': nothing matches with any confidence - needs a human. + "Confident" here means either a fairly specific existing folder (2+ + levels deep, e.g. IT/AWS) matched on even one shared keyword, or a + shallower folder matched on 2+ - a single shared word with a bare + top-level category ("IT") isn't enough to drop something straight + into it; that's exactly the case a new subfolder proposal is for. + """ + item_tokens_ordered = _tokenize_ordered(item_name) + item_tokens = set(item_tokens_ordered) + if not item_tokens: + return {'type': 'manual', 'destination': None, + 'reason': "Couldn't extract any usable keywords from this name."} + + scored = [] + for cat in categories: + overlap = item_tokens & cat['tokens'] + if overlap: + scored.append((len(overlap), cat['depth'], cat, overlap)) + + if not scored: + return {'type': 'manual', 'destination': None, + 'reason': "No existing folder shares any keywords with this name."} + + scored.sort(key=lambda s: (s[0], s[1]), reverse=True) + best_count, best_depth, best_cat, best_overlap = scored[0] + overlap_words = ', '.join(w for w in item_tokens_ordered if w in best_overlap) + + if best_depth >= 2 or best_count >= 2: + return { + 'type': 'existing', + 'destination': best_cat['relative'], + 'reason': f'Matches existing folder "{best_cat["relative"]}" on: {overlap_words}', + } + + leftover = item_tokens - best_cat['tokens'] + if leftover: + # preserve the item's own word order rather than the set's + # arbitrary one, so "Docker Compose Basics" suggests "Compose + # Basics" (dropping the already-covered "docker"), not a + # scrambled "Basics Compose" + leftover_ordered = [w for w in item_tokens_ordered if w in leftover] + new_name = ' '.join(w.upper() if len(w) <= 3 else w.capitalize() for w in leftover_ordered) + return { + 'type': 'new_folder', + 'destination': f"{best_cat['relative']}/{new_name}", + 'reason': f'"{best_cat["relative"]}" matches on {overlap_words}, but no existing subfolder covers this - suggesting a new one.', + } + + return { + 'type': 'existing', + 'destination': best_cat['relative'], + 'reason': f'Matches existing folder "{best_cat["relative"]}" on: {overlap_words}', + } + + def _probe_media_duration_seconds(file_path: Path) -> Optional[float]: """Read a single media file's duration via ffprobe. None if ffprobe is missing, the file isn't readable, or the output can't be parsed.""" @@ -2486,6 +2671,89 @@ def refresh_library_api(): return jsonify({'success': True}) +@app.route('/api/unsorted/scan') +def scan_unsorted_api(): + """Propose a destination for every item currently sitting in the Unsorted folder. Read-only - nothing moves until /api/unsorted/apply is called.""" + library_root = get_library_root() + unsorted_root = os.path.join(library_root, UNSORTED_FOLDER_NAME) + if not os.path.isdir(unsorted_root): + return jsonify({'items': [], 'unsorted_exists': False, 'unsorted_path': unsorted_root}) + + items = _list_unsorted_items() + categories = _build_category_index() + results = [] + for item in items: + proposal = _propose_destination(item.name, categories) + results.append({'name': item.name, 'path': str(item), **proposal}) + + return jsonify({ + 'items': results, + 'unsorted_exists': True, + 'unsorted_path': unsorted_root, + 'category_count': len(categories), + }) + + +@app.route('/api/unsorted/apply', methods=['POST']) +def apply_unsorted_api(): + """ + Move a batch of Unsorted items to their (possibly user-edited) + destinations, creating new subfolders as needed. Every source must + actually be inside the Unsorted folder and every destination must + stay inside the library root and outside Unsorted itself - guards + against a hand-edited destination path escaping the library, the same + zip-slip-style check used for backup restore. + """ + data = request.json or {} + moves = data.get('moves') + if not isinstance(moves, list) or not moves: + return jsonify({'success': False, 'error': 'No moves provided'}), 400 + + library_root = os.path.abspath(get_library_root()) + unsorted_root = os.path.abspath(os.path.join(library_root, UNSORTED_FOLDER_NAME)) + + moved = [] + errors = [] + for move in moves: + source = (move or {}).get('source', '') + destination_relative = ((move or {}).get('destination') or '').strip().strip('/') + + if not source or not destination_relative: + errors.append({'source': source, 'error': 'Missing source or destination'}) + continue + + source_abs = os.path.abspath(source) + if not (source_abs == unsorted_root or source_abs.startswith(unsorted_root + os.sep)): + errors.append({'source': source, 'error': 'Source is not inside the Unsorted folder'}) + continue + if not os.path.isdir(source_abs): + errors.append({'source': source, 'error': 'No longer exists - already moved?'}) + continue + + dest_dir = os.path.abspath(os.path.join(library_root, destination_relative)) + if not (dest_dir == library_root or dest_dir.startswith(library_root + os.sep)): + errors.append({'source': source, 'error': 'Destination is outside the library'}) + continue + if dest_dir == unsorted_root or dest_dir.startswith(unsorted_root + os.sep): + errors.append({'source': source, 'error': 'Destination cannot be inside Unsorted'}) + continue + + final_path = os.path.join(dest_dir, os.path.basename(source_abs)) + if os.path.exists(final_path): + errors.append({'source': source, 'error': f'"{os.path.basename(source_abs)}" already exists at that destination'}) + continue + + try: + os.makedirs(dest_dir, exist_ok=True) + shutil.move(source_abs, final_path) + moved.append({'source': source, 'destination': final_path}) + except OSError as e: + errors.append({'source': source, 'error': str(e)}) + + invalidate_cache() + return jsonify({'success': True, 'moved': len(moved), 'errors': errors}) + + # Single-process personal app, so a plain dict + background thread is # enough state for the duration prewarm - no job queue needed. Guarded by # 'running' so a second click while one's in flight just reports progress @@ -2981,6 +3249,12 @@ def notes_hub(): return render_template('notes_hub.html', notes=get_all_notes(), active_tab='notes') +@app.route('/unsorted') +def unsorted_page(): + """Review page for filing new courses out of the Unsorted folder - see /api/unsorted/scan for the actual proposals.""" + return render_template('unsorted.html', active_tab=None) + + @app.route('/library/search-transcripts') def search_transcripts_api(): """Search inside lesson subtitle files across the library.""" diff --git a/templates/settings.html b/templates/settings.html index d356b8b..af96a2d 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -462,6 +462,16 @@ +
+

Sort Unsorted Courses

+

+ Drop new courses in an "Unsorted" folder at the root of your library, then use this to + propose where each one belongs based on your existing folder structure - nothing moves + until you review and confirm on its own page. +

+ {{ icons.icon('folder', 14) }} Open Sorter +
+

Manage Library

diff --git a/templates/unsorted.html b/templates/unsorted.html new file mode 100644 index 0000000..6c18016 --- /dev/null +++ b/templates/unsorted.html @@ -0,0 +1,479 @@ +{% import '_icons.html' as icons %} + + + + + + Sort Unsorted - OfflineU + + + + + + +

+
+ + OfflineU +
+
+ +
+
+

{{ icons.icon('folder', 24) }} Sort Unsorted

+

Propose a destination for every course sitting in the Unsorted folder, based on your existing category structure. Nothing moves until you apply it.

+ +
+
+ + +
+
+ + + + +
+
+ + + + + + +