diff --git a/offlineu_core.py b/offlineu_core.py
index c19bfcd..6b5c38d 100644
--- a/offlineu_core.py
+++ b/offlineu_core.py
@@ -11,6 +11,7 @@ import re
import sys
import argparse
import io
+import time
import uuid
import zipfile
import urllib.request
@@ -39,6 +40,32 @@ THUMBNAIL_BASENAMES = ('cover', 'folder', 'thumbnail', 'thumb', 'poster')
# COURSES_LIBRARY_PATH env var.
LIBRARY_PATH = os.environ.get('COURSES_LIBRARY_PATH', '/app/courses')
+# The library is typically a network-mounted (SMB) directory, where every
+# iterdir()/rglob() call is a network round-trip - and the same expensive
+# scans (course listing, per-course media counts, subtitle text) get
+# recomputed from scratch on every single request with nothing shared
+# between them. The library itself only changes on human timescales
+# (someone adds a course), not per-request, so a short-TTL in-process cache
+# is a safe, high-leverage fix - no external cache needed for a
+# single-process personal app.
+_cache_store: Dict[str, Tuple[float, Any]] = {}
+CACHE_TTL_SECONDS = 300 # 5 minutes
+
+
+def cache_get_or_compute(key: str, compute, ttl: float = CACHE_TTL_SECONDS):
+ now = time.time()
+ cached = _cache_store.get(key)
+ if cached is not None and now - cached[0] < ttl:
+ return cached[1]
+ value = compute()
+ _cache_store[key] = (now, value)
+ return value
+
+
+def invalidate_cache():
+ """Clear all cached library-scan results - call after anything that changes what's on disk (rename, hide/show, library path change)."""
+ _cache_store.clear()
+
# App-wide (non per-course) persisted data, e.g. display settings. Matches
# the ./data volume mount in docker-compose.yml.
DATA_DIR = os.environ.get('OFFLINEU_DATA_DIR', '/app/data')
@@ -246,6 +273,8 @@ def save_settings(new_settings: Dict[str, Any]) -> Dict[str, Any]:
os.makedirs(DATA_DIR, exist_ok=True)
with open(SETTINGS_FILE, 'w') as f:
json.dump(current, f, indent=2)
+ if 'library_path' in new_settings:
+ invalidate_cache() # the library root itself changed
return current
@@ -503,6 +532,17 @@ class DynamicCourseParser:
app.jinja_env.globals['section_stats'] = DynamicCourseParser._calculate_completion_stats
+def get_course_tree(course_path: str) -> Course:
+ """
+ Cached DynamicCourseParser.scan_directory() - safe to cache the tree
+ *structure* this way because progress (watched/completed) is never
+ baked into it at scan time: ProgressTracker.apply_progress_to_tree()
+ always re-reads the live progress file and overwrites the Lesson
+ fields fresh on every render, same as before this cache existed.
+ """
+ return cache_get_or_compute(f'course_tree:{course_path}', lambda: DynamicCourseParser.scan_directory(course_path))
+
+
def _has_direct_media(directory: Path) -> bool:
"""Check whether a directory contains media files directly (not recursively)"""
try:
@@ -600,6 +640,7 @@ def set_path_hidden(path: str, hidden: bool) -> List[str]:
os.makedirs(DATA_DIR, exist_ok=True)
with open(HIDDEN_PATHS_FILE, 'w') as f:
json.dump(result, f, indent=2)
+ invalidate_cache() # hidden set affects which courses get_all_course_dirs() yields
return result
@@ -762,18 +803,9 @@ def list_library_directory(dir_path: str, skip_hidden: bool = True) -> Dict[str,
continue
if _looks_like_course(entry):
- media_count = len([
- f for f in entry.rglob('*')
- if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
- ])
- items.append({
- 'type': 'course',
- 'name': entry.name,
- 'path': entry_path_str,
- 'media_files': media_count,
- 'hidden': is_hidden,
- 'has_thumbnail': find_course_thumbnail(entry_path_str) is not None
- })
+ item = _course_summary(entry)
+ item['hidden'] = is_hidden
+ items.append(item)
else:
if skip_hidden:
# Only count courses that aren't themselves curated out -
@@ -841,34 +873,47 @@ def iter_all_courses(dir_path: str) -> Iterator[Path]:
yield from walk(Path(dir_path))
+def get_all_course_dirs() -> List[Path]:
+ """
+ Cached, materialized iter_all_courses(get_library_root()) - every
+ caller (search, transcript search, Recently Added, library stats,
+ Notes Hub, backup export) needs the exact same recursive directory
+ walk over the library root, so compute it once and share it instead
+ of every caller re-walking the filesystem independently.
+ """
+ return cache_get_or_compute('course_dirs', lambda: list(iter_all_courses(get_library_root())))
+
+
def _course_summary(course_dir: Path) -> Dict[str, Any]:
"""Build the {type, name, path, media_files, hidden, has_thumbnail} shape
shared by the Library browser, search results, and Recently Added."""
- media_count = len([
- f for f in course_dir.rglob('*')
- if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
- ])
- return {
- 'type': 'course',
- 'name': course_dir.name,
- 'path': str(course_dir),
- 'media_files': media_count,
- 'hidden': False,
- 'has_thumbnail': find_course_thumbnail(str(course_dir)) is not None
- }
+ def compute():
+ media_count = len([
+ f for f in course_dir.rglob('*')
+ if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
+ ])
+ return {
+ 'type': 'course',
+ 'name': course_dir.name,
+ 'path': str(course_dir),
+ 'media_files': media_count,
+ 'hidden': False,
+ 'has_thumbnail': find_course_thumbnail(str(course_dir)) is not None
+ }
+ return dict(cache_get_or_compute(f'course_summary:{course_dir}', compute))
-def search_library_courses(dir_path: str, query: str) -> List[Dict[str, Any]]:
+def search_library_courses(query: str) -> List[Dict[str, Any]]:
"""
Search the library for courses whose name contains `query`
(case-insensitive). The expensive per-course lookups (media_count,
thumbnail) only run for courses whose name actually matches - see
- iter_all_courses.
+ get_all_course_dirs.
"""
query_lower = query.lower()
return [
_course_summary(course)
- for course in iter_all_courses(dir_path)
+ for course in get_all_course_dirs()
if query_lower in course.name.lower()
]
@@ -891,14 +936,32 @@ def _extract_subtitle_snippet(text: str, query_lower: str, context_chars: int =
return ('…' if start > 0 else '') + snippet + ('…' if end < len(text) else '')
+def _course_subtitle_index(course_dir: Path) -> List[Tuple[Path, str, str]]:
+ """
+ Cached (subtitle_file, text, text_lower) triples for one course -
+ reading and lowercasing every subtitle file is identical work across
+ every search query, so do it once per cache window and just search the
+ already-read text in memory for each new query instead of re-reading
+ every file from disk (typically a network-mounted NAS) every time.
+ """
+ def compute():
+ pairs = []
+ for f in course_dir.rglob('*'):
+ if f.is_file() and f.suffix.lower() in SUBTITLE_EXTENSIONS:
+ try:
+ text = f.read_text(encoding='utf-8', errors='ignore')
+ except OSError:
+ continue
+ pairs.append((f, text, text.lower()))
+ return pairs
+ return cache_get_or_compute(f'subtitle_index:{course_dir}', compute)
+
+
def search_transcripts(query: str, limit: int = 30) -> List[Dict[str, Any]]:
"""
Search inside lesson subtitle files (.srt/.vtt/etc.) for `query`,
- case-insensitive. Unlike course-name search, subtitles live inside each
- course's own directory tree rather than at the course boundary, so this
- needs an actual per-course file walk - kept cheap by doing a raw
- substring check as the filter step and only extracting a cleaned-up
- snippet for files that actually match.
+ case-insensitive - see _course_subtitle_index for how the expensive
+ per-course file reads are cached and shared across queries.
Subtitle files aren't wired into the Lesson tree today (see
DynamicCourseParser._create_lesson_from_file - a subtitle file never
@@ -910,17 +973,11 @@ def search_transcripts(query: str, limit: int = 30) -> List[Dict[str, Any]]:
query_lower = query.lower()
results: List[Dict[str, Any]] = []
- for course_dir in iter_all_courses(get_library_root()):
- for subtitle_file in course_dir.rglob('*'):
+ for course_dir in get_all_course_dirs():
+ for subtitle_file, text, text_lower in _course_subtitle_index(course_dir):
if len(results) >= limit:
return results
- if not subtitle_file.is_file() or subtitle_file.suffix.lower() not in SUBTITLE_EXTENSIONS:
- continue
- try:
- text = subtitle_file.read_text(encoding='utf-8', errors='ignore')
- except OSError:
- continue
- if query_lower not in text.lower():
+ if query_lower not in text_lower:
continue
media_match = next(
@@ -963,9 +1020,8 @@ def get_recently_added_courses(limit: int = 5) -> List[Dict[str, Any]]:
the dashboard's "Recently Added" card - separate from Recently Viewed
(which tracks what you've *watched*, not what showed up in the library).
"""
- library_root = get_library_root()
dated = []
- for course_dir in iter_all_courses(library_root):
+ for course_dir in get_all_course_dirs():
try:
mtime = course_dir.stat().st_mtime
except OSError:
@@ -1000,7 +1056,7 @@ def _scan_library_activity() -> Dict[str, Any]:
courses: List[Dict[str, Any]] = []
heatmap_cutoff = datetime.now().date() - timedelta(days=89)
- for course_dir in iter_all_courses(get_library_root()):
+ for course_dir in get_all_course_dirs():
total_courses += 1
try:
with open(course_dir / '.offlineu_progress.json', 'r') as f:
@@ -1642,7 +1698,7 @@ def get_all_notes() -> List[Dict[str, Any]]:
lesson with several notes contributes one row per note.
"""
notes = []
- for course_dir in iter_all_courses(get_library_root()):
+ for course_dir in get_all_course_dirs():
try:
with open(course_dir / '.offlineu_progress.json', 'r') as f:
progress = json.load(f)
@@ -1939,10 +1995,17 @@ def library_search():
if not query:
return jsonify({'library_path': library_root, 'results': []})
- results = search_library_courses(library_root, query)
+ results = search_library_courses(query)
return jsonify({'library_path': library_root, 'results': results})
+@app.route('/api/library/refresh', methods=['POST'])
+def refresh_library_api():
+ """Clear cached library-scan results - for when files were added/removed directly on disk (e.g. on the NAS) rather than through the app, which the cache's TTL would otherwise take up to 5 minutes to notice on its own."""
+ invalidate_cache()
+ return jsonify({'success': True})
+
+
@app.route('/api/hidden-paths', methods=['GET'])
def get_hidden_paths_api():
"""List currently-hidden course/directory paths, with display names."""
@@ -2017,6 +2080,7 @@ def perform_rename(old_path: str, new_name: str) -> Dict[str, Any]:
except OSError as e:
return {'success': False, 'error': f'Rename failed: {e}', 'status': 500}
+ invalidate_cache() # renamed path invalidates any cached listing/summary/tree keyed by the old path
rebase_library_path(old_abs, new_abs)
active_course_reset = False
@@ -2209,7 +2273,7 @@ def download_backup():
zf.write(path, name)
library_root = get_library_root()
- for course_dir in iter_all_courses(library_root):
+ for course_dir in get_all_course_dirs():
progress_file = course_dir / '.offlineu_progress.json'
if progress_file.exists():
relative = os.path.relpath(str(course_dir), library_root)
@@ -2232,7 +2296,7 @@ def load_course():
return jsonify({'error': 'Invalid course path'}), 400
try:
- current_course = DynamicCourseParser.scan_directory(course_path)
+ current_course = get_course_tree(course_path)
return jsonify({'success': True, 'course_name': current_course.name})
except Exception as e:
return jsonify({'error': str(e)}), 500
@@ -2255,7 +2319,7 @@ def open_recent():
try:
if not current_course or current_course.path != course_path:
- current_course = DynamicCourseParser.scan_directory(course_path)
+ current_course = get_course_tree(course_path)
except Exception as e:
print(f"Could not load course for recent view: {e}")
return redirect(url_for('index'))
diff --git a/templates/settings.html b/templates/settings.html
index 2f49646..e9079b9 100644
--- a/templates/settings.html
+++ b/templates/settings.html
@@ -432,6 +432,11 @@
+
+
+
+
+ Courses are cached briefly for speed - use this after adding or removing files directly on disk if you don't want to wait a few minutes for it to notice.
@@ -951,6 +956,23 @@
});
}
+ function refreshLibraryCache() {
+ const status = document.getElementById('library-refresh-status');
+ status.style.color = 'var(--text-muted)';
+ status.textContent = 'Refreshing…';
+ fetch('/api/library/refresh', { method: 'POST' })
+ .then(r => r.json())
+ .then(data => {
+ status.style.color = data.success ? 'var(--success)' : 'var(--error)';
+ status.textContent = data.success ? '✓ Refreshed' : 'Could not refresh';
+ if (data.success) setTimeout(() => { status.textContent = ''; }, 3000);
+ })
+ .catch(() => {
+ status.style.color = 'var(--error)';
+ status.textContent = 'Could not reach the server';
+ });
+ }
+
function loadCourseFromPath() {
const input = document.getElementById('manual-course-path');
const status = document.getElementById('manual-course-status');