rmsitz/offlineu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cache library filesystem scans to fix ~90s NAS page loads

Browse Source 
There was no caching anywhere in the app, so every request re-walked
the SMB-mounted library from scratch - the dashboard alone triggered
2 duplicate full-library directory walks, 100+ individual
.offlineu_progress.json opens, and ~10-15 full per-course recursive
rglob() scans, none of it shared between requests. Measured ~90s per
dashboard load against the real mounted NAS, with a second immediate
reload taking just as long - proof nothing was being reused.

Add a minimal in-process TTL cache (5 min) and apply it at the actual
hot spots: the shared course-directory listing (get_all_course_dirs,
replacing 6 independent iter_all_courses() walks), the per-course
media-count/thumbnail summary (_course_summary, also now reused by
list_library_directory instead of a third duplicate implementation),
the per-course tree scan (get_course_tree - safe to cache since
progress is always re-applied fresh on top, never baked into the
cached structure), and the transcript-search subtitle index (the
worst offender - previously re-read and lowercased every subtitle
file in the library on every search).

Invalidates immediately on hide/show, rename, and library-path
changes; a "Refresh Library" button on Settings covers files added
directly on the NAS outside the app.

Measured after the fix, same real NAS mount: dashboard ~90s -> 1.3s
warm, Notes Hub ~1.15s warm, Library browser ~3.6s warm.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
rmsitz
2026-08-23 14:27:24 -04:00
co-authored by Claude Sonnet 5
parent bc6725ee4c
commit 2941910c54
2 changed files with 135 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
offlineu_core.py
+113 -49
View File
@@ -11,6 +11,7 @@ import re
import sys
import argparse
import io
import time
import uuid
import zipfile
import urllib.request
@@ -39,6 +40,32 @@ THUMBNAIL_BASENAMES = ('cover', 'folder', 'thumbnail', 'thumb', 'poster')
# COURSES_LIBRARY_PATH env var.
LIBRARY_PATH = os.environ.get('COURSES_LIBRARY_PATH', '/app/courses')
# The library is typically a network-mounted (SMB) directory, where every
# iterdir()/rglob() call is a network round-trip - and the same expensive
# scans (course listing, per-course media counts, subtitle text) get
# recomputed from scratch on every single request with nothing shared
# between them. The library itself only changes on human timescales
# (someone adds a course), not per-request, so a short-TTL in-process cache
# is a safe, high-leverage fix - no external cache needed for a
# single-process personal app.
_cache_store: Dict[str, Tuple[float, Any]] = {}
CACHE_TTL_SECONDS = 300 # 5 minutes
def cache_get_or_compute(key: str, compute, ttl: float = CACHE_TTL_SECONDS):
now = time.time()
cached = _cache_store.get(key)
if cached is not None and now - cached[0] < ttl:
return cached[1]
value = compute()
_cache_store[key] = (now, value)
return value
def invalidate_cache():
"""Clear all cached library-scan results - call after anything that changes what's on disk (rename, hide/show, library path change)."""
_cache_store.clear()
# App-wide (non per-course) persisted data, e.g. display settings. Matches
# the ./data volume mount in docker-compose.yml.
DATA_DIR = os.environ.get('OFFLINEU_DATA_DIR', '/app/data')
@@ -246,6 +273,8 @@ def save_settings(new_settings: Dict[str, Any]) -> Dict[str, Any]:
os.makedirs(DATA_DIR, exist_ok=True)
with open(SETTINGS_FILE, 'w') as f:
json.dump(current, f, indent=2)
if 'library_path' in new_settings:
invalidate_cache() # the library root itself changed
return current
@@ -503,6 +532,17 @@ class DynamicCourseParser:
app.jinja_env.globals['section_stats'] = DynamicCourseParser._calculate_completion_stats
def get_course_tree(course_path: str) -> Course:
"""
Cached DynamicCourseParser.scan_directory() - safe to cache the tree
*structure* this way because progress (watched/completed) is never
baked into it at scan time: ProgressTracker.apply_progress_to_tree()
always re-reads the live progress file and overwrites the Lesson
fields fresh on every render, same as before this cache existed.
"""
return cache_get_or_compute(f'course_tree:{course_path}', lambda: DynamicCourseParser.scan_directory(course_path))
def _has_direct_media(directory: Path) -> bool:
"""Check whether a directory contains media files directly (not recursively)"""
try:
@@ -600,6 +640,7 @@ def set_path_hidden(path: str, hidden: bool) -> List[str]:
os.makedirs(DATA_DIR, exist_ok=True)
with open(HIDDEN_PATHS_FILE, 'w') as f:
json.dump(result, f, indent=2)
invalidate_cache() # hidden set affects which courses get_all_course_dirs() yields
return result
@@ -762,18 +803,9 @@ def list_library_directory(dir_path: str, skip_hidden: bool = True) -> Dict[str,
continue
if _looks_like_course(entry):
media_count = len([
f for f in entry.rglob('*')
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
])
items.append({
'type': 'course',
'name': entry.name,
'path': entry_path_str,
'media_files': media_count,
'hidden': is_hidden,
'has_thumbnail': find_course_thumbnail(entry_path_str) is not None
})
item = _course_summary(entry)
item['hidden'] = is_hidden
items.append(item)
else:
if skip_hidden:
# Only count courses that aren't themselves curated out -
@@ -841,34 +873,47 @@ def iter_all_courses(dir_path: str) -> Iterator[Path]:
yield from walk(Path(dir_path))
def get_all_course_dirs() -> List[Path]:
"""
Cached, materialized iter_all_courses(get_library_root()) - every
caller (search, transcript search, Recently Added, library stats,
Notes Hub, backup export) needs the exact same recursive directory
walk over the library root, so compute it once and share it instead
of every caller re-walking the filesystem independently.
"""
return cache_get_or_compute('course_dirs', lambda: list(iter_all_courses(get_library_root())))
def _course_summary(course_dir: Path) -> Dict[str, Any]:
"""Build the {type, name, path, media_files, hidden, has_thumbnail} shape
shared by the Library browser, search results, and Recently Added."""
media_count = len([
f for f in course_dir.rglob('*')
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
])
return {
'type': 'course',
'name': course_dir.name,
'path': str(course_dir),
'media_files': media_count,
'hidden': False,
'has_thumbnail': find_course_thumbnail(str(course_dir)) is not None
}
def compute():
media_count = len([
f for f in course_dir.rglob('*')
if f.is_file() and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
])
return {
'type': 'course',
'name': course_dir.name,
'path': str(course_dir),
'media_files': media_count,
'hidden': False,
'has_thumbnail': find_course_thumbnail(str(course_dir)) is not None
}
return dict(cache_get_or_compute(f'course_summary:{course_dir}', compute))
def search_library_courses(dir_path: str, query: str) -> List[Dict[str, Any]]:
def search_library_courses(query: str) -> List[Dict[str, Any]]:
"""
Search the library for courses whose name contains `query`
(case-insensitive). The expensive per-course lookups (media_count,
thumbnail) only run for courses whose name actually matches - see
iter_all_courses.
get_all_course_dirs.
"""
query_lower = query.lower()
return [
_course_summary(course)
for course in iter_all_courses(dir_path)
for course in get_all_course_dirs()
if query_lower in course.name.lower()
]
@@ -891,14 +936,32 @@ def _extract_subtitle_snippet(text: str, query_lower: str, context_chars: int =
return ('…' if start > 0 else '') + snippet + ('…' if end < len(text) else '')
def _course_subtitle_index(course_dir: Path) -> List[Tuple[Path, str, str]]:
"""
Cached (subtitle_file, text, text_lower) triples for one course -
reading and lowercasing every subtitle file is identical work across
every search query, so do it once per cache window and just search the
already-read text in memory for each new query instead of re-reading
every file from disk (typically a network-mounted NAS) every time.
"""
def compute():
pairs = []
for f in course_dir.rglob('*'):
if f.is_file() and f.suffix.lower() in SUBTITLE_EXTENSIONS:
try:
text = f.read_text(encoding='utf-8', errors='ignore')
except OSError:
continue
pairs.append((f, text, text.lower()))
return pairs
return cache_get_or_compute(f'subtitle_index:{course_dir}', compute)
def search_transcripts(query: str, limit: int = 30) -> List[Dict[str, Any]]:
"""
Search inside lesson subtitle files (.srt/.vtt/etc.) for `query`,
case-insensitive. Unlike course-name search, subtitles live inside each
course's own directory tree rather than at the course boundary, so this
needs an actual per-course file walk - kept cheap by doing a raw
substring check as the filter step and only extracting a cleaned-up
snippet for files that actually match.
case-insensitive - see _course_subtitle_index for how the expensive
per-course file reads are cached and shared across queries.
Subtitle files aren't wired into the Lesson tree today (see
DynamicCourseParser._create_lesson_from_file - a subtitle file never
@@ -910,17 +973,11 @@ def search_transcripts(query: str, limit: int = 30) -> List[Dict[str, Any]]:
query_lower = query.lower()
results: List[Dict[str, Any]] = []
for course_dir in iter_all_courses(get_library_root()):
for subtitle_file in course_dir.rglob('*'):
for course_dir in get_all_course_dirs():
for subtitle_file, text, text_lower in _course_subtitle_index(course_dir):
if len(results) >= limit:
return results
if not subtitle_file.is_file() or subtitle_file.suffix.lower() not in SUBTITLE_EXTENSIONS:
continue
try:
text = subtitle_file.read_text(encoding='utf-8', errors='ignore')
except OSError:
continue
if query_lower not in text.lower():
if query_lower not in text_lower:
continue
media_match = next(
@@ -963,9 +1020,8 @@ def get_recently_added_courses(limit: int = 5) -> List[Dict[str, Any]]:
the dashboard's "Recently Added" card - separate from Recently Viewed
(which tracks what you've *watched*, not what showed up in the library).
"""
library_root = get_library_root()
dated = []
for course_dir in iter_all_courses(library_root):
for course_dir in get_all_course_dirs():
try:
mtime = course_dir.stat().st_mtime
except OSError:
@@ -1000,7 +1056,7 @@ def _scan_library_activity() -> Dict[str, Any]:
courses: List[Dict[str, Any]] = []
heatmap_cutoff = datetime.now().date() - timedelta(days=89)
for course_dir in iter_all_courses(get_library_root()):
for course_dir in get_all_course_dirs():
total_courses += 1
try:
with open(course_dir / '.offlineu_progress.json', 'r') as f:
@@ -1642,7 +1698,7 @@ def get_all_notes() -> List[Dict[str, Any]]:
lesson with several notes contributes one row per note.
"""
notes = []
for course_dir in iter_all_courses(get_library_root()):
for course_dir in get_all_course_dirs():
try:
with open(course_dir / '.offlineu_progress.json', 'r') as f:
progress = json.load(f)
@@ -1939,10 +1995,17 @@ def library_search():
if not query:
return jsonify({'library_path': library_root, 'results': []})
results = search_library_courses(library_root, query)
results = search_library_courses(query)
return jsonify({'library_path': library_root, 'results': results})
@app.route('/api/library/refresh', methods=['POST'])
def refresh_library_api():
"""Clear cached library-scan results - for when files were added/removed directly on disk (e.g. on the NAS) rather than through the app, which the cache's TTL would otherwise take up to 5 minutes to notice on its own."""
invalidate_cache()
return jsonify({'success': True})
@app.route('/api/hidden-paths', methods=['GET'])
def get_hidden_paths_api():
"""List currently-hidden course/directory paths, with display names."""
@@ -2017,6 +2080,7 @@ def perform_rename(old_path: str, new_name: str) -> Dict[str, Any]:
except OSError as e:
return {'success': False, 'error': f'Rename failed: {e}', 'status': 500}
invalidate_cache() # renamed path invalidates any cached listing/summary/tree keyed by the old path
rebase_library_path(old_abs, new_abs)
active_course_reset = False
@@ -2209,7 +2273,7 @@ def download_backup():
zf.write(path, name)
library_root = get_library_root()
for course_dir in iter_all_courses(library_root):
for course_dir in get_all_course_dirs():
progress_file = course_dir / '.offlineu_progress.json'
if progress_file.exists():
relative = os.path.relpath(str(course_dir), library_root)
@@ -2232,7 +2296,7 @@ def load_course():
return jsonify({'error': 'Invalid course path'}), 400
try:
current_course = DynamicCourseParser.scan_directory(course_path)
current_course = get_course_tree(course_path)
return jsonify({'success': True, 'course_name': current_course.name})
except Exception as e:
return jsonify({'error': str(e)}), 500
@@ -2255,7 +2319,7 @@ def open_recent():
try:
if not current_course or current_course.path != course_path:
current_course = DynamicCourseParser.scan_directory(course_path)
current_course = get_course_tree(course_path)
except Exception as e:
print(f"Could not load course for recent view: {e}")
return redirect(url_for('index'))
templates/settings.html
+22
View File
@@ -432,6 +432,11 @@
<button class="btn" onclick="saveLibraryPath()">Save</button>
</div>
<span id="library-path-status" style="font-size: 0.85em; min-height: 1.2em;"></span>
<div style="display: flex; align-items: center; gap: 10px;">
<button class="btn btn-secondary btn-sm" onclick="refreshLibraryCache()">🔄 Refresh Library</button>
<span id="library-refresh-status" style="font-size: 0.85em; color: var(--text-muted);"></span>
</div>
<span class="setting-desc">Courses are cached briefly for speed - use this after adding or removing files directly on disk if you don't want to wait a few minutes for it to notice.</span>
</div>
</div>
@@ -951,6 +956,23 @@
});
}
function refreshLibraryCache() {
const status = document.getElementById('library-refresh-status');
status.style.color = 'var(--text-muted)';
status.textContent = 'Refreshing…';
fetch('/api/library/refresh', { method: 'POST' })
.then(r => r.json())
.then(data => {
status.style.color = data.success ? 'var(--success)' : 'var(--error)';
status.textContent = data.success ? '✓ Refreshed' : 'Could not refresh';
if (data.success) setTimeout(() => { status.textContent = ''; }, 3000);
})
.catch(() => {
status.style.color = 'var(--error)';
status.textContent = 'Could not reach the server';
});
}
function loadCourseFromPath() {
const input = document.getElementById('manual-course-path');
const status = document.getElementById('manual-course-status');