Cache the category tree walk (fixes ~1s File Management page load)
_build_category_index() was the one tree-scanning function never wrapped in the app's existing 5-minute cache - it ran fresh on every File Management page load and Sort Unsorted scan (not a manual "this may take a moment" action like Duplicate Courses or Storage Usage), so the cost was invisible until it was already slow. Measured ~1.0-1.1s consistently against the live NAS-mounted library at 161 courses, confirmed via profiling that a single call makes dozens of iterdir() round-trips - each one a network hop over SMB. Now shares the same cache_get_or_compute pattern and invalidate_cache() call sites as get_all_course_dirs(), so no new invalidation logic was needed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -137,8 +137,12 @@ silently stay blank instead of erroring.
|
|||||||
ones that recur across many categories (a prolific creator's name, etc.)
|
ones that recur across many categories (a prolific creator's name, etc.)
|
||||||
so they can't outvote a genuinely specific word just by sharing more of
|
so they can't outvote a genuinely specific word just by sharing more of
|
||||||
them.
|
them.
|
||||||
- *Refresh Library*: manually bypasses the 5-minute filesystem-scan cache,
|
- *Refresh Library*: manually bypasses the 5-minute filesystem-scan cache
|
||||||
for when files were added/removed directly on disk.
|
(this now includes the category tree Sort Unsorted/Manage Library's
|
||||||
|
picker builds - previously rebuilt on every page load, a full,
|
||||||
|
uncached walk of the whole library that got noticeably slow over a
|
||||||
|
NAS-mounted (SMB) library as the course count grew), for when files
|
||||||
|
were added/removed directly on disk.
|
||||||
- *Bulk Rename*: find & replace across every course/folder name in the
|
- *Bulk Rename*: find & replace across every course/folder name in the
|
||||||
library at once, with a per-match preview and the ability to drop
|
library at once, with a per-match preview and the ability to drop
|
||||||
individual matches before applying. Three match modes: plain text
|
individual matches before applying. Three match modes: plain text
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
2026-08-24 18:40 UTC — search, bulk actions, undo, storage usage
|
2026-08-24 21:41 UTC — cache category tree walk (fixes ~1s picker lag)
|
||||||
|
|||||||
+58
-45
@@ -1445,54 +1445,67 @@ def _build_category_index() -> List[Dict[str, Any]]:
|
|||||||
courses and their non-lecture contents never show up as if they were
|
courses and their non-lecture contents never show up as if they were
|
||||||
categories to file things under. The Unsorted folder itself is
|
categories to file things under. The Unsorted folder itself is
|
||||||
excluded - it's the source, never a valid destination.
|
excluded - it's the source, never a valid destination.
|
||||||
|
|
||||||
|
This walk visits every directory in the library and, over a
|
||||||
|
NAS-mounted (SMB) library, each of those is a network round-trip -
|
||||||
|
the same reasoning behind get_all_course_dirs()'s cache, so this
|
||||||
|
result is cached the same way (same 5-minute TTL, same
|
||||||
|
invalidate_cache() call sites already busting it - no separate
|
||||||
|
invalidation needed): otherwise this ran fresh on every File
|
||||||
|
Management page load and Sort Unsorted scan, neither of which is a
|
||||||
|
manual "this may take a moment" action like Duplicate Courses or
|
||||||
|
Storage Usage, so the cost was invisible until it was already slow.
|
||||||
"""
|
"""
|
||||||
library_root = Path(get_library_root())
|
def compute() -> List[Dict[str, Any]]:
|
||||||
try:
|
library_root = Path(get_library_root())
|
||||||
unsorted_root = (library_root / UNSORTED_FOLDER_NAME).resolve()
|
|
||||||
except OSError:
|
|
||||||
unsorted_root = None
|
|
||||||
categories: List[Dict[str, Any]] = []
|
|
||||||
|
|
||||||
def walk(directory: Path, path_parts: List[str]):
|
|
||||||
try:
|
try:
|
||||||
entries = sorted(
|
unsorted_root = (library_root / UNSORTED_FOLDER_NAME).resolve()
|
||||||
(p for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')),
|
except OSError:
|
||||||
key=lambda p: p.name.lower()
|
unsorted_root = None
|
||||||
)
|
categories: List[Dict[str, Any]] = []
|
||||||
except (PermissionError, OSError):
|
|
||||||
return
|
|
||||||
for entry in entries:
|
|
||||||
try:
|
|
||||||
if unsorted_root is not None and entry.resolve() == unsorted_root:
|
|
||||||
continue
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
if (_looks_like_course(entry) or _is_unrecognized_leaf_item(entry)
|
|
||||||
or _is_media_free_subtree(entry) or _is_course_wrapper(entry)):
|
|
||||||
continue
|
|
||||||
new_parts = path_parts + [entry.name]
|
|
||||||
path_tokens: Set[str] = set()
|
|
||||||
for part in new_parts:
|
|
||||||
path_tokens |= _tokenize(part)
|
|
||||||
bonus_tokens: Set[str] = set()
|
|
||||||
try:
|
|
||||||
for child in entry.iterdir():
|
|
||||||
if child.is_dir() and not child.name.startswith('.') and _looks_like_course(child):
|
|
||||||
bonus_tokens |= _tokenize(child.name)
|
|
||||||
except (PermissionError, OSError):
|
|
||||||
pass
|
|
||||||
bonus_tokens -= path_tokens
|
|
||||||
categories.append({
|
|
||||||
'path': str(entry),
|
|
||||||
'relative': '/'.join(new_parts),
|
|
||||||
'depth': len(new_parts),
|
|
||||||
'path_tokens': path_tokens,
|
|
||||||
'bonus_tokens': bonus_tokens,
|
|
||||||
})
|
|
||||||
walk(entry, new_parts)
|
|
||||||
|
|
||||||
walk(library_root, [])
|
def walk(directory: Path, path_parts: List[str]):
|
||||||
return categories
|
try:
|
||||||
|
entries = sorted(
|
||||||
|
(p for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')),
|
||||||
|
key=lambda p: p.name.lower()
|
||||||
|
)
|
||||||
|
except (PermissionError, OSError):
|
||||||
|
return
|
||||||
|
for entry in entries:
|
||||||
|
try:
|
||||||
|
if unsorted_root is not None and entry.resolve() == unsorted_root:
|
||||||
|
continue
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
if (_looks_like_course(entry) or _is_unrecognized_leaf_item(entry)
|
||||||
|
or _is_media_free_subtree(entry) or _is_course_wrapper(entry)):
|
||||||
|
continue
|
||||||
|
new_parts = path_parts + [entry.name]
|
||||||
|
path_tokens: Set[str] = set()
|
||||||
|
for part in new_parts:
|
||||||
|
path_tokens |= _tokenize(part)
|
||||||
|
bonus_tokens: Set[str] = set()
|
||||||
|
try:
|
||||||
|
for child in entry.iterdir():
|
||||||
|
if child.is_dir() and not child.name.startswith('.') and _looks_like_course(child):
|
||||||
|
bonus_tokens |= _tokenize(child.name)
|
||||||
|
except (PermissionError, OSError):
|
||||||
|
pass
|
||||||
|
bonus_tokens -= path_tokens
|
||||||
|
categories.append({
|
||||||
|
'path': str(entry),
|
||||||
|
'relative': '/'.join(new_parts),
|
||||||
|
'depth': len(new_parts),
|
||||||
|
'path_tokens': path_tokens,
|
||||||
|
'bonus_tokens': bonus_tokens,
|
||||||
|
})
|
||||||
|
walk(entry, new_parts)
|
||||||
|
|
||||||
|
walk(library_root, [])
|
||||||
|
return categories
|
||||||
|
|
||||||
|
return cache_get_or_compute('category_index', compute)
|
||||||
|
|
||||||
|
|
||||||
def _bonus_token_frequency(categories: List[Dict[str, Any]]) -> Dict[str, int]:
|
def _bonus_token_frequency(categories: List[Dict[str, Any]]) -> Dict[str, int]:
|
||||||
|
|||||||
Reference in New Issue
Block a user