rmsitz/offlineu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Notes Hub, transcript search, stale nudges, study guide export, Next Up queue, activity heatmap

Browse Source 
Six more dashboard features, all built on the per-course progress-file
scanning infrastructure from earlier sessions:

- Notes Hub (/notes): every lesson note across the library in one place,
  newest first, reusing the existing /recent/open cross-course jump route.
- Transcript search: opt-in checkbox on the Library search box, searching
  inside .srt/.vtt files. Kept cheap by doing a raw substring match as the
  filter step and only extracting a snippet for files that actually match.
- Stale course nudges ("Pick Back Up"): courses with progress that haven't
  been touched in 14+ days.
- Study guide export: compiles a course's notes into one downloadable
  markdown file, section/lesson structure preserved.
- Next Up queue: an ordered, persisted "what to tackle next" list with
  up/down reordering and a queue button on every course row.
- Activity heatmap: a 90-day contribution-style calendar in the Library
  Stats card.

_scan_library_activity() now does one walk over every course's progress
file per dashboard load; library stats, stale courses, and the heatmap all
derive from that single scan instead of three independent ones.

Also refactored the duplicated lesson-progress-key lookup (used in three
places now) into _resolve_lesson_progress_key(), and flagged a pre-existing
bug found along the way (not fixed here, kept out of scope): subtitle files
never actually attach to a Lesson object, so the video player's caption
track has never had anything to render.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
rmsitz
2026-08-22 19:04:01 -04:00
co-authored by Claude Sonnet 5
parent 145f91a647
commit 0454834e62
7 changed files with 942 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
offlineu_core.py
+392 -34
View File
@@ -596,6 +596,59 @@ def set_path_hidden(path: str, hidden: bool) -> List[str]:
return result
NEXT_UP_FILE = os.path.join(DATA_DIR, 'next_up.json')
def get_next_up_paths() -> List[str]:
"""Load the ordered 'Next Up' course queue - order matters here, unlike hidden_paths, so it's a list, not a set."""
try:
if os.path.exists(NEXT_UP_FILE):
with open(NEXT_UP_FILE, 'r') as f:
data = json.load(f)
if isinstance(data, list):
return data
except (json.JSONDecodeError, OSError) as e:
print(f"Could not load Next Up queue: {e}")
return []
def _save_next_up_paths(paths: List[str]) -> None:
os.makedirs(DATA_DIR, exist_ok=True)
with open(NEXT_UP_FILE, 'w') as f:
json.dump(paths, f, indent=2)
def set_path_queued(path: str, queued: bool) -> List[str]:
"""Add (to the end) or remove a course from the Next Up queue; returns the updated ordered list."""
paths = get_next_up_paths()
normalized = os.path.abspath(path)
if queued:
if normalized not in paths:
paths.append(normalized)
else:
paths = [p for p in paths if p != normalized]
_save_next_up_paths(paths)
return paths
def reorder_next_up(paths: List[str]) -> List[str]:
"""Replace the Next Up order wholesale - the client computes the new order (via up/down buttons) and posts it back."""
normalized = [os.path.abspath(p) for p in paths]
_save_next_up_paths(normalized)
return normalized
def get_next_up_courses() -> List[Dict[str, Any]]:
"""Next Up queue resolved to displayable course summaries, silently dropping any path no longer on disk."""
results = []
for path in get_next_up_paths():
p = Path(path)
if p.is_dir():
results.append(_course_summary(p))
return results
return result
def _rebase_prefix(value: str, old_abs: str, new_abs: str) -> str:
"""If `value` equals or is nested under `old_abs`, rewrite that prefix to `new_abs`."""
if value == old_abs:
@@ -813,6 +866,78 @@ def search_library_courses(dir_path: str, query: str) -> List[Dict[str, Any]]:
]
def _extract_subtitle_snippet(text: str, query_lower: str, context_chars: int = 80) -> str:
"""A short excerpt around the first match, with subtitle sequence-number/timestamp lines stripped."""
lower = text.lower()
idx = lower.find(query_lower)
if idx == -1:
return ''
start = max(0, idx - context_chars)
end = min(len(text), idx + len(query_lower) + context_chars)
excerpt_lines = text[start:end].splitlines()
cleaned = [
line.strip() for line in excerpt_lines
if line.strip() and '-->' not in line and not line.strip().isdigit()
and line.strip().upper() != 'WEBVTT'
]
snippet = ' '.join(cleaned)
return ('…' if start > 0 else '') + snippet + ('…' if end < len(text) else '')
def search_transcripts(query: str, limit: int = 30) -> List[Dict[str, Any]]:
"""
Search inside lesson subtitle files (.srt/.vtt/etc.) for `query`,
case-insensitive. Unlike course-name search, subtitles live inside each
course's own directory tree rather than at the course boundary, so this
needs an actual per-course file walk - kept cheap by doing a raw
substring check as the filter step and only extracting a cleaned-up
snippet for files that actually match.
Subtitle files aren't wired into the Lesson tree today (see
DynamicCourseParser._create_lesson_from_file - a subtitle file never
becomes part of a Lesson, so lesson.subtitle_file is always empty).
Rather than depend on that, this matches a subtitle file to its lesson
by finding a same-named video/audio file next to it, which is how
these files are conventionally paired on disk regardless.
"""
query_lower = query.lower()
results: List[Dict[str, Any]] = []
for course_dir in iter_all_courses(get_library_root()):
for subtitle_file in course_dir.rglob('*'):
if len(results) >= limit:
return results
if not subtitle_file.is_file() or subtitle_file.suffix.lower() not in SUBTITLE_EXTENSIONS:
continue
try:
text = subtitle_file.read_text(encoding='utf-8', errors='ignore')
except OSError:
continue
if query_lower not in text.lower():
continue
media_match = next(
(f for f in subtitle_file.parent.iterdir()
if f.is_file() and f.stem == subtitle_file.stem
and f.suffix.lower() in VIDEO_EXTENSIONS | AUDIO_EXTENSIONS),
None
)
if not media_match:
continue # no lesson to navigate to - skip rather than dead-end
lesson_title = DynamicCourseParser._clean_lesson_name(media_match.stem)
lesson_relative = media_match.relative_to(course_dir).as_posix()
results.append({
'course_path': str(course_dir),
'course_name': course_dir.name,
'lesson_path': f"{lesson_relative}/{lesson_title.replace(' ', '_')}",
'lesson_title': lesson_title,
'snippet': _extract_subtitle_snippet(text, query_lower)
})
return results
def _humanize_days_ago(dt: datetime) -> str:
"""'Today' / 'Yesterday' / 'N days ago' / a plain date once it's old enough to matter less."""
days = (datetime.now().date() - dt.date()).days
@@ -849,12 +974,14 @@ def get_recently_added_courses(limit: int = 5) -> List[Dict[str, Any]]:
return results
def get_library_stats() -> Dict[str, Any]:
def _scan_library_activity() -> Dict[str, Any]:
"""
Library-wide overview for the dashboard: total courses, lessons with
any progress record, completed lessons, total time watched, and the
current daily streak. Reads each course's small progress JSON file
directly rather than re-scanning course directory contents, so this
One walk over every course's progress file, computing everything the
dashboard's stats card / stale-course nudges / activity heatmap need -
library_stats/stale_courses/activity_heatmap in index() all derive from
a single call to this rather than each re-scanning every course's
progress.json independently. Reads each course's small progress JSON
file directly, not by re-scanning course directory contents, so this
stays cheap regardless of how many files are inside each course.
"""
total_courses = 0
@@ -862,6 +989,9 @@ def get_library_stats() -> Dict[str, Any]:
completed_lessons = 0
watched_seconds = 0
active_dates = set()
activity_by_date: Dict[str, int] = {}
courses: List[Dict[str, Any]] = []
heatmap_cutoff = datetime.now().date() - timedelta(days=89)
for course_dir in iter_all_courses(get_library_root()):
total_courses += 1
@@ -871,22 +1001,44 @@ def get_library_stats() -> Dict[str, Any]:
except (FileNotFoundError, json.JSONDecodeError, OSError):
continue
course_total = 0
course_completed = 0
course_last_activity: Optional[datetime] = None
for key, entry in progress.items():
if key == 'last_accessed_path' or not isinstance(entry, dict):
continue
lessons_tracked += 1
course_total += 1
if entry.get('completed'):
completed_lessons += 1
course_completed += 1
watched_seconds += entry.get('duration_seconds') or 0
else:
watched_seconds += entry.get('progress_seconds') or 0
last_accessed = entry.get('last_accessed')
if last_accessed:
try:
active_dates.add(datetime.fromisoformat(last_accessed).date())
except ValueError:
pass
if not last_accessed:
continue
try:
accessed_dt = datetime.fromisoformat(last_accessed)
except ValueError:
continue
accessed_date = accessed_dt.date()
active_dates.add(accessed_date)
if course_last_activity is None or accessed_dt > course_last_activity:
course_last_activity = accessed_dt
if accessed_date >= heatmap_cutoff:
iso = accessed_date.isoformat()
activity_by_date[iso] = activity_by_date.get(iso, 0) + 1
if course_total > 0:
courses.append({
'path': str(course_dir),
'total': course_total,
'completed': course_completed,
'last_activity': course_last_activity
})
streak_days = 0
day = datetime.now().date()
@@ -898,11 +1050,61 @@ def get_library_stats() -> Dict[str, Any]:
'total_courses': total_courses,
'lessons_tracked': lessons_tracked,
'completed_lessons': completed_lessons,
'watched_display': format_duration(watched_seconds),
'streak_days': streak_days
'watched_seconds': watched_seconds,
'streak_days': streak_days,
'activity_by_date': activity_by_date,
'courses': courses,
}
def format_library_stats(scan: Dict[str, Any]) -> Dict[str, Any]:
"""Library-wide overview for the dashboard's stats card, from a _scan_library_activity() result."""
return {
'total_courses': scan['total_courses'],
'lessons_tracked': scan['lessons_tracked'],
'completed_lessons': scan['completed_lessons'],
'watched_display': format_duration(scan['watched_seconds']),
'streak_days': scan['streak_days']
}
def format_stale_courses(scan: Dict[str, Any], threshold_days: int = 14, limit: int = 5) -> List[Dict[str, Any]]:
"""
Courses with some progress that haven't been touched in a while, oldest
first - courses with zero progress (never started) and fully completed
ones are both excluded, since neither is something to "pick back up."
"""
cutoff = datetime.now() - timedelta(days=threshold_days)
candidates = [
c for c in scan['courses']
if c['completed'] < c['total'] and c['last_activity'] and c['last_activity'] < cutoff
]
candidates.sort(key=lambda c: c['last_activity'])
results = []
for c in candidates[:limit]:
item = _course_summary(Path(c['path']))
days_ago = (datetime.now() - c['last_activity']).days
item['last_touched_display'] = f"{days_ago} day{'s' if days_ago != 1 else ''} ago"
results.append(item)
return results
def format_activity_heatmap(scan: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
The last 90 days as a flat list (oldest first) for the dashboard's
contribution-style calendar, each with an ISO date and that day's
activity count.
"""
activity_by_date = scan['activity_by_date']
today = datetime.now().date()
return [
{'date': (today - timedelta(days=offset)).isoformat(),
'count': activity_by_date.get((today - timedelta(days=offset)).isoformat(), 0)}
for offset in range(89, -1, -1)
]
# ---- Outline integration ----
# Deliberately kept separate from DEFAULT_SETTINGS/load_settings/save_settings:
# those all flow through GET /api/settings, which theme.js fetches on every
@@ -1167,6 +1369,24 @@ def get_recent_views_for_display() -> List[Dict[str, Any]]:
return entries
def _resolve_lesson_progress_key(course: Course, lesson: Lesson, progress: Dict[str, Any]) -> Optional[str]:
"""
Match a Lesson to its progress-file key. Lessons have historically been
keyed two ways - by their relative file path alone, or with the
lesson's title suffix appended (see get_lesson_url) - so check both
rather than assuming one.
"""
lesson_path = os.path.relpath(lesson.path, course.path).replace('\\', '/')
if lesson_path.startswith('/'):
lesson_path = lesson_path[1:]
if lesson_path in progress:
return lesson_path
lesson_path_with_title = f"{lesson_path}/{lesson.title.replace(' ', '_')}"
if lesson_path_with_title in progress:
return lesson_path_with_title
return None
class ProgressTracker:
"""Handles progress tracking and persistence"""
@@ -1287,25 +1507,14 @@ class ProgressTracker:
def apply_to_node(node: DirectoryNode):
# Apply progress to lessons in this node
for lesson in node.lessons:
lesson_path = os.path.relpath(lesson.path, course.path)
lesson_path = lesson_path.replace('\\', '/')
if lesson_path.startswith('/'):
lesson_path = lesson_path[1:]
# Check both the base path and path with title
lesson_path_with_title = f"{lesson_path}/{lesson.title.replace(' ', '_')}"
if lesson_path in progress:
lesson.completed = progress[lesson_path].get('completed', False)
lesson.last_accessed = progress[lesson_path].get('last_accessed')
lesson.progress_seconds = progress[lesson_path].get('progress_seconds', 0)
lesson.duration_seconds = progress[lesson_path].get('duration_seconds', 0)
elif lesson_path_with_title in progress:
lesson.completed = progress[lesson_path_with_title].get('completed', False)
lesson.last_accessed = progress[lesson_path_with_title].get('last_accessed')
lesson.progress_seconds = progress[lesson_path_with_title].get('progress_seconds', 0)
lesson.duration_seconds = progress[lesson_path_with_title].get('duration_seconds', 0)
key = _resolve_lesson_progress_key(course, lesson, progress)
if key:
entry = progress[key]
lesson.completed = entry.get('completed', False)
lesson.last_accessed = entry.get('last_accessed')
lesson.progress_seconds = entry.get('progress_seconds', 0)
lesson.duration_seconds = entry.get('duration_seconds', 0)
# Recursively apply to children
for child in node.children.values():
apply_to_node(child)
@@ -1319,6 +1528,91 @@ class ProgressTracker:
return DynamicCourseParser._calculate_completion_stats(course.root_node)
def course_has_any_notes(course: Course) -> bool:
"""Whether any lesson in the course has a saved note - used to decide whether to offer a study-guide download."""
progress = ProgressTracker.load_progress(course)
return any(
isinstance(entry, dict) and entry.get('note')
for key, entry in progress.items() if key != 'last_accessed_path'
)
def get_all_notes() -> List[Dict[str, Any]]:
"""
Every lesson note across the whole library, newest first - a note is
otherwise only visible from its own lesson page (or Outline, once
pushed), so this is the one place to see everything you've written.
"""
notes = []
for course_dir in iter_all_courses(get_library_root()):
try:
with open(course_dir / '.offlineu_progress.json', 'r') as f:
progress = json.load(f)
except (FileNotFoundError, json.JSONDecodeError, OSError):
continue
for lesson_path, entry in progress.items():
if lesson_path == 'last_accessed_path' or not isinstance(entry, dict):
continue
note = entry.get('note')
if not note:
continue
notes.append({
'course_path': str(course_dir),
'course_name': course_dir.name,
'lesson_path': lesson_path,
'lesson_title': lesson_path.rsplit('/', 1)[-1].replace('_', ' '),
'note': note,
'last_accessed': entry.get('last_accessed', ''),
'pushed_to_outline': bool(entry.get('outline_document_id'))
})
notes.sort(key=lambda n: n['last_accessed'], reverse=True)
return notes
def build_study_guide_markdown(course: Course) -> str:
"""
Compile every note written for a course into one markdown document,
following the course's own section/lesson structure - only sections
and lessons that actually have a note appear; everything else is
skipped rather than padding the guide with empty headings.
"""
progress = ProgressTracker.load_progress(course)
def node_has_notes(node: DirectoryNode) -> bool:
for lesson in node.lessons:
key = _resolve_lesson_progress_key(course, lesson, progress)
if key and progress[key].get('note'):
return True
return any(node_has_notes(child) for child in node.children.values())
lines = [f"# {course.name}", ""]
def walk(node: DirectoryNode, heading_level: int):
if node.name and node.name != "Course Root":
if not node_has_notes(node):
return
lines.append(f"{'#' * min(heading_level, 6)} {node.name}")
lines.append("")
for lesson in node.lessons:
key = _resolve_lesson_progress_key(course, lesson, progress)
note = progress[key].get('note') if key else None
if not note:
continue
lines.append(f"{'#' * min(heading_level + 1, 6)} {lesson.title}")
lines.append("")
lines.append(note)
lines.append("")
for child in node.children.values():
walk(child, heading_level + 1)
walk(course.root_node, 2)
return "\n".join(lines)
# Global course storage
current_course = None
@@ -1355,14 +1649,18 @@ def index():
continue_watching, recent_views = _split_continue_watching(get_recent_views_for_display())
if current_course is None:
# Show dashboard with course selection option
# One scan covers stats/stale-courses/heatmap - see _scan_library_activity.
scan = _scan_library_activity()
return render_template('course_dashboard.html',
course=None,
stats={'total_lessons': 0, 'completed_lessons': 0, 'completion_percentage': 0},
continue_watching=continue_watching,
recent_views=recent_views,
recently_added=get_recently_added_courses(),
library_stats=get_library_stats())
library_stats=format_library_stats(scan),
stale_courses=format_stale_courses(scan),
activity_heatmap=format_activity_heatmap(scan),
next_up=get_next_up_courses())
# Apply progress data to tree
ProgressTracker.apply_progress_to_tree(current_course)
@@ -1374,7 +1672,9 @@ def index():
course=current_course,
stats=stats,
continue_watching=continue_watching,
recent_views=recent_views)
recent_views=recent_views,
has_note=bool(course_has_any_notes(current_course)),
is_queued=os.path.abspath(current_course.path) in get_next_up_paths())
@app.route('/browse')
@@ -1865,6 +2165,64 @@ def help_page():
return render_template('help.html')
@app.route('/notes')
def notes_hub():
"""Every lesson note across the whole library in one place."""
return render_template('notes_hub.html', notes=get_all_notes())
@app.route('/library/search-transcripts')
def search_transcripts_api():
"""Search inside lesson subtitle files across the library."""
query = request.args.get('q', '').strip()
if not query:
return jsonify({'results': []})
return jsonify({'results': search_transcripts(query)})
@app.route('/course/study-guide')
def download_study_guide():
"""Download every note written for the currently loaded course as one markdown file."""
global current_course
if not current_course:
return "No course loaded", 404
markdown = build_study_guide_markdown(current_course)
buffer = io.BytesIO(markdown.encode('utf-8'))
safe_name = re.sub(r'[^\w\-. ]', '_', current_course.name).strip() or 'course'
return send_file(buffer, mimetype='text/markdown', as_attachment=True,
download_name=f"{safe_name} - Study Guide.md")
@app.route('/api/next-up', methods=['POST'])
def set_next_up_api():
"""Add or remove a course from the Next Up queue."""
data = request.json or {}
path = data.get('path', '')
queued = bool(data.get('queued', True))
if not path:
return jsonify({'error': 'path is required'}), 400
library_root = os.path.abspath(get_library_root())
target = os.path.abspath(path)
if not (target == library_root or target.startswith(library_root + os.sep)):
return jsonify({'error': 'Path outside library root'}), 403
updated = set_path_queued(target, queued)
return jsonify({'success': True, 'next_up': updated})
@app.route('/api/next-up/reorder', methods=['POST'])
def reorder_next_up_api():
"""Replace the Next Up order wholesale, from the client's up/down-reordered list."""
data = request.json or {}
paths = data.get('paths')
if not isinstance(paths, list):
return jsonify({'error': 'paths must be a list'}), 400
updated = reorder_next_up(paths)
return jsonify({'success': True, 'next_up': updated})
@app.route('/lesson/<path:lesson_path>')
def view_lesson(lesson_path: str):
"""View specific lesson by path"""