feat: YouTube Music audio capture (ytm: shorthand, Spotify detection, stalled job recovery) (#19)

* feat: add YouTube Music and Spotify source detection - Add Source variants: YouTubeMusicTrack, YouTubeMusicPlaylist, SpotifyTrack, SpotifyAlbum, SpotifyPlaylist - ytm:ID shorthand → music.youtube.com/watch?v=ID (audio-only, forced in core regardless of caller quality hint) - ytm:playlist/ID and music.youtube.com/playlist URLs detected but fail with 'not yet implemented' via fail_run - Spotify URLs/shorthands detected and fail fast with clear DRM error via fail_run (after run item created, so status is visible in /runs) - source_metadata: youtube_music/music/audio and spotify/music/audio (entity_kind='music' for UI pill, representation_kind='audio' stored) - locator_to_ytdlp_url includes YouTubeMusicTrack for probe endpoint - generate_entry_title: 'Title — Artist' for YTM tracks - Frontend: isVideoSource handles ytm: and music.youtube.com/watch; Spotify returns false (no probe, clear server error on submit) - Placeholder updated to include ytm:ID - SOURCE_ICONS: youtube_music (red disc) and spotify (green waves) - 14 new tests covering all new sources (163 total, all pass) * fix: prevent yt-dlp playlist expansion and stalled run recovery - Add --no-playlist to ytdlp::download and fetch_metadata: URLs with a list= parameter (e.g. music.youtube.com/watch?v=ID&list=RDAMVM…) no longer cause yt-dlp to expand the full playlist and hang; both the metadata probe and the download are now single-item only - Fix fail_stalled_capture_jobs to also recover archive_runs and archive_run_items: capture_jobs.run_uid is NULL at crash time so a join is unreliable; instead fail all archive_runs/items still in_progress directly, then recount failed_count via subquery. Startup recovery now makes the Runs UI reflect the correct failed state after a hard shutdown - Expand fail_stalled_jobs_on_restart test to assert archive_run and archive_run_item rows are also marked failed, not just capture_jobs * fix: use play triangle for youtube_music icon
2026-07-21 18:55:36 +02:00 · 2026-07-06 15:34:14 +02:00 · 2026-07-06 15:34:14 +02:00 · 21b11c211f
commit 21b11c211f
parent 339076e6a2
7 changed files with 362 additions and 7 deletions
--- a/crates/archivr-core/src/capture.rs
+++ b/crates/archivr-core/src/capture.rs
@ -14,6 +14,11 @@ pub enum Source {
    YouTubeVideo,
    YouTubePlaylist,
    YouTubeChannel,
+    YouTubeMusicTrack,
+    YouTubeMusicPlaylist,
+    SpotifyTrack,
+    SpotifyAlbum,
+    SpotifyPlaylist,
    X,
    Tweet,
    TweetThread,
@ -72,6 +77,19 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
            "Archival of {}",
            meta.author.as_deref().unwrap_or("Unknown Channel")
        ),
+        Source::YouTubeMusicTrack => {
+            let title = meta.title.as_deref().unwrap_or("Unknown Track");
+            match meta.author.as_deref() {
+                Some(a) => format!("{title} \u{2014} {a}"),
+                None => title.to_string(),
+            }
+        }
+        Source::YouTubeMusicPlaylist => {
+            meta.title.clone().unwrap_or_else(|| "YouTube Music Playlist".to_string())
+        }
+        Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist => {
+            meta.title.clone().unwrap_or_else(|| "Spotify Content".to_string())
+        }
        Source::X => format!("X Media by {}", meta.author.as_deref().unwrap_or("unknown")),
        Source::Tweet => {
            let excerpt = meta.caption_excerpt().unwrap_or_else(|| "Tweet".to_string());
@ -124,6 +142,32 @@ fn expand_shorthand_to_url(path: &str, source: &Source) -> String {
        }
    }

+    // YouTube Music shorthands: ytm:ID (track) or ytm:playlist/ID
+    if matches!(source, Source::YouTubeMusicTrack | Source::YouTubeMusicPlaylist) {
+        if let Some(after) = path.strip_prefix("ytm:") {
+            if let Some(id) = after.strip_prefix("playlist/") {
+                return format!("https://music.youtube.com/playlist?list={id}");
+            }
+            // bare ytm:ID → track
+            return format!("https://music.youtube.com/watch?v={after}");
+        }
+    }
+
+    // Spotify shorthands: spotify:track:ID, spotify:album:ID, spotify:playlist:ID
+    if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
+        if let Some(after) = path.strip_prefix("spotify:") {
+            if let Some(id) = after.strip_prefix("track:") {
+                return format!("https://open.spotify.com/track/{id}");
+            }
+            if let Some(id) = after.strip_prefix("album:") {
+                return format!("https://open.spotify.com/album/{id}");
+            }
+            if let Some(id) = after.strip_prefix("playlist:") {
+                return format!("https://open.spotify.com/playlist/{id}");
+            }
+        }
+    }
+
    if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) {
        if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) {
            return format!("https://x.com/i/status/{tweet_id}");
@ -187,6 +231,27 @@ fn determine_source(path: &str) -> Source {
        }
    }

+    // Shorthand scheme: ytm:
+    if let Some(after_scheme) = path.strip_prefix("ytm:") {
+        if after_scheme.starts_with("playlist/") {
+            return Source::YouTubeMusicPlaylist;
+        }
+        // Any other suffix is treated as a track ID
+        return Source::YouTubeMusicTrack;
+    }
+
+    // Shorthand scheme: spotify:
+    if let Some(after_scheme) = path.strip_prefix("spotify:") {
+        if after_scheme.starts_with("album:") {
+            return Source::SpotifyAlbum;
+        }
+        if after_scheme.starts_with("playlist:") {
+            return Source::SpotifyPlaylist;
+        }
+        // track: or anything else maps to a track
+        return Source::SpotifyTrack;
+    }
+
    // Shorthand schemes: tweet:, x:, or twitter:
    if let Some(after_scheme) = path
        .strip_prefix("x:")
@ -275,6 +340,37 @@ fn determine_source(path: &str) -> Source {
            return Source::YouTubeChannel;
        }

+        // YouTube Music track URLs: music.youtube.com/watch?v=ID
+        if path.starts_with("https://music.youtube.com/watch")
+            || path.starts_with("http://music.youtube.com/watch")
+        {
+            return Source::YouTubeMusicTrack;
+        }
+
+        // YouTube Music playlist URLs: music.youtube.com/playlist?list=ID
+        if path.starts_with("https://music.youtube.com/playlist")
+            || path.starts_with("http://music.youtube.com/playlist")
+        {
+            return Source::YouTubeMusicPlaylist;
+        }
+
+        // Spotify URLs: open.spotify.com/{track,album,playlist}/ID
+        if path.starts_with("https://open.spotify.com/track/")
+            || path.starts_with("http://open.spotify.com/track/")
+        {
+            return Source::SpotifyTrack;
+        }
+        if path.starts_with("https://open.spotify.com/album/")
+            || path.starts_with("http://open.spotify.com/album/")
+        {
+            return Source::SpotifyAlbum;
+        }
+        if path.starts_with("https://open.spotify.com/playlist/")
+            || path.starts_with("http://open.spotify.com/playlist/")
+        {
+            return Source::SpotifyPlaylist;
+        }
+
        if path.starts_with("https://x.com/") {
            return Source::X;
        }
@ -341,6 +437,7 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
    let source = determine_source(locator);
    match source {
        Source::YouTubeVideo
+        | Source::YouTubeMusicTrack
        | Source::X
        | Source::Instagram
        | Source::Facebook
@ -426,6 +523,11 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
        Source::YouTubeVideo => ("youtube", "video", "video"),
        Source::YouTubePlaylist => ("youtube", "playlist", "container"),
        Source::YouTubeChannel => ("youtube", "channel", "container"),
+        Source::YouTubeMusicTrack => ("youtube_music", "music", "audio"),
+        Source::YouTubeMusicPlaylist => ("youtube_music", "playlist", "container"),
+        Source::SpotifyTrack => ("spotify", "music", "audio"),
+        Source::SpotifyAlbum => ("spotify", "album", "container"),
+        Source::SpotifyPlaylist => ("spotify", "playlist", "container"),
        Source::X => ("x", "post", "video"),
        Source::Tweet => ("x", "tweet", "tweet_json"),
        Source::TweetThread => ("x", "tweet_thread", "tweet_json"),
@ -745,6 +847,27 @@ pub fn perform_capture(
        ));
    }

+    // Sources: Spotify — not downloadable; Spotify audio is DRM-protected.
+    if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
+        return Err(fail_run(
+            &conn,
+            &run,
+            &item,
+            "Spotify downloads are not supported: Spotify audio is DRM-protected and cannot \
+             be downloaded by yt-dlp. Archive the equivalent YouTube Music track instead.",
+        ));
+    }
+
+    // Sources: YouTube Music Playlist — container expansion not yet implemented.
+    if source == Source::YouTubeMusicPlaylist {
+        return Err(fail_run(
+            &conn,
+            &run,
+            &item,
+            "YouTube Music playlist archiving is not yet implemented.",
+        ));
+    }
+
    // Source: generic HTTP/S file URL
    if source == Source::Url {
        match downloader::http::download(locator, store_path, &timestamp) {
@ -1002,6 +1125,7 @@ pub fn perform_capture(
    // because --dump-json is a simulate flag that suppresses the download.
    let ytdlp_metadata_json: Option<String> = match source {
        Source::YouTubeVideo
+        | Source::YouTubeMusicTrack
        | Source::X
        | Source::Instagram
        | Source::Facebook
@ -1042,6 +1166,20 @@ pub fn perform_capture(
                }
            }
        }
+        Source::YouTubeMusicTrack => {
+            // Music tracks are always audio-only regardless of the caller's quality hint.
+            match downloader::ytdlp::download(path.clone(), store_path, &timestamp, Some("audio")) {
+                Ok(result) => result,
+                Err(e) => {
+                    return Err(fail_run(
+                        &conn,
+                        &run,
+                        &item,
+                        &format!("Failed to download audio: {e}"),
+                    ));
+                }
+            }
+        }
        Source::Local => {
            match downloader::local::save(path.clone(), store_path, &timestamp) {
                Ok(h) => (h, local_file_extension(&path)),
@ -1412,6 +1550,142 @@ mod tests {
        }
    }

+    #[test]
+    fn test_youtube_music_sources() {
+        // --- determine_source ---
+        let cases = [
+            TestCase {
+                url: "https://music.youtube.com/watch?v=MntbN1DdEP0",
+                expected: Source::YouTubeMusicTrack,
+            },
+            TestCase {
+                url: "http://music.youtube.com/watch?v=MntbN1DdEP0",
+                expected: Source::YouTubeMusicTrack,
+            },
+            TestCase {
+                url: "https://music.youtube.com/playlist?list=PLtest123",
+                expected: Source::YouTubeMusicPlaylist,
+            },
+            TestCase {
+                url: "ytm:MntbN1DdEP0",
+                expected: Source::YouTubeMusicTrack,
+            },
+            TestCase {
+                url: "ytm:playlist/PLtest123",
+                expected: Source::YouTubeMusicPlaylist,
+            },
+        ];
+        for case in &cases {
+            assert_eq!(
+                determine_source(case.url),
+                case.expected,
+                "Failed for URL: {}",
+                case.url
+            );
+        }
+
+        // --- expand_shorthand_to_url ---
+        assert_eq!(
+            expand_shorthand_to_url("ytm:MntbN1DdEP0", &Source::YouTubeMusicTrack),
+            "https://music.youtube.com/watch?v=MntbN1DdEP0"
+        );
+        assert_eq!(
+            expand_shorthand_to_url("ytm:playlist/PLtest123", &Source::YouTubeMusicPlaylist),
+            "https://music.youtube.com/playlist?list=PLtest123"
+        );
+        // Full URL passes through unchanged
+        assert_eq!(
+            expand_shorthand_to_url(
+                "https://music.youtube.com/watch?v=MntbN1DdEP0",
+                &Source::YouTubeMusicTrack
+            ),
+            "https://music.youtube.com/watch?v=MntbN1DdEP0"
+        );
+
+        // --- locator_to_ytdlp_url ---
+        assert_eq!(
+            locator_to_ytdlp_url("ytm:MntbN1DdEP0"),
+            Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
+        );
+        assert_eq!(
+            locator_to_ytdlp_url("https://music.youtube.com/watch?v=MntbN1DdEP0"),
+            Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
+        );
+        // Playlist is not exposed to the probe endpoint
+        assert_eq!(locator_to_ytdlp_url("ytm:playlist/PLtest123"), None);
+
+        // --- source_metadata ---
+        assert_eq!(
+            source_metadata(Source::YouTubeMusicTrack),
+            ("youtube_music", "music", "audio")
+        );
+        assert_eq!(
+            source_metadata(Source::YouTubeMusicPlaylist),
+            ("youtube_music", "playlist", "container")
+        );
+    }
+
+    #[test]
+    fn test_spotify_sources() {
+        // --- determine_source ---
+        let cases = [
+            TestCase {
+                url: "https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh",
+                expected: Source::SpotifyTrack,
+            },
+            TestCase {
+                url: "https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3",
+                expected: Source::SpotifyAlbum,
+            },
+            TestCase {
+                url: "https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M",
+                expected: Source::SpotifyPlaylist,
+            },
+            TestCase {
+                url: "spotify:track:4iV5W9uYEdYUVa79Axb7Rh",
+                expected: Source::SpotifyTrack,
+            },
+            TestCase {
+                url: "spotify:album:1DFixLWuPkv3KT3TnV35m3",
+                expected: Source::SpotifyAlbum,
+            },
+            TestCase {
+                url: "spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
+                expected: Source::SpotifyPlaylist,
+            },
+        ];
+        for case in &cases {
+            assert_eq!(
+                determine_source(case.url),
+                case.expected,
+                "Failed for URL: {}",
+                case.url
+            );
+        }
+
+        // --- expand_shorthand_to_url ---
+        assert_eq!(
+            expand_shorthand_to_url("spotify:track:4iV5W9uYEdYUVa79Axb7Rh", &Source::SpotifyTrack),
+            "https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh"
+        );
+        assert_eq!(
+            expand_shorthand_to_url("spotify:album:1DFixLWuPkv3KT3TnV35m3", &Source::SpotifyAlbum),
+            "https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3"
+        );
+        assert_eq!(
+            expand_shorthand_to_url(
+                "spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
+                &Source::SpotifyPlaylist
+            ),
+            "https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M"
+        );
+
+        // --- source_metadata ---
+        assert_eq!(source_metadata(Source::SpotifyTrack), ("spotify", "music", "audio"));
+        assert_eq!(source_metadata(Source::SpotifyAlbum), ("spotify", "album", "container"));
+        assert_eq!(source_metadata(Source::SpotifyPlaylist), ("spotify", "playlist", "container"));
+    }
+
    #[test]
    fn test_x_sources() {
        let x_cases = [
--- a/crates/archivr-core/src/database.rs
+++ b/crates/archivr-core/src/database.rs
@ -1082,10 +1082,40 @@ pub fn get_capture_job(conn: &Connection, job_uid: &str) -> Result<Option<Captur
    .map_err(Into::into)
 }

-/// Marks all 'running' capture jobs as 'failed' with a restart message.
-/// Called at server startup to clean up jobs interrupted by a previous shutdown.
+/// Marks all interrupted capture jobs, runs, and run items as failed.
+/// Called at server startup to recover from a hard shutdown mid-capture.
+///
+/// `capture_jobs.run_uid` is NULL when the server crashes before `perform_capture`
+/// returns, so we cannot join; instead we fail every `archive_runs` row still
+/// `in_progress` directly — any run that survived shutdown unfinished was interrupted.
+///
+/// Returns the number of `capture_jobs` rows updated (used for the startup log).
 pub fn fail_stalled_capture_jobs(conn: &Connection) -> Result<usize> {
    let now = now_timestamp();
+
+    // 1. Fail in-progress run items.
+    conn.execute(
+        "UPDATE archive_run_items
+         SET status = 'failed', error_text = 'interrupted by server restart'
+         WHERE status = 'in_progress'",
+        [],
+    )?;
+
+    // 2. Fail in-progress archive runs; recount failed items from the updated rows.
+    conn.execute(
+        "UPDATE archive_runs
+         SET status     = 'failed',
+             finished_at = ?1,
+             failed_count = (
+                 SELECT COUNT(*) FROM archive_run_items
+                 WHERE run_id = archive_runs.id AND status = 'failed'
+             ),
+             error_summary = 'interrupted by server restart'
+         WHERE status = 'in_progress'",
+        [now.clone()],
+    )?;
+
+    // 3. Fail running capture jobs (the polling layer).
    let n = conn.execute(
        "UPDATE capture_jobs SET status = 'failed',
         error_text = 'interrupted by server restart',
@ -1093,6 +1123,7 @@ pub fn fail_stalled_capture_jobs(conn: &Connection) -> Result<usize> {
         WHERE status = 'running'",
        [now],
    )?;
+
    Ok(n)
 }

@ -2686,13 +2717,40 @@ mod tests {
    #[test]
    fn fail_stalled_jobs_on_restart() {
        let conn = conn();
+
+        // Simulate an in-progress capture_job (run_uid still NULL — common crash case).
        let uid = create_capture_job(&conn, "test").unwrap();
        update_capture_job_status(&conn, &uid, "running", None, None).unwrap();
+
+        // Simulate an in-progress archive_run and item with no associated capture_job
+        // (covers the case where run_uid was never written back before the crash).
+        let user_id = ensure_default_user(&conn).unwrap();
+        let run = create_archive_run(&conn, user_id, 1).unwrap();
+        create_archive_run_item(&conn, run.id, None, 0, "https://example.com", None, "web", "file").unwrap();
+
        let n = fail_stalled_capture_jobs(&conn).unwrap();
-        assert_eq!(n, 1);
+        assert_eq!(n, 1); // one capture_job updated
+
+        // capture_job is failed
        let job = get_capture_job(&conn, &uid).unwrap().unwrap();
        assert_eq!(job.status, "failed");
        assert!(job.error_text.as_deref().unwrap().contains("interrupted"));
+
+        // archive_run is failed
+        let updated_run: String = conn.query_row(
+            "SELECT status FROM archive_runs WHERE id = ?1",
+            [run.id],
+            |r| r.get(0),
+        ).unwrap();
+        assert_eq!(updated_run, "failed");
+
+        // archive_run_item is failed
+        let item_status: String = conn.query_row(
+            "SELECT status FROM archive_run_items WHERE run_id = ?1",
+            [run.id],
+            |r| r.get(0),
+        ).unwrap();
+        assert_eq!(item_status, "failed");
    }

    fn make_auth_conn_for_mgmt() -> Connection {
--- a/crates/archivr-core/src/downloader/ytdlp.rs
+++ b/crates/archivr-core/src/downloader/ytdlp.rs
@ -109,7 +109,12 @@ pub fn download(
    let out_template = temp_dir.join(format!("{timestamp}.%(ext)s"));

    let mut cmd = Command::new(&ytdlp);
-    cmd.arg(&path).arg("-f").arg(quality_format(quality));
+    cmd.arg(&path)
+        .arg("-f").arg(quality_format(quality))
+        // This function is only called for single-item sources; --no-playlist
+        // prevents yt-dlp from expanding a list= query parameter into a full
+        // playlist download (e.g. music.youtube.com/watch?v=ID&list=RDAMVM…).
+        .arg("--no-playlist");
    if is_audio {
        // -x guarantees audio-only even when /best falls back to a combined
        // A/V format. No --audio-format → native remux only, no re-encode.
@ -166,6 +171,10 @@ pub fn fetch_metadata(path: &str) -> Option<String> {

    let out = std::process::Command::new(&ytdlp)
        .arg("--dump-json")
+        // Same rationale as download(): only called for single-item sources;
+        // prevents --dump-json from emitting one JSON object per playlist item
+        // when the URL contains a list= parameter.
+        .arg("--no-playlist")
        .arg(path)
        .output()
        .ok()?;
--- a/crates/archivr-server/static/assets/index-D2gm2yNL.js
+++ b/crates/archivr-server/static/assets/index-D2gm2yNL.js
--- a/crates/archivr-server/static/index.html
+++ b/crates/archivr-server/static/index.html
@ -4,7 +4,7 @@
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>Archivr</title>
-    <script type="module" crossorigin src="/assets/index-bXPw5mQ7.js"></script>
+    <script type="module" crossorigin src="/assets/index-D2gm2yNL.js"></script>
    <link rel="stylesheet" crossorigin href="/assets/index-BW0QKHXE.css">
  </head>
  <body>