mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat: YouTube Music audio capture (ytm: shorthand, Spotify detection, stalled job recovery) (#19)
* feat: add YouTube Music and Spotify source detection - Add Source variants: YouTubeMusicTrack, YouTubeMusicPlaylist, SpotifyTrack, SpotifyAlbum, SpotifyPlaylist - ytm:ID shorthand → music.youtube.com/watch?v=ID (audio-only, forced in core regardless of caller quality hint) - ytm:playlist/ID and music.youtube.com/playlist URLs detected but fail with 'not yet implemented' via fail_run - Spotify URLs/shorthands detected and fail fast with clear DRM error via fail_run (after run item created, so status is visible in /runs) - source_metadata: youtube_music/music/audio and spotify/music/audio (entity_kind='music' for UI pill, representation_kind='audio' stored) - locator_to_ytdlp_url includes YouTubeMusicTrack for probe endpoint - generate_entry_title: 'Title — Artist' for YTM tracks - Frontend: isVideoSource handles ytm: and music.youtube.com/watch; Spotify returns false (no probe, clear server error on submit) - Placeholder updated to include ytm:ID - SOURCE_ICONS: youtube_music (red disc) and spotify (green waves) - 14 new tests covering all new sources (163 total, all pass) * fix: prevent yt-dlp playlist expansion and stalled run recovery - Add --no-playlist to ytdlp::download and fetch_metadata: URLs with a list= parameter (e.g. music.youtube.com/watch?v=ID&list=RDAMVM…) no longer cause yt-dlp to expand the full playlist and hang; both the metadata probe and the download are now single-item only - Fix fail_stalled_capture_jobs to also recover archive_runs and archive_run_items: capture_jobs.run_uid is NULL at crash time so a join is unreliable; instead fail all archive_runs/items still in_progress directly, then recount failed_count via subquery. Startup recovery now makes the Runs UI reflect the correct failed state after a hard shutdown - Expand fail_stalled_jobs_on_restart test to assert archive_run and archive_run_item rows are also marked failed, not just capture_jobs * fix: use play triangle for youtube_music icon
This commit is contained in:
GitHub
parent
339076e6a2
commit
21b11c211f
7 changed files with 362 additions and 7 deletions
|
|
@ -14,6 +14,11 @@ pub enum Source {
|
|||
YouTubeVideo,
|
||||
YouTubePlaylist,
|
||||
YouTubeChannel,
|
||||
YouTubeMusicTrack,
|
||||
YouTubeMusicPlaylist,
|
||||
SpotifyTrack,
|
||||
SpotifyAlbum,
|
||||
SpotifyPlaylist,
|
||||
X,
|
||||
Tweet,
|
||||
TweetThread,
|
||||
|
|
@ -72,6 +77,19 @@ fn generate_entry_title(source: Source, meta: &PlatformMetadata) -> String {
|
|||
"Archival of {}",
|
||||
meta.author.as_deref().unwrap_or("Unknown Channel")
|
||||
),
|
||||
Source::YouTubeMusicTrack => {
|
||||
let title = meta.title.as_deref().unwrap_or("Unknown Track");
|
||||
match meta.author.as_deref() {
|
||||
Some(a) => format!("{title} \u{2014} {a}"),
|
||||
None => title.to_string(),
|
||||
}
|
||||
}
|
||||
Source::YouTubeMusicPlaylist => {
|
||||
meta.title.clone().unwrap_or_else(|| "YouTube Music Playlist".to_string())
|
||||
}
|
||||
Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist => {
|
||||
meta.title.clone().unwrap_or_else(|| "Spotify Content".to_string())
|
||||
}
|
||||
Source::X => format!("X Media by {}", meta.author.as_deref().unwrap_or("unknown")),
|
||||
Source::Tweet => {
|
||||
let excerpt = meta.caption_excerpt().unwrap_or_else(|| "Tweet".to_string());
|
||||
|
|
@ -124,6 +142,32 @@ fn expand_shorthand_to_url(path: &str, source: &Source) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
// YouTube Music shorthands: ytm:ID (track) or ytm:playlist/ID
|
||||
if matches!(source, Source::YouTubeMusicTrack | Source::YouTubeMusicPlaylist) {
|
||||
if let Some(after) = path.strip_prefix("ytm:") {
|
||||
if let Some(id) = after.strip_prefix("playlist/") {
|
||||
return format!("https://music.youtube.com/playlist?list={id}");
|
||||
}
|
||||
// bare ytm:ID → track
|
||||
return format!("https://music.youtube.com/watch?v={after}");
|
||||
}
|
||||
}
|
||||
|
||||
// Spotify shorthands: spotify:track:ID, spotify:album:ID, spotify:playlist:ID
|
||||
if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
|
||||
if let Some(after) = path.strip_prefix("spotify:") {
|
||||
if let Some(id) = after.strip_prefix("track:") {
|
||||
return format!("https://open.spotify.com/track/{id}");
|
||||
}
|
||||
if let Some(id) = after.strip_prefix("album:") {
|
||||
return format!("https://open.spotify.com/album/{id}");
|
||||
}
|
||||
if let Some(id) = after.strip_prefix("playlist:") {
|
||||
return format!("https://open.spotify.com/playlist/{id}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if *source == Source::X && (path.starts_with("tweet:media:") || path.starts_with("x:media:")) {
|
||||
if let Some(tweet_id) = path.split(':').next_back().and_then(parse_tweet_id) {
|
||||
return format!("https://x.com/i/status/{tweet_id}");
|
||||
|
|
@ -187,6 +231,27 @@ fn determine_source(path: &str) -> Source {
|
|||
}
|
||||
}
|
||||
|
||||
// Shorthand scheme: ytm:
|
||||
if let Some(after_scheme) = path.strip_prefix("ytm:") {
|
||||
if after_scheme.starts_with("playlist/") {
|
||||
return Source::YouTubeMusicPlaylist;
|
||||
}
|
||||
// Any other suffix is treated as a track ID
|
||||
return Source::YouTubeMusicTrack;
|
||||
}
|
||||
|
||||
// Shorthand scheme: spotify:
|
||||
if let Some(after_scheme) = path.strip_prefix("spotify:") {
|
||||
if after_scheme.starts_with("album:") {
|
||||
return Source::SpotifyAlbum;
|
||||
}
|
||||
if after_scheme.starts_with("playlist:") {
|
||||
return Source::SpotifyPlaylist;
|
||||
}
|
||||
// track: or anything else maps to a track
|
||||
return Source::SpotifyTrack;
|
||||
}
|
||||
|
||||
// Shorthand schemes: tweet:, x:, or twitter:
|
||||
if let Some(after_scheme) = path
|
||||
.strip_prefix("x:")
|
||||
|
|
@ -275,6 +340,37 @@ fn determine_source(path: &str) -> Source {
|
|||
return Source::YouTubeChannel;
|
||||
}
|
||||
|
||||
// YouTube Music track URLs: music.youtube.com/watch?v=ID
|
||||
if path.starts_with("https://music.youtube.com/watch")
|
||||
|| path.starts_with("http://music.youtube.com/watch")
|
||||
{
|
||||
return Source::YouTubeMusicTrack;
|
||||
}
|
||||
|
||||
// YouTube Music playlist URLs: music.youtube.com/playlist?list=ID
|
||||
if path.starts_with("https://music.youtube.com/playlist")
|
||||
|| path.starts_with("http://music.youtube.com/playlist")
|
||||
{
|
||||
return Source::YouTubeMusicPlaylist;
|
||||
}
|
||||
|
||||
// Spotify URLs: open.spotify.com/{track,album,playlist}/ID
|
||||
if path.starts_with("https://open.spotify.com/track/")
|
||||
|| path.starts_with("http://open.spotify.com/track/")
|
||||
{
|
||||
return Source::SpotifyTrack;
|
||||
}
|
||||
if path.starts_with("https://open.spotify.com/album/")
|
||||
|| path.starts_with("http://open.spotify.com/album/")
|
||||
{
|
||||
return Source::SpotifyAlbum;
|
||||
}
|
||||
if path.starts_with("https://open.spotify.com/playlist/")
|
||||
|| path.starts_with("http://open.spotify.com/playlist/")
|
||||
{
|
||||
return Source::SpotifyPlaylist;
|
||||
}
|
||||
|
||||
if path.starts_with("https://x.com/") {
|
||||
return Source::X;
|
||||
}
|
||||
|
|
@ -341,6 +437,7 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
|
|||
let source = determine_source(locator);
|
||||
match source {
|
||||
Source::YouTubeVideo
|
||||
| Source::YouTubeMusicTrack
|
||||
| Source::X
|
||||
| Source::Instagram
|
||||
| Source::Facebook
|
||||
|
|
@ -426,6 +523,11 @@ fn source_metadata(source: Source) -> (&'static str, &'static str, &'static str)
|
|||
Source::YouTubeVideo => ("youtube", "video", "video"),
|
||||
Source::YouTubePlaylist => ("youtube", "playlist", "container"),
|
||||
Source::YouTubeChannel => ("youtube", "channel", "container"),
|
||||
Source::YouTubeMusicTrack => ("youtube_music", "music", "audio"),
|
||||
Source::YouTubeMusicPlaylist => ("youtube_music", "playlist", "container"),
|
||||
Source::SpotifyTrack => ("spotify", "music", "audio"),
|
||||
Source::SpotifyAlbum => ("spotify", "album", "container"),
|
||||
Source::SpotifyPlaylist => ("spotify", "playlist", "container"),
|
||||
Source::X => ("x", "post", "video"),
|
||||
Source::Tweet => ("x", "tweet", "tweet_json"),
|
||||
Source::TweetThread => ("x", "tweet_thread", "tweet_json"),
|
||||
|
|
@ -745,6 +847,27 @@ pub fn perform_capture(
|
|||
));
|
||||
}
|
||||
|
||||
// Sources: Spotify — not downloadable; Spotify audio is DRM-protected.
|
||||
if matches!(source, Source::SpotifyTrack | Source::SpotifyAlbum | Source::SpotifyPlaylist) {
|
||||
return Err(fail_run(
|
||||
&conn,
|
||||
&run,
|
||||
&item,
|
||||
"Spotify downloads are not supported: Spotify audio is DRM-protected and cannot \
|
||||
be downloaded by yt-dlp. Archive the equivalent YouTube Music track instead.",
|
||||
));
|
||||
}
|
||||
|
||||
// Sources: YouTube Music Playlist — container expansion not yet implemented.
|
||||
if source == Source::YouTubeMusicPlaylist {
|
||||
return Err(fail_run(
|
||||
&conn,
|
||||
&run,
|
||||
&item,
|
||||
"YouTube Music playlist archiving is not yet implemented.",
|
||||
));
|
||||
}
|
||||
|
||||
// Source: generic HTTP/S file URL
|
||||
if source == Source::Url {
|
||||
match downloader::http::download(locator, store_path, ×tamp) {
|
||||
|
|
@ -1002,6 +1125,7 @@ pub fn perform_capture(
|
|||
// because --dump-json is a simulate flag that suppresses the download.
|
||||
let ytdlp_metadata_json: Option<String> = match source {
|
||||
Source::YouTubeVideo
|
||||
| Source::YouTubeMusicTrack
|
||||
| Source::X
|
||||
| Source::Instagram
|
||||
| Source::Facebook
|
||||
|
|
@ -1042,6 +1166,20 @@ pub fn perform_capture(
|
|||
}
|
||||
}
|
||||
}
|
||||
Source::YouTubeMusicTrack => {
|
||||
// Music tracks are always audio-only regardless of the caller's quality hint.
|
||||
match downloader::ytdlp::download(path.clone(), store_path, ×tamp, Some("audio")) {
|
||||
Ok(result) => result,
|
||||
Err(e) => {
|
||||
return Err(fail_run(
|
||||
&conn,
|
||||
&run,
|
||||
&item,
|
||||
&format!("Failed to download audio: {e}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Source::Local => {
|
||||
match downloader::local::save(path.clone(), store_path, ×tamp) {
|
||||
Ok(h) => (h, local_file_extension(&path)),
|
||||
|
|
@ -1412,6 +1550,142 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_youtube_music_sources() {
|
||||
// --- determine_source ---
|
||||
let cases = [
|
||||
TestCase {
|
||||
url: "https://music.youtube.com/watch?v=MntbN1DdEP0",
|
||||
expected: Source::YouTubeMusicTrack,
|
||||
},
|
||||
TestCase {
|
||||
url: "http://music.youtube.com/watch?v=MntbN1DdEP0",
|
||||
expected: Source::YouTubeMusicTrack,
|
||||
},
|
||||
TestCase {
|
||||
url: "https://music.youtube.com/playlist?list=PLtest123",
|
||||
expected: Source::YouTubeMusicPlaylist,
|
||||
},
|
||||
TestCase {
|
||||
url: "ytm:MntbN1DdEP0",
|
||||
expected: Source::YouTubeMusicTrack,
|
||||
},
|
||||
TestCase {
|
||||
url: "ytm:playlist/PLtest123",
|
||||
expected: Source::YouTubeMusicPlaylist,
|
||||
},
|
||||
];
|
||||
for case in &cases {
|
||||
assert_eq!(
|
||||
determine_source(case.url),
|
||||
case.expected,
|
||||
"Failed for URL: {}",
|
||||
case.url
|
||||
);
|
||||
}
|
||||
|
||||
// --- expand_shorthand_to_url ---
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url("ytm:MntbN1DdEP0", &Source::YouTubeMusicTrack),
|
||||
"https://music.youtube.com/watch?v=MntbN1DdEP0"
|
||||
);
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url("ytm:playlist/PLtest123", &Source::YouTubeMusicPlaylist),
|
||||
"https://music.youtube.com/playlist?list=PLtest123"
|
||||
);
|
||||
// Full URL passes through unchanged
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url(
|
||||
"https://music.youtube.com/watch?v=MntbN1DdEP0",
|
||||
&Source::YouTubeMusicTrack
|
||||
),
|
||||
"https://music.youtube.com/watch?v=MntbN1DdEP0"
|
||||
);
|
||||
|
||||
// --- locator_to_ytdlp_url ---
|
||||
assert_eq!(
|
||||
locator_to_ytdlp_url("ytm:MntbN1DdEP0"),
|
||||
Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
locator_to_ytdlp_url("https://music.youtube.com/watch?v=MntbN1DdEP0"),
|
||||
Some("https://music.youtube.com/watch?v=MntbN1DdEP0".to_string())
|
||||
);
|
||||
// Playlist is not exposed to the probe endpoint
|
||||
assert_eq!(locator_to_ytdlp_url("ytm:playlist/PLtest123"), None);
|
||||
|
||||
// --- source_metadata ---
|
||||
assert_eq!(
|
||||
source_metadata(Source::YouTubeMusicTrack),
|
||||
("youtube_music", "music", "audio")
|
||||
);
|
||||
assert_eq!(
|
||||
source_metadata(Source::YouTubeMusicPlaylist),
|
||||
("youtube_music", "playlist", "container")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_spotify_sources() {
|
||||
// --- determine_source ---
|
||||
let cases = [
|
||||
TestCase {
|
||||
url: "https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh",
|
||||
expected: Source::SpotifyTrack,
|
||||
},
|
||||
TestCase {
|
||||
url: "https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3",
|
||||
expected: Source::SpotifyAlbum,
|
||||
},
|
||||
TestCase {
|
||||
url: "https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M",
|
||||
expected: Source::SpotifyPlaylist,
|
||||
},
|
||||
TestCase {
|
||||
url: "spotify:track:4iV5W9uYEdYUVa79Axb7Rh",
|
||||
expected: Source::SpotifyTrack,
|
||||
},
|
||||
TestCase {
|
||||
url: "spotify:album:1DFixLWuPkv3KT3TnV35m3",
|
||||
expected: Source::SpotifyAlbum,
|
||||
},
|
||||
TestCase {
|
||||
url: "spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
|
||||
expected: Source::SpotifyPlaylist,
|
||||
},
|
||||
];
|
||||
for case in &cases {
|
||||
assert_eq!(
|
||||
determine_source(case.url),
|
||||
case.expected,
|
||||
"Failed for URL: {}",
|
||||
case.url
|
||||
);
|
||||
}
|
||||
|
||||
// --- expand_shorthand_to_url ---
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url("spotify:track:4iV5W9uYEdYUVa79Axb7Rh", &Source::SpotifyTrack),
|
||||
"https://open.spotify.com/track/4iV5W9uYEdYUVa79Axb7Rh"
|
||||
);
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url("spotify:album:1DFixLWuPkv3KT3TnV35m3", &Source::SpotifyAlbum),
|
||||
"https://open.spotify.com/album/1DFixLWuPkv3KT3TnV35m3"
|
||||
);
|
||||
assert_eq!(
|
||||
expand_shorthand_to_url(
|
||||
"spotify:playlist:37i9dQZF1DXcBWIGoYBM5M",
|
||||
&Source::SpotifyPlaylist
|
||||
),
|
||||
"https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M"
|
||||
);
|
||||
|
||||
// --- source_metadata ---
|
||||
assert_eq!(source_metadata(Source::SpotifyTrack), ("spotify", "music", "audio"));
|
||||
assert_eq!(source_metadata(Source::SpotifyAlbum), ("spotify", "album", "container"));
|
||||
assert_eq!(source_metadata(Source::SpotifyPlaylist), ("spotify", "playlist", "container"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_x_sources() {
|
||||
let x_cases = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue