thegeneralist01/archivr
1
Fork 0
mirror of https://github.com/thegeneralist01/archivr synced 2026-07-21 18:55:36 +02:00
Code Issues Activity

feat(core+frontend): Spotify album/playlist capture via yt-dlp container path

Browse source 
Previously SpotifyAlbum and SpotifyPlaylist hard-failed with a DRM error.
yt-dlp does support Spotify (experimentally), so they now go through the
same probe/container/child path as YouTube playlists.

ytdlp.rs:
- Extract normalize_item_url() helper used by both fetch_playlist_info
  and probe_playlist_qualities; eliminates the duplicate URL-normalization
  blocks and the YouTube-specific fallback_host variable
- Fallback logic is now platform-aware: YouTube/YTM bare IDs → watch URL,
  Spotify → open.spotify.com/track/{id}, unknown → skip with warning

capture.rs:
- locator_to_playlist_url: accept SpotifyAlbum | SpotifyPlaylist so the
  probe-playlist endpoint accepts Spotify album/playlist URLs
- Container branch: add SpotifyAlbum | SpotifyPlaylist to the matches!
- is_audio: true for SpotifyAlbum | SpotifyPlaylist (audio-only, like YTM)
- child_source: SpotifyTrack for Spotify containers (not YouTubeMusicTrack)
- generate_entry_title and record_media_entry both use the correct child
  source so entity_kind, source_kind, and representation_kind are right

CaptureDialog.jsx:
- isPlaylistSource: recognise open.spotify.com/album/ and /playlist/,
  and spotify:album:ID / spotify:playlist:ID shorthands, so Spotify
  containers get the probe UI, per-track excludes, and sync toggle
This commit is contained in:
TheGeneralist 2026-07-21 17:58:55 +02:00
parent 5e7ce4774b
commit 6a7826eba5
Signed by: thegeneralist01
SSH key fingerprint: SHA256:pp9qddbCNmVNoSjevdvQvM5z0DHN7LTa8qBMbcMq/R4
5 changed files with 82 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

51
crates/archivr-core/src/capture.rs
View file

@ -568,9 +568,11 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
pub fn locator_to_playlist_url(locator: &str) -> Option<String> {
let source = determine_source(locator);
match source {
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist => {
Some(expand_shorthand_to_url(locator, &source))
}
Source::YouTubePlaylist
| Source::YouTubeChannel
| Source::YouTubeMusicPlaylist
| Source::SpotifyAlbum
| Source::SpotifyPlaylist => Some(expand_shorthand_to_url(locator, &source)),
_ => None,
}
}
@ -1051,26 +1053,17 @@ pub fn perform_capture(
));
}
// Sources: Spotify — SpotifyTrack attempts yt-dlp and fails gracefully if the
// extractor cannot handle the content. Albums and playlists are not yet supported
// as containers (fetch_playlist_info has YouTube-specific URL fallback logic).
if matches!(source, Source::SpotifyAlbum | Source::SpotifyPlaylist) {
return Err(fail_run(
&conn,
&run,
&item,
"Spotify album/playlist capture is not yet implemented. \
Archive individual tracks instead.",
));
}
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists.
// Fetch container metadata, create a root container entry, then download
// each video/track as a child entry.
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists,
// Spotify albums, Spotify playlists — probe via yt-dlp flat-playlist,
// create a root container entry, then download each track as a child entry.
// `run` and `item` are already created above; `item` acts as the container item.
if matches!(
source,
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist
Source::YouTubePlaylist
| Source::YouTubeChannel
| Source::YouTubeMusicPlaylist
| Source::SpotifyAlbum
| Source::SpotifyPlaylist
) {
// `canonical_url` already holds the expanded URL (same value as `path` later).
let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) {
@ -1149,7 +1142,10 @@ pub fn perform_capture(
(e.id, std::collections::HashSet::new())
};
let is_audio = source == Source::YouTubeMusicPlaylist;
let is_audio = matches!(
source,
Source::YouTubeMusicPlaylist | Source::SpotifyAlbum | Source::SpotifyPlaylist
);
let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality };
let child_entity_kind = if is_audio { "music" } else { "video" };
@ -1199,7 +1195,11 @@ pub fn perform_capture(
Some(json) => {
let meta = downloader::metadata::extract_from_ytdlp_json(json);
Some(generate_entry_title(
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo },
match source {
Source::SpotifyAlbum | Source::SpotifyPlaylist => Source::SpotifyTrack,
_ if is_audio => Source::YouTubeMusicTrack,
_ => Source::YouTubeVideo,
},
&meta,
))
}
@ -1256,8 +1256,11 @@ pub fn perform_capture(
}
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
let child_source =
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo };
let child_source = match source {
Source::SpotifyAlbum | Source::SpotifyPlaylist => Source::SpotifyTrack,
_ if is_audio => Source::YouTubeMusicTrack,
_ => Source::YouTubeVideo,
};
match record_media_entry(
&conn,
store_path,

84
crates/archivr-core/src/downloader/ytdlp.rs
View file

@ -298,6 +298,41 @@ pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<S
if json.trim().is_empty() { None } else { Some(json) }
}
/// Resolves an absolute item URL from a flat-playlist entry JSON object.
///
/// Priority:
/// 1. `webpage_url` — yt-dlp makes this absolute when present.
/// 2. `url` when it is already an absolute HTTP(S) URL.
/// 3. Platform-specific fallback constructed from `id` + `container_url`:
/// - YouTube Music → `https://music.youtube.com/watch?v={id}`
/// - YouTube → `https://www.youtube.com/watch?v={id}`
/// - Spotify → `https://open.spotify.com/track/{id}`
/// - Other → `None` (caller should skip the item and warn).
fn normalize_item_url(
entry: &serde_json::Value,
id: &str,
container_url: &str,
) -> Option<String> {
let is_abs = |s: &str| s.starts_with("http://") || s.starts_with("https://");
if let Some(u) = entry.get("webpage_url").and_then(|v| v.as_str()).filter(|s| is_abs(s)) {
return Some(u.to_owned());
}
if let Some(u) = entry.get("url").and_then(|v| v.as_str()).filter(|s| is_abs(s)) {
return Some(u.to_owned());
}
// Bare-ID fallback keyed on the container's platform.
if container_url.contains("music.youtube.com") {
Some(format!("https://music.youtube.com/watch?v={id}"))
} else if container_url.contains("youtube.com") || container_url.contains("youtu.be") {
Some(format!("https://www.youtube.com/watch?v={id}"))
} else if container_url.contains("open.spotify.com") {
Some(format!("https://open.spotify.com/track/{id}"))
} else {
eprintln!("warn: skipping playlist item {id:?} — no absolute URL from yt-dlp");
None
}
}
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the single-JSON result.
///
/// `-J` / `--dump-single-json` returns one JSON object for the whole
@ -359,42 +394,19 @@ pub fn fetch_playlist_info(url: &str, cookies: &HashMap<String, String>) -> Resu
.map(|a| a.as_slice())
.unwrap_or(&[]);
// Infer the fallback base host from the container URL so YouTube Music
// entries stay on music.youtube.com rather than www.youtube.com.
let fallback_host = if url.contains("music.youtube.com") {
"music.youtube.com"
} else {
"www.youtube.com"
};
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
let mut items = Vec::with_capacity(raw_entries.len());
for entry in raw_entries {
if entry.is_null() {
continue; // unavailable/private video in flat listing
continue; // unavailable/private item in flat listing
}
let id = match entry.get("id").and_then(|v| v.as_str()) {
Some(s) => s.to_owned(),
None => continue,
};
// Normalize the item URL:
// 1. Prefer `webpage_url` — yt-dlp always makes this absolute when present.
// 2. Accept `url` only when it is already an absolute HTTP(S) URL; in
// flat-playlist mode `url` is often just the bare video ID.
// 3. Fallback: construct from `id` using the same host as the container URL.
let item_url = entry
.get("webpage_url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
.or_else(|| {
entry
.get("url")
.and_then(|v| v.as_str())
.filter(|s| is_absolute(s))
.map(str::to_owned)
})
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
let item_url = match normalize_item_url(entry, &id, url) {
Some(u) => u,
None => continue,
};
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader });
@ -455,14 +467,6 @@ pub fn probe_playlist_qualities(
let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
// Same URL normalization as fetch_playlist_info
let fallback_host = if url.contains("music.youtube.com") {
"music.youtube.com"
} else {
"www.youtube.com"
};
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
let raw_entries = json
.get("entries")
.and_then(|v| v.as_array())
@ -476,10 +480,10 @@ pub fn probe_playlist_qualities(
Some(s) => s.to_owned(),
None => continue,
};
let item_url = entry
.get("webpage_url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned)
.or_else(|| entry.get("url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned))
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
let item_url = match normalize_item_url(entry, &id, url) {
Some(u) => u,
None => continue,
};
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
let heights = available_video_heights_from_value(entry);
let qualities: Vec<String> = heights.iter().map(|h| format!("{h}p")).collect();

2
crates/archivr-server/static/assets/index-B2Gbe8XQ.js → crates/archivr-server/static/assets/index-De3b80Fv.js
View file

File diff suppressed because one or more lines are too long

2
crates/archivr-server/static/index.html
View file

@ -4,7 +4,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Archivr</title>
<script type="module" crossorigin src="/assets/index-B2Gbe8XQ.js"></script>
<script type="module" crossorigin src="/assets/index-De3b80Fv.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D8ic-z4p.css">
</head>
<body>

14
frontend/src/components/CaptureDialog.jsx
View file

@ -85,24 +85,28 @@ function isPlaylistSource(locator) {
return ll.slice(4).startsWith('playlist/')
}
// spotify: shorthands — album and playlist (not track)
if (ll.startsWith('spotify:')) {
const after = ll.slice(8)
return after.startsWith('album:') || after.startsWith('playlist:')
}
// HTTP/HTTPS URLs
if (ll.startsWith('http://') || ll.startsWith('https://')) {
try {
const url = new URL(l)
const host = url.hostname
if (host === 'youtube.com' || host === 'www.youtube.com') {
// Mirror determine_source exactly:
// - /playlist?list=... → YouTubePlaylist
// - /@handle, /channel/, /c/, /user/ → YouTubeChannel
// - everything else (incl. /watch&list=, /shorts?list=) → NOT a playlist
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
if (url.pathname.startsWith('/@') || url.pathname.startsWith('/channel/') ||
url.pathname.startsWith('/c/') || url.pathname.startsWith('/user/')) return true
}
if (host === 'music.youtube.com') {
// /watch is a single YTM track; only /playlist?list=... is a YTM playlist
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
}
if (host === 'open.spotify.com') {
if (url.pathname.startsWith('/album/') || url.pathname.startsWith('/playlist/')) return true
}
} catch {}
}