mirror of
https://github.com/thegeneralist01/archivr
synced 2026-07-21 18:55:36 +02:00
feat(core+frontend): Spotify album/playlist capture via yt-dlp container path
Previously SpotifyAlbum and SpotifyPlaylist hard-failed with a DRM error.
yt-dlp does support Spotify (experimentally), so they now go through the
same probe/container/child path as YouTube playlists.
ytdlp.rs:
- Extract normalize_item_url() helper used by both fetch_playlist_info
and probe_playlist_qualities; eliminates the duplicate URL-normalization
blocks and the YouTube-specific fallback_host variable
- Fallback logic is now platform-aware: YouTube/YTM bare IDs → watch URL,
Spotify → open.spotify.com/track/{id}, unknown → skip with warning
capture.rs:
- locator_to_playlist_url: accept SpotifyAlbum | SpotifyPlaylist so the
probe-playlist endpoint accepts Spotify album/playlist URLs
- Container branch: add SpotifyAlbum | SpotifyPlaylist to the matches!
- is_audio: true for SpotifyAlbum | SpotifyPlaylist (audio-only, like YTM)
- child_source: SpotifyTrack for Spotify containers (not YouTubeMusicTrack)
- generate_entry_title and record_media_entry both use the correct child
source so entity_kind, source_kind, and representation_kind are right
CaptureDialog.jsx:
- isPlaylistSource: recognise open.spotify.com/album/ and /playlist/,
and spotify:album:ID / spotify:playlist:ID shorthands, so Spotify
containers get the probe UI, per-track excludes, and sync toggle
This commit is contained in:
parent
5e7ce4774b
commit
6a7826eba5
5 changed files with 82 additions and 71 deletions
|
|
@ -568,9 +568,11 @@ pub fn locator_to_ytdlp_url(locator: &str) -> Option<String> {
|
||||||
pub fn locator_to_playlist_url(locator: &str) -> Option<String> {
|
pub fn locator_to_playlist_url(locator: &str) -> Option<String> {
|
||||||
let source = determine_source(locator);
|
let source = determine_source(locator);
|
||||||
match source {
|
match source {
|
||||||
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist => {
|
Source::YouTubePlaylist
|
||||||
Some(expand_shorthand_to_url(locator, &source))
|
| Source::YouTubeChannel
|
||||||
}
|
| Source::YouTubeMusicPlaylist
|
||||||
|
| Source::SpotifyAlbum
|
||||||
|
| Source::SpotifyPlaylist => Some(expand_shorthand_to_url(locator, &source)),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1051,26 +1053,17 @@ pub fn perform_capture(
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sources: Spotify — SpotifyTrack attempts yt-dlp and fails gracefully if the
|
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists,
|
||||||
// extractor cannot handle the content. Albums and playlists are not yet supported
|
// Spotify albums, Spotify playlists — probe via yt-dlp flat-playlist,
|
||||||
// as containers (fetch_playlist_info has YouTube-specific URL fallback logic).
|
// create a root container entry, then download each track as a child entry.
|
||||||
if matches!(source, Source::SpotifyAlbum | Source::SpotifyPlaylist) {
|
|
||||||
return Err(fail_run(
|
|
||||||
&conn,
|
|
||||||
&run,
|
|
||||||
&item,
|
|
||||||
"Spotify album/playlist capture is not yet implemented. \
|
|
||||||
Archive individual tracks instead.",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sources: YouTube playlists, YouTube channels, YouTube Music playlists.
|
|
||||||
// Fetch container metadata, create a root container entry, then download
|
|
||||||
// each video/track as a child entry.
|
|
||||||
// `run` and `item` are already created above; `item` acts as the container item.
|
// `run` and `item` are already created above; `item` acts as the container item.
|
||||||
if matches!(
|
if matches!(
|
||||||
source,
|
source,
|
||||||
Source::YouTubePlaylist | Source::YouTubeChannel | Source::YouTubeMusicPlaylist
|
Source::YouTubePlaylist
|
||||||
|
| Source::YouTubeChannel
|
||||||
|
| Source::YouTubeMusicPlaylist
|
||||||
|
| Source::SpotifyAlbum
|
||||||
|
| Source::SpotifyPlaylist
|
||||||
) {
|
) {
|
||||||
// `canonical_url` already holds the expanded URL (same value as `path` later).
|
// `canonical_url` already holds the expanded URL (same value as `path` later).
|
||||||
let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) {
|
let playlist_info = match downloader::ytdlp::fetch_playlist_info(&canonical_url, &cookies) {
|
||||||
|
|
@ -1149,7 +1142,10 @@ pub fn perform_capture(
|
||||||
(e.id, std::collections::HashSet::new())
|
(e.id, std::collections::HashSet::new())
|
||||||
};
|
};
|
||||||
|
|
||||||
let is_audio = source == Source::YouTubeMusicPlaylist;
|
let is_audio = matches!(
|
||||||
|
source,
|
||||||
|
Source::YouTubeMusicPlaylist | Source::SpotifyAlbum | Source::SpotifyPlaylist
|
||||||
|
);
|
||||||
let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality };
|
let child_quality: Option<&str> = if is_audio { Some("audio") } else { quality };
|
||||||
let child_entity_kind = if is_audio { "music" } else { "video" };
|
let child_entity_kind = if is_audio { "music" } else { "video" };
|
||||||
|
|
||||||
|
|
@ -1199,7 +1195,11 @@ pub fn perform_capture(
|
||||||
Some(json) => {
|
Some(json) => {
|
||||||
let meta = downloader::metadata::extract_from_ytdlp_json(json);
|
let meta = downloader::metadata::extract_from_ytdlp_json(json);
|
||||||
Some(generate_entry_title(
|
Some(generate_entry_title(
|
||||||
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo },
|
match source {
|
||||||
|
Source::SpotifyAlbum | Source::SpotifyPlaylist => Source::SpotifyTrack,
|
||||||
|
_ if is_audio => Source::YouTubeMusicTrack,
|
||||||
|
_ => Source::YouTubeVideo,
|
||||||
|
},
|
||||||
&meta,
|
&meta,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
@ -1256,8 +1256,11 @@ pub fn perform_capture(
|
||||||
}
|
}
|
||||||
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
|
let _ = fs::remove_dir_all(store_path.join("temp").join(&child_timestamp));
|
||||||
|
|
||||||
let child_source =
|
let child_source = match source {
|
||||||
if is_audio { Source::YouTubeMusicTrack } else { Source::YouTubeVideo };
|
Source::SpotifyAlbum | Source::SpotifyPlaylist => Source::SpotifyTrack,
|
||||||
|
_ if is_audio => Source::YouTubeMusicTrack,
|
||||||
|
_ => Source::YouTubeVideo,
|
||||||
|
};
|
||||||
match record_media_entry(
|
match record_media_entry(
|
||||||
&conn,
|
&conn,
|
||||||
store_path,
|
store_path,
|
||||||
|
|
|
||||||
|
|
@ -298,6 +298,41 @@ pub fn fetch_metadata(path: &str, cookies: &HashMap<String, String>) -> Option<S
|
||||||
if json.trim().is_empty() { None } else { Some(json) }
|
if json.trim().is_empty() { None } else { Some(json) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolves an absolute item URL from a flat-playlist entry JSON object.
|
||||||
|
///
|
||||||
|
/// Priority:
|
||||||
|
/// 1. `webpage_url` — yt-dlp makes this absolute when present.
|
||||||
|
/// 2. `url` when it is already an absolute HTTP(S) URL.
|
||||||
|
/// 3. Platform-specific fallback constructed from `id` + `container_url`:
|
||||||
|
/// - YouTube Music → `https://music.youtube.com/watch?v={id}`
|
||||||
|
/// - YouTube → `https://www.youtube.com/watch?v={id}`
|
||||||
|
/// - Spotify → `https://open.spotify.com/track/{id}`
|
||||||
|
/// - Other → `None` (caller should skip the item and warn).
|
||||||
|
fn normalize_item_url(
|
||||||
|
entry: &serde_json::Value,
|
||||||
|
id: &str,
|
||||||
|
container_url: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
let is_abs = |s: &str| s.starts_with("http://") || s.starts_with("https://");
|
||||||
|
if let Some(u) = entry.get("webpage_url").and_then(|v| v.as_str()).filter(|s| is_abs(s)) {
|
||||||
|
return Some(u.to_owned());
|
||||||
|
}
|
||||||
|
if let Some(u) = entry.get("url").and_then(|v| v.as_str()).filter(|s| is_abs(s)) {
|
||||||
|
return Some(u.to_owned());
|
||||||
|
}
|
||||||
|
// Bare-ID fallback keyed on the container's platform.
|
||||||
|
if container_url.contains("music.youtube.com") {
|
||||||
|
Some(format!("https://music.youtube.com/watch?v={id}"))
|
||||||
|
} else if container_url.contains("youtube.com") || container_url.contains("youtu.be") {
|
||||||
|
Some(format!("https://www.youtube.com/watch?v={id}"))
|
||||||
|
} else if container_url.contains("open.spotify.com") {
|
||||||
|
Some(format!("https://open.spotify.com/track/{id}"))
|
||||||
|
} else {
|
||||||
|
eprintln!("warn: skipping playlist item {id:?} — no absolute URL from yt-dlp");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the single-JSON result.
|
/// Runs `yt-dlp -J --flat-playlist <url>` and parses the single-JSON result.
|
||||||
///
|
///
|
||||||
/// `-J` / `--dump-single-json` returns one JSON object for the whole
|
/// `-J` / `--dump-single-json` returns one JSON object for the whole
|
||||||
|
|
@ -359,42 +394,19 @@ pub fn fetch_playlist_info(url: &str, cookies: &HashMap<String, String>) -> Resu
|
||||||
.map(|a| a.as_slice())
|
.map(|a| a.as_slice())
|
||||||
.unwrap_or(&[]);
|
.unwrap_or(&[]);
|
||||||
|
|
||||||
// Infer the fallback base host from the container URL so YouTube Music
|
|
||||||
// entries stay on music.youtube.com rather than www.youtube.com.
|
|
||||||
let fallback_host = if url.contains("music.youtube.com") {
|
|
||||||
"music.youtube.com"
|
|
||||||
} else {
|
|
||||||
"www.youtube.com"
|
|
||||||
};
|
|
||||||
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
|
|
||||||
|
|
||||||
let mut items = Vec::with_capacity(raw_entries.len());
|
let mut items = Vec::with_capacity(raw_entries.len());
|
||||||
for entry in raw_entries {
|
for entry in raw_entries {
|
||||||
if entry.is_null() {
|
if entry.is_null() {
|
||||||
continue; // unavailable/private video in flat listing
|
continue; // unavailable/private item in flat listing
|
||||||
}
|
}
|
||||||
let id = match entry.get("id").and_then(|v| v.as_str()) {
|
let id = match entry.get("id").and_then(|v| v.as_str()) {
|
||||||
Some(s) => s.to_owned(),
|
Some(s) => s.to_owned(),
|
||||||
None => continue,
|
None => continue,
|
||||||
};
|
};
|
||||||
// Normalize the item URL:
|
let item_url = match normalize_item_url(entry, &id, url) {
|
||||||
// 1. Prefer `webpage_url` — yt-dlp always makes this absolute when present.
|
Some(u) => u,
|
||||||
// 2. Accept `url` only when it is already an absolute HTTP(S) URL; in
|
None => continue,
|
||||||
// flat-playlist mode `url` is often just the bare video ID.
|
};
|
||||||
// 3. Fallback: construct from `id` using the same host as the container URL.
|
|
||||||
let item_url = entry
|
|
||||||
.get("webpage_url")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.filter(|s| is_absolute(s))
|
|
||||||
.map(str::to_owned)
|
|
||||||
.or_else(|| {
|
|
||||||
entry
|
|
||||||
.get("url")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.filter(|s| is_absolute(s))
|
|
||||||
.map(str::to_owned)
|
|
||||||
})
|
|
||||||
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
|
|
||||||
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
||||||
let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
|
let item_uploader = entry.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
|
||||||
items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader });
|
items.push(PlaylistItem { id, url: item_url, title: item_title, uploader: item_uploader });
|
||||||
|
|
@ -455,14 +467,6 @@ pub fn probe_playlist_qualities(
|
||||||
let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
let title = json.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
||||||
let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
|
let uploader = json.get("uploader").and_then(|v| v.as_str()).map(str::to_owned);
|
||||||
|
|
||||||
// Same URL normalization as fetch_playlist_info
|
|
||||||
let fallback_host = if url.contains("music.youtube.com") {
|
|
||||||
"music.youtube.com"
|
|
||||||
} else {
|
|
||||||
"www.youtube.com"
|
|
||||||
};
|
|
||||||
let is_absolute = |s: &str| s.starts_with("http://") || s.starts_with("https://");
|
|
||||||
|
|
||||||
let raw_entries = json
|
let raw_entries = json
|
||||||
.get("entries")
|
.get("entries")
|
||||||
.and_then(|v| v.as_array())
|
.and_then(|v| v.as_array())
|
||||||
|
|
@ -476,10 +480,10 @@ pub fn probe_playlist_qualities(
|
||||||
Some(s) => s.to_owned(),
|
Some(s) => s.to_owned(),
|
||||||
None => continue,
|
None => continue,
|
||||||
};
|
};
|
||||||
let item_url = entry
|
let item_url = match normalize_item_url(entry, &id, url) {
|
||||||
.get("webpage_url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned)
|
Some(u) => u,
|
||||||
.or_else(|| entry.get("url").and_then(|v| v.as_str()).filter(|s| is_absolute(s)).map(str::to_owned))
|
None => continue,
|
||||||
.unwrap_or_else(|| format!("https://{fallback_host}/watch?v={id}"));
|
};
|
||||||
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
let item_title = entry.get("title").and_then(|v| v.as_str()).map(str::to_owned);
|
||||||
let heights = available_video_heights_from_value(entry);
|
let heights = available_video_heights_from_value(entry);
|
||||||
let qualities: Vec<String> = heights.iter().map(|h| format!("{h}p")).collect();
|
let qualities: Vec<String> = heights.iter().map(|h| format!("{h}p")).collect();
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -4,7 +4,7 @@
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>Archivr</title>
|
<title>Archivr</title>
|
||||||
<script type="module" crossorigin src="/assets/index-B2Gbe8XQ.js"></script>
|
<script type="module" crossorigin src="/assets/index-De3b80Fv.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-D8ic-z4p.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-D8ic-z4p.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|
|
||||||
|
|
@ -85,24 +85,28 @@ function isPlaylistSource(locator) {
|
||||||
return ll.slice(4).startsWith('playlist/')
|
return ll.slice(4).startsWith('playlist/')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// spotify: shorthands — album and playlist (not track)
|
||||||
|
if (ll.startsWith('spotify:')) {
|
||||||
|
const after = ll.slice(8)
|
||||||
|
return after.startsWith('album:') || after.startsWith('playlist:')
|
||||||
|
}
|
||||||
|
|
||||||
// HTTP/HTTPS URLs
|
// HTTP/HTTPS URLs
|
||||||
if (ll.startsWith('http://') || ll.startsWith('https://')) {
|
if (ll.startsWith('http://') || ll.startsWith('https://')) {
|
||||||
try {
|
try {
|
||||||
const url = new URL(l)
|
const url = new URL(l)
|
||||||
const host = url.hostname
|
const host = url.hostname
|
||||||
if (host === 'youtube.com' || host === 'www.youtube.com') {
|
if (host === 'youtube.com' || host === 'www.youtube.com') {
|
||||||
// Mirror determine_source exactly:
|
|
||||||
// - /playlist?list=... → YouTubePlaylist
|
|
||||||
// - /@handle, /channel/, /c/, /user/ → YouTubeChannel
|
|
||||||
// - everything else (incl. /watch&list=, /shorts?list=) → NOT a playlist
|
|
||||||
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
|
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
|
||||||
if (url.pathname.startsWith('/@') || url.pathname.startsWith('/channel/') ||
|
if (url.pathname.startsWith('/@') || url.pathname.startsWith('/channel/') ||
|
||||||
url.pathname.startsWith('/c/') || url.pathname.startsWith('/user/')) return true
|
url.pathname.startsWith('/c/') || url.pathname.startsWith('/user/')) return true
|
||||||
}
|
}
|
||||||
if (host === 'music.youtube.com') {
|
if (host === 'music.youtube.com') {
|
||||||
// /watch is a single YTM track; only /playlist?list=... is a YTM playlist
|
|
||||||
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
|
if (url.pathname === '/playlist' && url.searchParams.has('list')) return true
|
||||||
}
|
}
|
||||||
|
if (host === 'open.spotify.com') {
|
||||||
|
if (url.pathname.startsWith('/album/') || url.pathname.startsWith('/playlist/')) return true
|
||||||
|
}
|
||||||
} catch {}
|
} catch {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue