diff --git a/TODO.md b/TODO.md new file mode 100755 index 0000000..55a515e --- /dev/null +++ b/TODO.md @@ -0,0 +1,8 @@ +# Task list + +Tasks that have been completed in commits before this one have been omitted from this list. + +- [x] /video PUT route +- [x] /channel PUT route +- [ ] OpenAPI documentation +- [ ] Video versioning diff --git a/src/channel.rs b/src/channel.rs index 12cf60d..c824379 100755 --- a/src/channel.rs +++ b/src/channel.rs @@ -19,8 +19,8 @@ pub enum ChannelError { IOError(#[from] io::Error), #[error("UTF-8 error: {0}")] Utf8Error(#[from] Utf8Error), - #[error("Invalid output from ID extraction function")] - InvalidOutput, + #[error("Got erroneous output for yt-dlp task")] + TaskFailed, #[error("Failed to extract channel ID from '{0}'")] ExtractID(String), #[error("Could not serialize info JSON: {0}")] @@ -43,7 +43,7 @@ pub struct Channel { } impl Channel { - async fn yt_dlp_task(url: &str) -> Result<(), ChannelError> { + async fn run_task(url: &str) -> Result<(), ChannelError> { let mut child = Command::new("yt-dlp") .args([ "--write-info-json", @@ -57,12 +57,60 @@ impl Channel { ]) .spawn()?; info!("yt-dlp task invoked"); - child.wait().await?; + child.wait().await.map_err(|_| ChannelError::TaskFailed)?; info!("yt-dlp task completed successfully"); Ok(()) } + async fn run_update_task(url: &str) -> Result<(), ChannelError> { + let mut child = Command::new("yt-dlp") + .args([ + "--write-info-json", + "--skip-download", + "--playlist-items", + "1", + "-v", + "-o", + "media/channels/%(id)s/%(id)s.%(ext)s", + url, + ]) + .spawn()?; + info!("yt-dlp task invoked"); + child.wait().await.map_err(|_| ChannelError::TaskFailed)?; + info!("yt-dlp task completed successfully"); + + Ok(()) + } + + pub async fn update_metadata(&mut self) -> Result<(), ChannelError> { + Self::run_update_task(&self.url).await?; + + let json_path = format!("media/channels/{0}/{0}.info.json", self.youtube_id); + let info: Value = serde_json::from_str(&fs::read_to_string(&json_path).await?)?; + + self.name = Self::get_json_value(&info, "channel")?.to_unquoted_string(); + self.handle_url = Self::get_json_value(&info, "uploader_url")?.to_unquoted_string(); + (self.avatar_url, self.banner_url) = Self::get_thumbnails(&info); + self.description = Self::get_json_value(&info, "description")?.to_unquoted_string(); + self.subscribers = Self::get_json_value(&info, "channel_follower_count")? + .as_i64() + .unwrap_or_default(); + + info!( + "New information for '{}': {{ name: {}, handle_url: {}, avatar_url: {}, banner_url: {}, description: {}, subscribers: {} }}", + self.youtube_id, + self.name, + self.handle_url, + self.avatar_url, + self.banner_url, + self.description, + self.subscribers + ); + + Ok(()) + } + /// Transforms a channel handle or username into a canonical channel ID. /// /// While it invariably returns a canonical ID even if the URL already contains one, @@ -85,7 +133,7 @@ impl Channel { error!( "yt-dlp could not download channel. Does the URL point to a valid/existing channel?" ); - return Err(ChannelError::InvalidOutput); + return Err(ChannelError::TaskFailed); } let stdout = str::from_utf8(&output.stdout)?; @@ -97,6 +145,43 @@ impl Channel { ) } + fn get_json_value(json: &Value, key: &str) -> Result { + json.get(key) + .ok_or_else(|| ChannelError::JsonKey(key.to_string())) + .cloned() + } + + fn get_thumbnails(info_json: &Value) -> (String, String) { + let mut avatar_url = String::new(); + let mut banner_url = String::new(); + + if let Some(thumbnails) = info_json.get("thumbnails").and_then(Value::as_array) { + for thumbnail in thumbnails { + if let Some(id) = thumbnail.get("id").and_then(Value::as_str) { + match id { + "avatar_uncropped" => { + avatar_url = thumbnail + .get("url") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + } + "banner_uncropped" => { + banner_url = thumbnail + .get("url") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + } + _ => {} + } + } + } + } + + (avatar_url, banner_url) + } + pub async fn from_url(url: &Url, id: i64) -> Result { let url_path = url.path(); @@ -130,51 +215,23 @@ impl Channel { fs::create_dir(dir).await?; - Self::yt_dlp_task(url).await?; + Self::run_task(url).await?; let info: Value = serde_json::from_str(&fs::read_to_string("media/channels/tmp/tmp.info.json").await?)?; - let get_info_value = |key: &str| { - info.get(key) - .ok_or_else(|| ChannelError::JsonKey(key.to_string())) - .unwrap() - }; + let url = Self::get_json_value(&info, "channel_url")?.to_unquoted_string(); + let name = Self::get_json_value(&info, "channel")?.to_unquoted_string(); + let handle_url = Self::get_json_value(&info, "uploader_url")?.to_unquoted_string(); - let url = get_info_value("channel_url").to_unquoted_string(); - let name = get_info_value("channel").to_unquoted_string(); - let handle_url = get_info_value("uploader_url").to_unquoted_string(); - - let mut avatar_url = String::new(); - let mut banner_url = String::new(); - - if let Some(thumbnails) = info.get("thumbnails").and_then(Value::as_array) { - for thumbnail in thumbnails { - if let Some(id) = thumbnail.get("id").and_then(Value::as_str) { - match id { - "avatar_uncropped" => { - avatar_url = thumbnail - .get("url") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - } - "banner_uncropped" => { - banner_url = thumbnail - .get("url") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - } - _ => {} - } - } - } - } else { - warn!("Channel {youtube_id} has no thumbnails!"); + let (avatar_url, banner_url) = Self::get_thumbnails(&info); + if avatar_url.is_empty() { + warn!("Channel {youtube_id} has no avatar!"); } - - let description = get_info_value("description").to_unquoted_string(); - let subscribers = get_info_value("channel_follower_count") + if banner_url.is_empty() { + warn!("Channel {youtube_id} has no banner!"); + } + let description = Self::get_json_value(&info, "description")?.to_unquoted_string(); + let subscribers = Self::get_json_value(&info, "channel_follower_count")? .as_i64() .unwrap_or_default(); diff --git a/src/routes/channel.rs b/src/routes/channel.rs index 1ae9c2a..48b6208 100755 --- a/src/routes/channel.rs +++ b/src/routes/channel.rs @@ -109,7 +109,7 @@ pub async fn upload_channel( let new_channel = Channel::from_url(&url, id).await.map_err(|e| match e { ChannelError::AlreadyExists => StatusCode::OK, - ChannelError::InvalidOutput => StatusCode::BAD_REQUEST, + ChannelError::TaskFailed => StatusCode::BAD_REQUEST, e => { error!("{e:?}"); StatusCode::INTERNAL_SERVER_ERROR @@ -161,10 +161,37 @@ pub async fn update_channel(State(state): State, Path(id): Path StatusCode::OK, + Ok(mut channel) => match channel.update_metadata().await { + Ok(()) => { + match sqlx::query!( + "UPDATE channel SET name = ?, handle_url = ?, avatar_url = ?, banner_url = ?, description = ?, subscribers = ? WHERE youtube_id = ?", + channel.name, + channel.handle_url, + channel.avatar_url, + channel.banner_url, + channel.description, + channel.subscribers, + id + ).execute(&state.pool).await { + Ok(r) => { + info!("Updated channel '{id}' successfully: {r:?}"); + StatusCode::OK + }, + Err(e) => { + error!("Update channel failed: {e:?}"); + StatusCode::INTERNAL_SERVER_ERROR + } + } + } + Err(e) => { + error!("Update video failed: {e:?}"); + StatusCode::INTERNAL_SERVER_ERROR + } + }, Err(status) => { - error!("Failed to update video '{id}'"); + if status != StatusCode::NOT_FOUND { + error!("Failed to update video '{id}'"); + } status } } diff --git a/src/routes/video.rs b/src/routes/video.rs index ad8bb79..6b93031 100755 --- a/src/routes/video.rs +++ b/src/routes/video.rs @@ -182,10 +182,35 @@ pub async fn update_video(State(state): State, Path(id): Path) .map_or(Err(StatusCode::INTERNAL_SERVER_ERROR), |video| { video.ok_or(StatusCode::NOT_FOUND) }) { - // TODO - Ok(_video) => StatusCode::OK, + Ok(mut video) => match video.update_metadata().await { + Ok(()) => { + match sqlx::query!( + "UPDATE video SET title = ?, views = ?, likes = ?, dislikes = ? WHERE youtube_id = ?", + video.title, + video.views, + video.likes, + video.dislikes, + id + ).execute(&state.pool).await { + Ok(r) => { + info!("Updated video '{id}' successfully: {r:?}"); + StatusCode::OK + }, + Err(e) => { + error!("Update video failed: {e:?}"); + StatusCode::INTERNAL_SERVER_ERROR + } + } + } + Err(e) => { + error!("Update video failed: {e:?}"); + StatusCode::INTERNAL_SERVER_ERROR + } + }, Err(status) => { - error!("Failed to update video '{id}'"); + if status != StatusCode::NOT_FOUND { + error!("Failed to update video '{id}'"); + } status } } diff --git a/src/video.rs b/src/video.rs index 2e3c0fd..5dcdd15 100755 --- a/src/video.rs +++ b/src/video.rs @@ -18,6 +18,8 @@ pub enum VideoError { AlreadyExists, #[error("IO Error: {0}")] IOError(#[from] io::Error), + #[error("Got erroneous status code for yt-dlp task")] + TaskFailed, #[error("Could not serialize info JSON: {0}")] SerializeInfoJSON(#[from] serde_json::Error), #[error("Failed to parse value from key '{0}'")] @@ -48,7 +50,7 @@ pub struct Video { } impl Video { - async fn yt_dlp_task(url: &str) -> Result<(), VideoError> { + async fn run_task(url: &str) -> Result<(), VideoError> { let args = vec![ "--write-info-json", "--write-thumbnail", @@ -66,12 +68,66 @@ impl Video { ]; let mut child = tokio::process::Command::new("yt-dlp").args(args).spawn()?; info!("yt-dlp task invoked"); - child.wait().await?; + child.wait().await.map_err(|_| VideoError::TaskFailed)?; info!("yt-dlp task completed successfully"); Ok(()) } + async fn run_update_task(url: &str) -> Result<(), VideoError> { + let args = vec![ + "--write-info-json", + "--skip-download", + "--write-thumbnail", + "--write-description", + "--write-comments", + "--force-write", + "--no-playlist", + "--use-postprocessor", + "ReturnYoutubeDislike:when=pre_process", + "-v", + url, + "-o", + "media/videos/%(id)s/%(id)s.%(ext)s", + ]; + let mut child = tokio::process::Command::new("yt-dlp").args(args).spawn()?; + info!("yt-dlp task invoked"); + child.wait().await.map_err(|_| VideoError::TaskFailed)?; + info!("yt-dlp task completed successfully"); + + Ok(()) + } + + pub async fn update_metadata(&mut self) -> Result<(), VideoError> { + Self::run_update_task(&self.url).await?; + + let json_path = format!("media/videos/{0}/{0}.info.json", self.youtube_id); + + let info: Value = serde_json::from_str(&fs::read_to_string(&json_path).await?)?; + + self.title = Self::get_json_value(&info, "title")?.to_unquoted_string(); + self.views = Self::get_json_value(&info, "view_count")? + .as_i64() + .unwrap_or(-1); + + // Use RYD field from info JSON + self.likes = Self::get_json_value(&info["RYD"]["response"], "likes")?.as_i64(); + self.dislikes = Self::get_json_value(&info["RYD"]["response"], "dislikes")?.as_i64(); + + info!( + "New information for video '{}': {{ title: {}, views: {}, likes: {:?}, dislikes: {:?} }}", + self.youtube_id, self.title, self.views, self.likes, self.dislikes + ); + + Ok(()) + } + + fn get_json_value(json: &Value, key: &str) -> Result { + json.get(key) + .ok_or_else(|| VideoError::JsonKey(key.to_string())) + .cloned() + } + pub async fn from_url(url: &Url, id: i64) -> Result { let mut pairs = url.query_pairs(); let Some(query_v) = pairs.find(|(key, _)| key == "v") else { @@ -96,34 +152,23 @@ impl Video { let info_json = format!("{file_stem}.info.json"); let info_json = Path::new(&info_json); - Self::yt_dlp_task(url.as_str()).await?; + Self::run_task(url.as_str()).await?; let info: Value = serde_json::from_str(&fs::read_to_string(info_json).await?)?; // info!("Info JSON for {youtube_id}\n{info}"); - let get_info_value = |key: &str| { - info.get(key) - .ok_or_else(|| VideoError::JsonKey(key.to_string())) - .unwrap() - }; - - let get_ryd_value = |key: &str| { - info["RYD"]["response"] - .get(key) - .ok_or_else(|| VideoError::JsonKey(key.to_string())) - .unwrap() - }; - let url = url.to_string(); let description = format!("{file_stem}.description"); - let title = get_info_value("title").to_unquoted_string(); - let author = get_info_value("uploader").to_unquoted_string(); - let views = get_info_value("view_count").as_i64().unwrap_or(-1); + let title = Self::get_json_value(&info, "title")?.to_unquoted_string(); + let author = Self::get_json_value(&info, "uploader")?.to_unquoted_string(); + let views = Self::get_json_value(&info, "view_count")? + .as_i64() + .unwrap_or(-1); // Use RYD field from info JSON - let likes = get_ryd_value("likes").as_i64(); - let dislikes = get_ryd_value("dislikes").as_i64(); + let likes = Self::get_json_value(&info["RYD"]["response"], "likes")?.as_i64(); + let dislikes = Self::get_json_value(&info["RYD"]["response"], "dislikes")?.as_i64(); let possible_extensions = ["mkv", "mp4", "webm"]; @@ -149,9 +194,9 @@ impl Video { let sha256 = format!("{:x}", Sha3_256::digest(&buffer)); #[allow(clippy::cast_possible_wrap)] let file_size = buffer.len() as i64; - let author_id = get_info_value("channel_id").to_unquoted_string(); - let author_url = get_info_value("channel_url").to_unquoted_string(); - let upload_date = get_info_value("upload_date").to_unquoted_string(); + let author_id = Self::get_json_value(&info, "channel_id")?.to_unquoted_string(); + let author_url = Self::get_json_value(&info, "channel_url")?.to_unquoted_string(); + let upload_date = Self::get_json_value(&info, "upload_date")?.to_unquoted_string(); let thumbnail = format!("{file_stem}.webp");