From: Jesús E. Date: Fri, 12 Jul 2019 20:42:12 +0100 Subject: Added the `use_invidious_api` config-option (enabled by default). When enabled, it will the API of `invidio.us` to extract the streaming URLs for videos with encrypted signatures. Example: youtube-viewer --invidious # use invidio.us API (default) youtube-viewer --no-invidious # don't use invidio.us API Hyperbola: https://www.hyperbola.info/ --- diff --git a/bin/gtk-youtube-viewer b/bin/gtk-youtube-viewer index 3018152..0bce3d8 100755 --- a/bin/gtk-youtube-viewer +++ b/bin/gtk-youtube-viewer @@ -225,9 +225,9 @@ my %CONFIG = ( # URI options thumbnail_type => 'medium', youtube_thumb_url => 'https://i1.ytimg.com/vi/%s/%s.jpg', - youtube_video_url => 'https://www.youtube.com/watch?v=%s', - youtube_playlist_url => 'https://www.youtube.com/playlist?list=%s', - youtube_channel_url => 'https://www.youtube.com/channel/%s', + youtube_video_url => 'https://invidio.us/watch?v=%s', + youtube_playlist_url => 'https://invidio.us/playlist?list=%s', + youtube_channel_url => 'https://invidio.us/channel/%s', # Subtitle options srt_languages => ['en', 'es'], @@ -244,6 +244,8 @@ my %CONFIG = ( fullscreen => 0, audio_only => 0, + use_invidious_api => 1, # 0 disable, 1 enable + use_threads => 0, use_threads_for_thumbs => 0, # this is unstable @@ -736,6 +738,7 @@ my $yv_obj = WWW::YoutubeViewer->new( hl => $CONFIG{hl}, lwp_env_proxy => $CONFIG{env_proxy}, cache_dir => $CONFIG{cache_dir}, + use_invidious_api => $CONFIG{use_invidious_api}, authentication_file => $authentication_file, ); @@ -795,7 +798,7 @@ sub apply_configuration { videoEmbeddable videoLicense publishedAfter publishedBefore regionCode videoCategoryId - debug http_proxy + debug http_proxy use_invidious_api ) ) { diff --git a/bin/youtube-viewer b/bin/youtube-viewer index 64c32f3..6d894f4 100755 --- a/bin/youtube-viewer +++ b/bin/youtube-viewer @@ -287,7 +287,7 @@ my %CONFIG = ( regionCode => undef, # URI options - youtube_video_url => 'https://www.youtube.com/watch?v=%s', + youtube_video_url => 'https://invidio.us/watch?v=%s', # Subtitle options srt_languages => ['en', 'es'], @@ -298,6 +298,7 @@ my %CONFIG = ( cache_dir => undef, # will be defined later # Others + use_invidious_api => 1, # 0 disable, 1 enable http_proxy => undef, env_proxy => 1, confirm => 0, @@ -566,6 +567,7 @@ my $yv_obj = WWW::YoutubeViewer->new( config_dir => $config_dir, cache_dir => $opt{cache_dir}, lwp_env_proxy => $opt{env_proxy}, + use_invidious_api => $opt{use_invidious_api}, authentication_file => $authentication_file, ); @@ -750,6 +752,7 @@ usage: $execname [options] ([url] | [keywords]) --use-colors! : enable or disable the ANSI colors for text * Other + --invidious! : use the API of invidio.us to get the streaming URLs --http_proxy=s : HTTP proxy to use, format 'http://domain.tld:port/'. If authentication required, use 'http://user:pass\@domain.tld:port/' @@ -1051,6 +1054,7 @@ sub apply_configuration { publishedAfter publishedBefore safeSearch regionCode debug hl http_proxy page subscriptions_order + use_invidious_api ) ) { @@ -1427,6 +1431,7 @@ sub parse_arguments { 'dash!' => \$opt{dash_support}, 'confirm!' => \$opt{confirm}, + 'invidious!' => \$opt{use_invidious_api}, 'convert-command|convert-cmd=s' => \$opt{convert_cmd}, 'dash-m4a|dash-mp4-audio|dash-mp4a!' => \$opt{dash_mp4_audio}, 'wget-dl|wget-download!' => \$opt{download_with_wget}, diff --git a/lib/WWW/YoutubeViewer/Videos.pm b/lib/WWW/YoutubeViewer/Videos.pm index 58c5c40..a841dc0 100644 --- a/lib/WWW/YoutubeViewer/Videos.pm +++ b/lib/WWW/YoutubeViewer/Videos.pm @@ -97,7 +97,7 @@ sub send_rating_to_video { if ($rating eq 'none' or $rating eq 'like' or $rating eq 'dislike') { my $url = $self->_simple_feeds_url('videos/rate', id => $video_id, rating => $rating); - return defined($self->lwp_post($url, $self->_get_lwp_header())); + return defined($self->lwp_post($url, $self->_auth_lwp_header())); } return; diff --git a/lib/WWW/YoutubeViewer.pm b/lib/WWW/YoutubeViewer.pm index 11dc9d9..d5fba12 100644 --- a/lib/WWW/YoutubeViewer.pm +++ b/lib/WWW/YoutubeViewer.pm @@ -83,6 +83,8 @@ my %valid_options = ( lwp_env_proxy => {valid => [1, 0], default => 1}, escape_utf8 => {valid => [1, 0], default => 0}, + use_invidious_api => {valid => [1, 0], default => 0}, + # OAuth stuff client_id => {valid => [qr/^.{5}/], default => undef}, client_secret => {valid => [qr/^.{5}/], default => undef}, @@ -229,15 +231,15 @@ sub set_lwp_useragent { or $response->request->method ne 'GET' # cache only GET requests # don't cache if "cache-control" specifies "max-age=0" or "no-store" - or $response->header('cache-control') =~ /\b(?:max-age=0|no-store)\b/ + or (($response->header('cache-control') // '') =~ /\b(?:max-age=0|no-store)\b/) # don't cache video or audio files - or $response->header('content-type') =~ /\b(?:video|audio)\b/; + or (($response->header('content-type') // '') =~ /\b(?:video|audio)\b/); }, recache_if => sub { my ($response, $path) = @_; - not($response->is_fresh) # recache if the response expired + not($response->is_fresh) # recache if the response expired or ($response->code == 404 && -M $path > 1); # recache any 404 response older than 1 day } ) @@ -247,10 +249,10 @@ sub set_lwp_useragent { ); require LWP::ConnCache; - my $cache = LWP::ConnCache->new; + state $cache = LWP::ConnCache->new; $cache->total_capacity(undef); # no limit - my $accepted_encodings = do { + state $accepted_encodings = do { require HTTP::Message; HTTP::Message::decodable(); }; @@ -287,7 +289,7 @@ sub prepare_access_token { return; } -sub _get_lwp_header { +sub _auth_lwp_header { my ($self) = @_; my %lwp_header; @@ -310,7 +312,7 @@ sub lwp_get { $url // return; $self->{lwp} // $self->set_lwp_useragent(); - my %lwp_header = ($simple ? () : $self->_get_lwp_header); + my %lwp_header = ($simple ? () : $self->_auth_lwp_header); my $response = $self->{lwp}->get($url, %lwp_header); if ($response->is_success) { @@ -324,7 +326,7 @@ sub lwp_get { $self->set_access_token($refresh_token->{access_token}); # Don't be tempted to use recursion here, because bad things will happen! - $response = $self->{lwp}->get($url, $self->_get_lwp_header); + $response = $self->{lwp}->get($url, $self->_auth_lwp_header); if ($response->is_success) { $self->save_authentication_tokens(); @@ -525,38 +527,89 @@ sub get_video_tops { ... # NEEDS WORK!!! } -sub _get_formats_from_ytdl { +sub _extract_from_invidious { my ($self, $videoID) = @_; - ((state $x = $self->proxy_system('youtube-dl', '--version')) == 0) + my $url = sprintf("https://invidio.us/api/v1/videos/%s?fields=formatStreams,adaptiveFormats", $videoID); + + (my $resp = $self->{lwp}->get($url))->is_success() || return; + my $json = $resp->decoded_content() // return; + my $ref = $self->parse_json_string($json) // return; + + my @formats; + + # The entries are already in the format that we want. + if (exists($ref->{adaptiveFormats}) and ref($ref->{adaptiveFormats}) eq 'ARRAY') { + push @formats, @{$ref->{adaptiveFormats}}; + } + + if (exists($ref->{formatStreams}) and ref($ref->{formatStreams}) eq 'ARRAY') { + push @formats, @{$ref->{formatStreams}}; + } + + return @formats; +} + +sub _fallback_extract_urls { + my ($self, $videoID) = @_; + + my @array; + + if ($self->get_use_invidious_api) { # use the API of invidio.us + + if ($self->get_debug) { + say STDERR ":: Using invidio.us to extract the streaming URLs..."; + } + + push @array, $self->_extract_from_invidious($videoID); + + if ($self->get_debug) { + say STDERR ":: Found ", scalar(@array), " streaming URLs."; + } + + if (@array) { + return @array; + } + } + + ((state $x = $self->proxy_system('hypervideo', '--version')) == 0) || return; - my $json = $self->proxy_stdout('youtube-dl', '--all-formats', '--dump-single-json', + if ($self->get_debug) { + say STDERR ":: Using hypervideo to extract the streaming URLs..."; + } + + my $json = $self->proxy_stdout('hypervideo', '--all-formats', '--dump-single-json', quotemeta("https://www.youtube.com/watch?v=" . $videoID)); - my @array; my $ref = $self->parse_json_string($json) // return; + if (ref($ref) eq 'HASH' and exists($ref->{formats}) and ref($ref->{formats}) eq 'ARRAY') { foreach my $format (@{$ref->{formats}}) { if (exists($format->{format_id}) and exists($format->{url})) { - push @array, - { - itag => $format->{format_id}, - url => $format->{url}, - type => ( - ( - (defined($format->{format_note}) && $format->{format_note} eq 'DASH audio') - ? 'audio/' - : 'video/' - ) - . $format->{ext} - ), - }; + my $entry = { + itag => $format->{format_id}, + url => $format->{url}, + type => ( + ( + (defined($format->{format_note}) and $format->{format_note} eq 'DASH audio') + ? 'audio/' + : 'video/' + ) + . $format->{ext} + ), + }; + + push @array, $entry; } } } + if ($self->get_debug) { + say STDERR ":: Found ", scalar(@array), " streaming URLs."; + } + return @array; } @@ -638,7 +691,7 @@ sub _extract_streaming_urls { foreach my $video (@result) { if (exists $video->{s}) { # has an encrypted signature :( - my @formats = $self->_get_formats_from_ytdl($videoID); + my @formats = $self->_fallback_extract_urls($videoID); foreach my $format (@formats) { foreach my $ref (@result) { @@ -675,9 +728,9 @@ Returns a list of streaming URLs for a videoID. sub get_streaming_urls { my ($self, $videoID) = @_; - my $url = ($self->get_video_info_url() . sprintf($self->get_video_info_args(), $videoID)); + my $url = ($self->get_video_info_url() . sprintf($self->get_video_info_args(), $videoID)); my $content = $self->lwp_get($url) // return; - my %info = $self->parse_query_string($content); + my %info = $self->parse_query_string($content); my @streaming_urls = $self->_extract_streaming_urls(\%info, $videoID); @@ -700,9 +753,9 @@ sub get_streaming_urls { Data::Dump::pp(\@caption_urls); } - # Try again with youtube-dl + # Try again with hypervideo if (!@streaming_urls or $info{status} !~ /ok/i) { - @streaming_urls = $self->_get_formats_from_ytdl($videoID); + @streaming_urls = $self->_fallback_extract_urls($videoID); } return (\@streaming_urls, \@caption_urls, \%info);