#!/usr/bin/perl use strict; use warnings; use File::Basename; use Date::Parse; use Date::Format; use LWP::Simple; use URI::Escape; use JSON; use XML::LibXML; use IPC::System::Simple qw( system ); use WWW::YouTube::Download; use PrettyPrint; # Paramters my $CURL_BIN = 'curl'; my @CURL_ARGS = ('-4', '--insecure', '-C', '-'); my $BATCH_SIZE = 50; my $MAX_INDEX = 500; my $URL_PREFIX = 'http://gdata.youtube.com/feeds/api/users/'; my $URL_SUFFIX = '/uploads'; my %CHAN_PARAMS = ( 'strict' => 1, 'v' => 2, 'alt' => 'json', ); my %LIST_PARAMS = ( 'strict' => 1, 'v' => 2, 'alt' => 'jsonc', 'orderby' => 'published', ); # Prototypes sub findVideos($); sub findFiles($); sub ytURL($); sub buildNFO($); sub buildSeriesNFO($); sub getChannel($); sub fetchParse($); sub buildURL($$); sub saveString($$); # Sanity check if (scalar(@ARGV) < 1) { die('Usage: ' . basename($0) . " output_directory\n"); } # Command-line parameters my ($dir) = @ARGV; $dir =~ s/\/+$//; if (!-d $dir) { die('Invalid output directory: ' . $dir . "\n"); } my $user = basename($dir); if (length($user) < 1 || !($user =~ /^\w+$/)) { die('Invalid user: ' . $user . "\n"); } # Environmental parameters my $DEBUG = 0; if ($ENV{'DEBUG'}) { if ($ENV{'DEBUG'} =~ /(\d+)/) { $DEBUG = $1; } else { $DEBUG = 1; } } my $NO_FETCH = 0; if ($ENV{'NO_FETCH'}) { $NO_FETCH = 1; } if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) { $MAX_INDEX = $1; } if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) { $BATCH_SIZE = $1; } # Construct globals $LIST_PARAMS{'max-results'} = $BATCH_SIZE; if (!$DEBUG) { push(@CURL_ARGS, '--silent'); } # Grab the channel data my $channel = getChannel($user); # Create the channel NFO and poster, if needed my $nfo = $dir . '/tvshow.nfo'; if (!-e $nfo) { if ($DEBUG) { print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n"; } # Save the poster if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) { my $poster = $dir . '/poster.jpg'; my $jpg = get($channel->{'thumbnail'}); saveString($poster, $jpg); } # Save the series NFO my $xml = buildSeriesNFO($channel); if ($DEBUG > 1) { print STDERR 'Saving NFO: ' . $xml . "\n"; } saveString($nfo, $xml); } # Find all the user's videos on YT my $videos = findVideos($user); # Find all existing YT files on disk my $files = findFiles($dir); # Fill in missing videos and NFOs foreach my $id (keys(%{$videos})) { my $basePath = $dir . '/' . sprintf('%02d', $videos->{$id}->{'number'}) . ' - ' . $id . '.'; $nfo = $basePath . 'nfo'; # If we haven't heard of the file, or don't have an NFO for it # Checking for the NFO allows use to resume failed downloads if (!exists($files->{$id}) || !-e $nfo) { if ($DEBUG) { print STDERR 'Fetching video: ' . $id . "\n"; } # Find the download URL and file suffix my ($url, $suffix) = ytURL($id); if (!defined($url) || length($url) < 5) { warn('Could not determine URL for video: ' . $id . "\n"); next; } # Fetch with cURL # I know LWP exists (and is even loaded) but cURL makes my life easier { my @cmd = ($CURL_BIN); push(@cmd, @CURL_ARGS); push(@cmd, '-o', $basePath . $suffix, $url); if ($NO_FETCH) { print STDERR 'Not fetching: ' . join(' ', @cmd) . "\n"; } else { if ($DEBUG > 1) { print STDERR join(' ', @cmd) . "\n"; } system(@cmd); } } # Build and save the XML document my $xml = buildNFO($videos->{$id}); if ($DEBUG > 1) { print STDERR 'Saving NFO: ' . $xml . "\n"; } saveString($nfo, $xml); } } sub saveString($$) { my ($path, $str) = @_; my $fh = undef(); if (!open($fh, '>', $path)) { warn('Cannot open file for writing: ' . $! . "\n"); return undef(); } print $fh $str; close($fh); return 1; } sub buildSeriesNFO($) { my ($channel) = @_; # Create an XML tree my $doc = XML::LibXML::Document->new('1.0', 'UTF-8'); $doc->setStandalone(1); my $show = $doc->createElement('tvshow'); $doc->setDocumentElement($show); my $elm; # Add data $elm = $doc->createElement('title'); $elm->appendText($channel->{'title'}); $show->appendChild($elm); $elm = $doc->createElement('premiered'); $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'})); $show->appendChild($elm); $elm = $doc->createElement('plot'); $elm->appendText($channel->{'description'}); $show->appendChild($elm); # Return the string return $doc->toString(); } sub buildNFO($) { my ($video) = @_; # Create an XML tree my $doc = XML::LibXML::Document->new('1.0', 'UTF-8'); $doc->setStandalone(1); my $show = $doc->createElement('episodedetails'); $doc->setDocumentElement($show); my $elm; # Add data $elm = $doc->createElement('season'); $elm->appendText('1'); $show->appendChild($elm); $elm = $doc->createElement('episode'); $elm->appendText($video->{'number'}); $show->appendChild($elm); $elm = $doc->createElement('title'); $elm->appendText($video->{'title'}); $show->appendChild($elm); $elm = $doc->createElement('aired'); $elm->appendText(time2str('%Y-%m-%d', $video->{'date'})); $show->appendChild($elm); $elm = $doc->createElement('plot'); $elm->appendText($video->{'description'}); $show->appendChild($elm); $elm = $doc->createElement('runtime'); $elm->appendText($video->{'duration'}); $show->appendChild($elm); $elm = $doc->createElement('rating'); $elm->appendText($video->{'rating'}); $show->appendChild($elm); $elm = $doc->createElement('director'); $elm->appendText($video->{'creator'}); $show->appendChild($elm); # Return the string return $doc->toString(); } sub ytURL($) { my ($id) = @_; # Init the YT object my $tube = WWW::YouTube::Download->new; # Fetch metadata my $meta = $tube->prepare_download($id); if (!defined($meta) || ref($meta) ne 'HASH' || !exists($meta->{'video_url_map'}) || ref($meta->{'video_url_map'}) ne 'HASH') { return undef(); } # Find the best stream (i.e. highest resolution, prefer mp4) my $bestStream = undef(); my $bestResolution = 0; foreach my $streamID (keys(%{ $meta->{'video_url_map'} })) { my $stream = $meta->{'video_url_map'}->{$streamID}; my ($res) = $stream->{'resolution'} =~ /^\s*(\d+)/; if ($DEBUG > 1) { print STDERR $streamID . ' (' . $stream->{'suffix'} . ')' . ' => ' . $stream->{'resolution'} . ' : ' . $stream->{'url'} . "\n"; } if ( ($res > $bestResolution) || ($res == $bestResolution && defined($bestStream) && $bestStream->{'suffix'} ne 'mp4')) { $bestStream = $stream; $bestResolution = $res; } } if (!exists($bestStream->{'suffix'}) || length($bestStream->{'suffix'}) < 2) { $bestStream->{'suffix'} = 'mp4'; } return ($bestStream->{'url'}, $bestStream->{'suffix'}); } sub findFiles($) { my ($dir) = @_; my %files = (); # Read the output directory my $fh = undef(); opendir($fh, $dir) or die('Unable to open output directory: ' . $! . "\n"); while (my $file = readdir($fh)) { my ($num, $id, $suffix) = $file =~ /(\d+) - ([\w\-]+)\.(\w\w\w)$/; if (defined($id) && length($id) > 0) { if ($suffix eq 'nfo') { next; } my %tmp = ( 'number' => $num, 'suffix' => $suffix, 'path' => $dir . '/' . $file, ); $tmp{'nfo'} = $tmp{'path'}; $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/; $files{$id} = \%tmp; } } close($fh); if ($DEBUG) { print STDERR "\nFound " . scalar(keys(%files)) . " files:\n\t" . join("\n\t", keys(%files)) . "\n"; if ($DEBUG > 1) { print STDERR prettyPrint(\%files, "\t") . "\n"; } } return \%files; } sub buildURL($$) { my ($base, $params) = @_; my $url = $base . '?'; foreach my $name (keys(%{$params})) { $url .= '&' . uri_escape($name) . '=' . uri_escape($params->{$name}); } return $url; } sub fetchParse($) { my ($url) = @_; # Fetch if ($DEBUG) { print STDERR 'Fetching list URL: ' . $url . "\n"; } my $content = get($url); if (!defined($content) || length($content) < 10) { die('Invalid content from URL: ' . $url . "\n"); } # Parse my $data = decode_json($content); if (!defined($data) || ref($data) ne 'HASH') { die('Invalid JSON: ' . $content . "\n"); } if ($DEBUG > 2) { print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n"; } return $data; } sub getChannel($) { my ($user) = @_; # Build, fetch, parse my $url = buildURL($URL_PREFIX . $user, \%CHAN_PARAMS); my $data = fetchParse($url); if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') { die("Invalid channel data\n"); } $data = $data->{'entry'}; # Extract the data we want my %channel = ( 'id' => $data->{'yt$channelId'}->{'$t'}, 'title' => $data->{'title'}->{'$t'}, 'date' => str2time($data->{'published'}->{'$t'}), 'description' => $data->{'summary'}->{'$t'}, 'thumbnail' => $data->{'media$thumbnail'}->{'url'}, ); return \%channel; } sub findVideos($) { my ($user) = @_; my %videos = (); # Prepare the static URL elements my $baseURL = $URL_PREFIX . $user . $URL_SUFFIX; my %params = %LIST_PARAMS; # Loop through until we have all the entries my $index = 1; my $itemCount = undef(); LOOP: { # Build, fetch, parse $params{'start-index'} = $index; my $url = buildURL($baseURL, \%params); my $data = fetchParse($url); # Grab the total count, so we know when to stop if (!exists($data->{'data'})) { die("Invalid video list\n"); } $data = $data->{'data'}; if (!defined($itemCount) && exists($data->{'totalItems'})) { $itemCount = $data->{'totalItems'}; } # Process each item if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') { die("Invalid video list\n"); } my $items = $data->{'items'}; my $offset = 0; foreach my $item (@{$items}) { my %tmp = ( 'number' => ($itemCount - $index - $offset + 1), 'title' => $item->{'title'}, 'date' => str2time($item->{'uploaded'}), 'description' => $item->{'description'}, 'duration' => $item->{'duration'}, 'rating' => $item->{'rating'}, 'creator' => $item->{'uploader'}, ); $videos{ $item->{'id'} } = \%tmp; $offset++; } # Loop if there are results left to fetch $index += $BATCH_SIZE; if (defined($itemCount) && $itemCount > $index) { # But don't go past the max supported index if ($index <= $MAX_INDEX) { redo LOOP; } } } if ($DEBUG) { print STDERR "\nFound " . scalar(keys(%videos)) . ' videos for YouTube user ' . $user . ":\n\t" . join("\n\t", keys(%videos)) . "\n"; if ($DEBUG > 1) { print STDERR prettyPrint(\%videos, "\t") . "\n"; } } return \%videos; }