| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373 |
- #!/usr/bin/perl
- use strict;
- use warnings;
- use FindBin;
- use lib "$FindBin::Bin/";
- use File::Touch;
- use File::Basename;
- use Date::Parse;
- use Date::Format;
- use LWP::UserAgent;
- use URI::Escape;
- use JSON;
- use XML::LibXML;
- use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
- use IPC::Cmd qw( can_run );
- use PrettyPrint;
- # Paramters
- my %USERS = ('profplump' => 'UCkj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'UChfwMHzkPXOOFDce5hyQkTA');
- my $EXTRAS_FILE = 'extra_videos.ini';
- my $EXCLUDES_FILE = 'exclude_videos.ini';
- my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
- my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
- my @YTDL_QUIET = ('--quiet', '--no-warnings');
- my @YTDL_DEBUG = ('--verbose');
- my $BATCH_SIZE = 50;
- my $MAX_INDEX = 25000;
- my $FETCH_LIMIT = 50;
- my $DELAY = 0.5;
- my $HTTP_TIMEOUT = 10;
- my $HTTP_UA = 'ZachBot/1.0 (Plex)';
- my $HTTP_VERIFY = 0;
- my $API_URL = 'https://www.googleapis.com/youtube/v3/';
- my $API_KEY = $ENV{'YT_API_KEY'};
- my $API_ST_REL = 0.50;
- my $API_ST_ABS = 5;
- my %VIDEO_FIELDS = (
- 'id' => '',
- 'title' => '',
- 'description' => '',
- 'creator' => '',
- 'publishedAt' => '',
- 'channelId' => '',
- 'rawDuration' => '',
- 'date', => 0,
- 'season' => 0,
- 'duration' => 0,
- );
- my %API = (
- #https://www.googleapis.com/youtube/v3/channels?part=id&forUsername={channelName}&key={YOUR_API_KEY}
- 'channelID' => {
- 'url' => $API_URL . 'channels',
- 'params' => {
- 'part' => 'id',
- 'key' => $API_KEY,
- },
- },
- #https://www.googleapis.com/youtube/v3/channels?part=snippet&id={channelID}&key={YOUR_API_KEY}
- 'channel' => {
- 'url' => $API_URL . 'channels',
- 'params' => {
- 'part' => 'snippet',
- 'key' => $API_KEY,
- },
- },
- #https://www.googleapis.com/youtube/v3/search?part=snippet&channelId={channelID}&maxResults={maxResults}&pageToken={foo}&safeSearch=none&type=video&key={YOUR_API_KEY}
- 'search' => {
- 'url' => $API_URL . 'search',
- 'params' => {
- 'part' => 'id',
- 'key' => $API_KEY,
- 'maxResults' => 5,
- 'safeSearch' => 'none',
- 'type' => 'video',
- 'order' => 'date',
- },
- },
- #https://www.googleapis.com/youtube/v3/videos?part=snippet&id={videoID}&key={YOUR_API_KEY}
- 'video' => {
- 'url' => $API_URL . 'videos',
- 'params' => {
- 'part' => 'snippet,contentDetails',
- 'key' => $API_KEY,
- },
- },
- #https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&channelId={channelID}&key={YOUR_API_KEY}
- 'subscriptions' => {
- 'url' => $API_URL . 'subscriptions',
- 'params' => {
- 'part' => 'snippet',
- 'key' => $API_KEY,
- 'maxResults' => 5,
- },
- },
- );
- # Prototypes
- sub getVideoData($);
- sub localMetadata($$);
- sub findVideos($);
- sub findFiles($);
- sub buildNFO($);
- sub buildSeriesNFO($);
- sub getSubscriptions($);
- sub saveSubscriptions($$);
- sub saveChannel($);
- sub getChannel($);
- sub fetchParse($$);
- sub saveString($$);
- sub readExcludes();
- sub readExtras();
- sub parseVideoData($);
- sub calcVideoData($);
- sub dropExcludes($);
- sub addExtras($);
- sub updateNFOData($$$);
- sub videoNumberStr($$);
- sub videoSE($$);
- sub videoPath($$$$);
- sub renameVideo($$$$$$);
- sub parseFilename($);
- sub fetch($$);
- # Sanity check
- if (scalar(@ARGV) < 1) {
- die('Usage: ' . basename($0) . " output_directory\n");
- }
- my $SUBSCRIPTIONS = 0;
- if ($0 =~ /subscription/i) {
- $SUBSCRIPTIONS = 1;
- }
- # Command-line parameters
- our ($DIR) = @ARGV;
- $DIR =~ s/\/+$//;
- if (!-d $DIR) {
- die('Invalid output directory: ' . $DIR . "\n");
- }
- our $NAME = basename($DIR);
- our $ID = $NAME;
- if ($NAME =~ /^(.*)\s\(([\w\-]+)\)$/) {
- $NAME = $1;
- $ID = $2;
- if (length($ID) < 1 || !($ID =~ /^[\w\-]+$/)) {
- die('Invalid channel ID: ' . $ID . "\n");
- }
- } elsif (!$SUBSCRIPTIONS) {
- die('Invalid folder: ' . $DIR . "\n");
- }
- # Move to the target directory so we can use relative paths later
- chdir($DIR) or die('Unable to chdir to: ' . $DIR . "\n");
- # Environmental parameters (debug)
- my $DEBUG = 0;
- if ($ENV{'DEBUG'}) {
- if ($ENV{'DEBUG'} =~ /(\d+)/) {
- $DEBUG = $1;
- } else {
- $DEBUG = 1;
- }
- $DELAY /= 2;
- }
- my $NO_FETCH = 0;
- if ($ENV{'NO_FETCH'}) {
- $NO_FETCH = 1;
- }
- my $NO_NFO = 0;
- if ($ENV{'NO_NFO'}) {
- $NO_NFO = 1;
- }
- my $NO_SEARCH = 0;
- if ($ENV{'NO_SEARCH'}) {
- $NO_SEARCH = 1;
- }
- my $NO_FILES = 0;
- if ($ENV{'NO_FILES'}) {
- $NO_FILES = 1;
- }
- my $NO_CHANNEL = 0;
- if ($ENV{'NO_CHANNEL'}) {
- $NO_CHANNEL = 1;
- }
- my $NO_EXTRAS = 0;
- if ($ENV{'NO_EXTRAS'}) {
- $NO_EXTRAS = 1;
- }
- my $NO_EXCLUDES = 0;
- if ($ENV{'NO_EXCLUDES'}) {
- $NO_EXCLUDES = 1;
- }
- my $NO_RENAME = 0;
- if ($ENV{'NO_RENAME'}) {
- $NO_RENAME = 1;
- }
- # Environmental parameters (functional)
- my $SUDO_CHATTR = 1;
- if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
- $SUDO_CHATTR = 0;
- }
- if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
- $MAX_INDEX = $1;
- }
- if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
- $BATCH_SIZE = $1;
- }
- if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
- $FETCH_LIMIT = $1;
- }
- # Construct globals
- my $LWP = undef();
- foreach my $key (keys(%API)) {
- if (exists($API{$key}{'params'}{'maxResults'})) {
- $API{$key}{'params'}{'maxResults'} = $BATCH_SIZE;
- }
- }
- if ($DEBUG > 1) {
- push(@YTDL_ARGS, @YTDL_DEBUG);
- } elsif (!$DEBUG) {
- push(@YTDL_ARGS, @YTDL_QUIET);
- }
- # Allow use as a subscription manager
- if ($SUBSCRIPTIONS) {
- my %subs = ();
- foreach my $user (keys(%USERS)) {
- my $tmp = getSubscriptions($USERS{$user});
- foreach my $sub (keys(%{$tmp})) {
- $subs{$sub} = $tmp->{$sub};
- }
- }
- saveSubscriptions($DIR, \%subs);
- exit(0);
- }
- # Disable channel fetching if the ID channel is "None"
- if ($ID =~ /^\s*None\s*$/i) {
- if ($DEBUG) {
- print STDERR 'Channel features disabled for: ' . $NAME . "\n";
- }
- $NO_CHANNEL = 1;
- $NO_SEARCH = 1;
- }
- # Run the local "update" script if any
- {
- my @update = ($DIR . '/update', '');
- if (-x $update[0]) {
- if ($DEBUG) {
- print STDERR "Executing local update script...\n";
- }
- system(@update);
- }
- }
- # Grab the channel data
- my $channel = {};
- if (!$NO_CHANNEL) {
- $channel = getChannel($ID);
- saveChannel($channel);
- }
- # Find all the user's videos on YT
- my $videos = {};
- if (!$NO_SEARCH) {
- $videos = findVideos($ID);
- }
- # Find any requested "extra" videos
- my $extras = {};
- if (!$NO_EXTRAS) {
- $extras = addExtras($videos);
- }
- # Drop any "excludes" videos
- my $excludes = {};
- if (!$NO_EXCLUDES) {
- $excludes = dropExcludes($videos);
- }
- # Find all existing YT files on disk
- my $files = {};
- if (!$NO_FILES) {
- $files = findFiles($videos);
- }
- # Deal with unknown videos
- foreach my $id (keys(%{$files})) {
- if ((!exists($videos->{$id}) || !$videos->{$id}->{'date'}) && $files->{$id}->{'season'} > 0) {
- # Check the ID specifically to find things the APIv3 fuzzy search misses
- if ($DEBUG) {
- print STDERR 'Validating local video by ID: ' . $id . "\n";
- }
- my $video = getVideoData([$id]);
- if ($video->[0] && $video->[0]->{'season'} > 0 && $video->[0]->{'channelId'} eq $ID) {
- $videos->{$id} = $video->[0];
- next;
- }
- # Whine when videos do go away
- warn('Local video not known to YT channel (' . $NAME . '): ' . $id . "\n");
- renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
- }
- }
- # Calculate the episode number using the publish dates
- {
- my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
- my $num = undef();
- my $lastSeason = undef();
- foreach my $id (@byDate) {
- if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
- $num = 1;
- $lastSeason = $videos->{$id}->{'season'};
- }
- $videos->{$id}->{'number'} = $num++;
- }
- }
- # Fill in missing videos and NFOs
- my $fetched = 0;
- FETCH_LOOP: foreach my $id (keys(%{$videos})) {
- if ($DEBUG > 1) {
- print STDERR 'Checking remote video: ' . $id . "\n";
- if (!exists($files->{$id})) {
- print STDERR "\tLocal file: <missing>\n";
- } else {
- print STDERR "\tLocal media: " . $files->{$id}->{'path'} . "\n";
- print STDERR "\tLocal NFO: " . $files->{$id}->{'nfo'} . "\n";
- }
- }
- my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
- # Determine if we may or must rename
- my $rename = 0;
- if (exists($files->{$id})) {
- # Rename if we drift
- if (!$rename && $files->{$id}->{'path'}) {
- if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
- || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
- {
- if ($DEBUG) {
- print STDERR "Rename due to drift\n";
- }
- $rename = 1;
- }
- }
- # Always rename if the NFO does not match the media
- if (!$rename && $files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
- my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
- if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
- if ($DEBUG) {
- print STDERR "Rename due to media-metadata mismatch\n";
- }
- $rename = 1;
- }
- }
- }
- # Warn (and optionally rename) as selected
- if ($rename) {
- print STDERR 'Video ' . $id . ' had number ' . $files->{$id}->{'number'} . ' but now has number ' . $videos->{$id}->{'number'} . "\n";
- renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
- }
- # If we haven't heard of the file, or don't have an NFO for it
- # Checking for the NFO allows use to resume failed downloads
- if (!exists($files->{$id}) || !-e $nfo) {
- if ($DEBUG) {
- print STDERR 'Fetching video: ' . $id . "\n";
- }
- # Let youtube-dl handle the URLs and downloading
- {
- my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
- my @name = ($YTDL_BIN);
- push(@name, @YTDL_ARGS);
- my @fetch = @name;
- push(@name, '--get-filename');
- push(@fetch, @args);
- push(@name, @args);
- if ($NO_FETCH) {
- print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
- } else {
- # Count fetch attempts (even if they fail later)
- $fetched++;
- if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
- print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $NAME . "\n";
- if ($DEBUG) {
- print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
- }
- exit 1;
- }
- # Find the output file name
- if ($DEBUG > 1) {
- print STDERR join(' ', @name) . "\n";
- }
- sleep($DELAY);
- my $file = capture(EXIT_ANY, @name);
- if ($EXITVAL != 0) {
- warn('Error executing youtube-dl for name: ' . $id . ' (' . $EXITVAL . ")\n");
- next FETCH_LOOP;
- }
- $file =~ s/^\s+//;
- $file =~ s/\s+$//;
- # Sanity check
- if (!$file) {
- warn('No file name available for video: ' . $id . "\n");
- next FETCH_LOOP;
- }
- # Download
- if ($DEBUG > 1) {
- print STDERR join(' ', @fetch) . "\n";
- }
- sleep($DELAY);
- my $exit = run(EXIT_ANY, @fetch);
- if ($exit != 0) {
- warn('Error executing youtube-dl for video: ' . $file . "\n");
- next FETCH_LOOP;
- }
- # Ensure we found something useful
- if (-e $file . '.part') {
- warn('Partial download detected: ' . $file . "\n");
- next FETCH_LOOP;
- }
- if (!-s $file) {
- warn('No output video file: ' . $file . "\n");
- next FETCH_LOOP;
- }
- # Touch the file to reflect the download time rather than the upload time
- touch($file);
- }
- }
- # Build and save the XML document
- my $xml = buildNFO($videos->{$id});
- if ($DEBUG > 1) {
- print STDERR 'Saving NFO: ' . $xml . "\n";
- }
- if ($NO_NFO) {
- print STDERR "Not saving NFO\n";
- } else {
- saveString($nfo, $xml);
- }
- }
- }
- sub saveString($$) {
- my ($path, $str) = @_;
- my $fh = undef();
- if (!open($fh, '>', $path)) {
- warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
- return undef();
- }
- print $fh $str;
- close($fh);
- return 1;
- }
- sub buildSeriesNFO($) {
- my ($channel) = @_;
- # Create an XML tree
- my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
- $doc->setStandalone(1);
- my $show = $doc->createElement('tvshow');
- $doc->setDocumentElement($show);
- my $elm;
- # Add data
- $elm = $doc->createElement('title');
- $elm->appendText($channel->{'title'});
- $show->appendChild($elm);
- $elm = $doc->createElement('premiered');
- $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
- $show->appendChild($elm);
- $elm = $doc->createElement('plot');
- $elm->appendText($channel->{'description'});
- $show->appendChild($elm);
- # Return the string
- return $doc->toString();
- }
- sub buildNFO($) {
- my ($video) = @_;
- # Create an XML tree
- my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
- $doc->setStandalone(1);
- my $show = $doc->createElement('episodedetails');
- $doc->setDocumentElement($show);
- my $file = $doc->createElement('fileinfo');
- $show->appendChild($file);
- my $stream = $doc->createElement('streamdetails');
- $file->appendChild($stream);
- my $stream_video = $doc->createElement('video');
- $stream->appendChild($stream_video);
- my $elm;
- # Add data
- $elm = $doc->createElement('season');
- $elm->appendText($video->{'season'});
- $show->appendChild($elm);
- $elm = $doc->createElement('episode');
- $elm->appendText(videoNumberStr($video->{'season'}, $video->{'number'}));
- $show->appendChild($elm);
- if (!defined($video->{'title'})) {
- $video->{'title'} = 'Episode ' . $video->{'number'};
- }
- $elm = $doc->createElement('title');
- $elm->appendText($video->{'title'});
- $show->appendChild($elm);
- if (!defined($video->{'date'})) {
- $video->{'date'} = time();
- }
- $elm = $doc->createElement('aired');
- $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
- $show->appendChild($elm);
- if (defined($video->{'description'})) {
- $elm = $doc->createElement('plot');
- $elm->appendText($video->{'description'});
- $show->appendChild($elm);
- }
- if (defined($video->{'duration'})) {
- $elm = $doc->createElement('runtime');
- $elm->appendText($video->{'duration'});
- $show->appendChild($elm);
- $elm = $doc->createElement('durationinseconds');
- $elm->appendText($video->{'duration'});
- $stream_video->appendChild($elm);
- }
- if (defined($video->{'creator'})) {
- $elm = $doc->createElement('director');
- $elm->appendText($video->{'creator'});
- $show->appendChild($elm);
- }
- # Return the string
- return $doc->toString();
- }
- sub findFiles($) {
- my ($videos) = @_;
- my %files = ();
- our $NAME;
- # Allow complete bypass
- if ($NO_FILES) {
- return \%files;
- }
- # Read the output directory
- my $fh = undef();
- opendir($fh, '.')
- or die($NAME . ': Unable to open files directory: ' . $! . "\n");
- while (my $file = readdir($fh)) {
- my ($season, $number, $id, $suffix) = parseFilename($file);
- if (defined($id) && length($id) > 0) {
- # Determine if there is another video with the same season/number
- my $exists = 0;
- if ( exists($videos->{$id})
- && exists($videos->{$id}->{'season'})
- && exists($videos->{$id}->{'number'})
- && $season == $videos->{$id}->{'season'}
- && $number == $videos->{$id}->{'number'})
- {
- $exists = 1;
- }
- # Create the record as needed
- if (!exists($files{$id})) {
- my %tmp = (
- 'season' => $season,
- 'number' => $number,
- );
- $files{$id} = \%tmp;
- }
- # Video or NFO?
- my $type = 'path';
- if ($suffix eq 'nfo') {
- $type = 'nfo';
- }
- # Deal with duplicates
- if (exists($files{$id}->{$type})) {
- if ($type eq 'nfo') {
- warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
- if (!$NO_RENAME) {
- my $del = $file;
- if ($exists) {
- $del = $files{$id}->{'nfo'};
- }
- warn("\tDeleting: " . $file . "\n");
- if ($SUDO_CHATTR) {
- system('sudo', 'chattr', '-i', $file);
- }
- unlink($file);
- next;
- }
- } else {
- warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
- if (!$NO_RENAME) {
- my $del = $file;
- if ($exists) {
- $del = $files{$id}->{'path'};
- } elsif ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
- $del = $files{$id}->{'path'};
- }
- warn("\tDeleting: " . $del . "\n");
- if ($SUDO_CHATTR) {
- system('sudo', 'chattr', '-i', $del);
- }
- unlink($del);
- next;
- }
- }
- }
- # Assign the component paths
- $files{$id}->{$type} = $file;
- if ($type ne 'nfo') {
- $files{$id}->{'suffix'} = $suffix;
- }
- }
- }
- close($fh);
- if ($DEBUG) {
- print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
- if ($DEBUG > 1) {
- print STDERR prettyPrint(\%files, "\t") . "\n";
- }
- }
- return \%files;
- }
- sub fetchParse($$) {
- my ($name, $params) = @_;
- our $NAME;
- my $url = $API{$name}{'url'} . '?';
- foreach my $key (keys(%{ $API{$name}{'params'} }), keys(%{$params})) {
- $url .= '&' . uri_escape($key) . '=';
- if (exists($params->{$key})) {
- $url .= uri_escape($params->{$key});
- } else {
- $url .= uri_escape($API{$name}{'params'}{$key});
- }
- }
- # Fetch
- if ($DEBUG > 1) {
- print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
- }
- sleep($DELAY);
- my $content;
- my $code = fetch($url, \$content);
- if ($code != 200 || !defined($content) || length($content) < 10) {
- my $msg = $NAME . ': Invalid content from URL (' . $code . '): ' . $url . "\n" . $content . "\n";
- if ($code >= 500 && $code < 600) {
- if ($DEBUG) {
- print STDERR $msg;
- }
- exit(0);
- }
- if ($code >= 400 && $code < 500) {
- if ($DEBUG) {
- print STDERR "Wainting 30 seconds after 400 error\n";
- }
- sleep(30);
- }
- die($msg);
- }
- # Parse
- if ($DEBUG > 2) {
- print STDERR "Raw JSON data:\n" . $content . "\n";
- }
- my $data = decode_json($content);
- if (!defined($data) || ref($data) ne 'HASH') {
- die($NAME . ': Invalid JSON: ' . $content . "\n");
- }
- return $data;
- }
- sub getSubscriptions($) {
- my ($id) = @_;
- our $NAME;
- # Loop through until we have all the entries
- my $pageToken = '';
- my %subs = ();
- SUBS_LOOP:
- {
- # Build, fetch, parse, check
- my $data = fetchParse('subscriptions', { 'channelId' => $id, 'pageToken' => $pageToken });
- $pageToken = '';
- if ( exists($data->{'pageInfo'})
- && ref($data->{'pageInfo'}) eq 'HASH'
- && exists($data->{'pageInfo'}->{'totalResults'})
- && $data->{'pageInfo'}->{'totalResults'} > 0
- && exists($data->{'pageInfo'}->{'resultsPerPage'})
- && exists($data->{'items'})
- && ref($data->{'items'}) eq 'ARRAY')
- {
- if (exists($data->{'nextPageToken'})) {
- $pageToken = $data->{'nextPageToken'};
- }
- $data = $data->{'items'};
- } else {
- die($NAME . ": Invalid subscription data\n");
- }
- foreach my $item (@{$data}) {
- if ( !exists($item->{'snippet'})
- || ref($item->{'snippet'}) ne 'HASH'
- || !exists($item->{'snippet'}->{'title'})
- || !exists($item->{'snippet'}->{'resourceId'})
- || ref($item->{'snippet'}->{'resourceId'}) ne 'HASH'
- || !exists($item->{'snippet'}->{'resourceId'}->{'channelId'}))
- {
- warn($NAME . ": Skipping invalid subscription\n");
- if ($DEBUG > 2) {
- print STDERR prettyPrint($item, "\t") . "\n";
- }
- next;
- }
- if ($DEBUG) {
- print STDERR $item->{'snippet'}->{'title'} . ' => ' . $item->{'snippet'}->{'resourceId'}->{'channelId'} . "\n";
- }
- $subs{ $item->{'snippet'}->{'resourceId'}->{'channelId'} } = $item->{'snippet'}->{'title'};
- }
- # Loop if there are results left to fetch
- if ($pageToken) {
- redo SUBS_LOOP;
- }
- }
- # Return the list of subscribed channel IDs
- return \%subs;
- }
- sub saveSubscriptions($$) {
- my ($folder, $subs) = @_;
- our $NAME;
- # Check for local subscriptions missing from YT
- my %locals = ();
- my $fh = undef();
- opendir($fh, '.')
- or die($NAME . ': Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
- while (my $file = readdir($fh)) {
- # Skip dotfiles
- if ($file =~ /^\./) {
- next;
- }
- # Skip non-directories
- if (!-d $file) {
- next;
- }
- # Extract the YTID
- my ($id) = $file =~ /\(([\w\-]+)\)$/;
- if (!$id) {
- warn('Skipping invalid local subscription: ' . $file . "\n");
- next;
- }
- # Skip "None" directories (i.e. non-channel subscriptions)
- if ($id eq 'None') {
- if ($DEBUG) {
- print STDERR 'Ignoring non-channel subscription: ' . $file . "\n";
- }
- next;
- }
- # Anything else should be in the list
- if (!$subs->{$id}) {
- my (undef(), undef(), $hour) = localtime(time);
- if ($hour == 0 || $DEBUG) {
- print STDERR 'Missing YT subscription for: ' . $file . "\n";
- }
- }
- # Note local subscriptions
- $locals{$id} = 1;
- }
- closedir($fh);
- # Check for YT subscriptions missing locally
- foreach my $id (keys(%{$subs})) {
- if (!exists($locals{$id})) {
- my $dir = $subs->{$id} . ' (' . $id . ')';
- print STDERR 'Adding local subscription for: ' . $dir . "\n";
- mkdir($folder . '/' . $dir);
- }
- }
- }
- sub saveChannel($) {
- my ($channel) = @_;
- my $nfo = 'tvshow.nfo';
- if (!-e $nfo) {
- if ($DEBUG) {
- print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
- }
- # Save the poster
- if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
- my $jpg;
- if (fetch($channel->{'thumbnail'}, \$jpg) == 200) {
- saveString('poster.jpg', $jpg);
- }
- }
- # Save the series NFO
- my $xml = buildSeriesNFO($channel);
- if ($DEBUG > 1) {
- print STDERR 'Saving NFO: ' . $xml . "\n";
- }
- saveString($nfo, $xml);
- }
- }
- sub getChannel($) {
- my ($id) = @_;
- our $NAME;
- # Build, fetch, parse, check
- my $data = fetchParse('channel', { 'id' => $id });
- if ( exists($data->{'pageInfo'})
- && ref($data->{'pageInfo'}) eq 'HASH'
- && exists($data->{'pageInfo'}->{'totalResults'})
- && $data->{'pageInfo'}->{'totalResults'} == 1
- && exists($data->{'items'})
- && ref($data->{'items'}) eq 'ARRAY'
- && scalar(@{ $data->{'items'} }) > 0
- && ref($data->{'items'}[0]) eq 'HASH'
- && exists($data->{'items'}[0]->{'snippet'})
- && ref($data->{'items'}[0]->{'snippet'}) eq 'HASH')
- {
- $data = $data->{'items'}[0]->{'snippet'};
- } else {
- die($NAME . ": Invalid channel data\n");
- }
- # Extract the data we want
- my %channel = (
- 'id' => $id,
- 'title' => $data->{'title'},
- 'date' => str2time($data->{'publishedAt'}),
- 'description' => $data->{'description'},
- 'thumbnail' => $data->{'thumbnails'}->{'high'}->{'url'},
- );
- return \%channel;
- }
- sub readExcludes() {
- my %excludes = ();
- our $NAME;
- # Read and parse the excludes videos file, if it exists
- if (-e $EXCLUDES_FILE) {
- my $fh;
- open($fh, $EXCLUDES_FILE)
- or die($NAME . ': Unable to open excludes videos file: ' . $! . "\n");
- while (<$fh>) {
- # Skip blank lines and comments
- if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
- next;
- }
- # Match our specific format or whine
- if ($_ =~ /^\s*([\w\-]+)\s*$/) {
- if ($DEBUG > 1) {
- print STDERR 'Adding exclude video: ' . $1 . "\n";
- }
- $excludes{$1} = 1;
- } else {
- print STDERR 'Skipped exclude video line: ' . $_;
- }
- }
- close($fh);
- }
- return \%excludes;
- }
- sub readExtras() {
- my %extras = ();
- our $NAME;
- # Read and parse the extra videos file, if it exists
- if (-e $EXTRAS_FILE) {
- my $fh;
- open($fh, $EXTRAS_FILE)
- or die($NAME . ': Unable to open extra videos file: ' . $! . "\n");
- while (<$fh>) {
- # Skip blank lines and comments
- if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
- next;
- }
- # Match our specific format or whine
- if ($_ =~ /^\s*([\w\-]+)\s*$/) {
- if ($DEBUG > 1) {
- print STDERR 'Adding extra video: ' . $1 . "\n";
- }
- $extras{$1} = 1;
- } else {
- print STDERR 'Skipped extra video line: ' . $_;
- }
- }
- close($fh);
- }
- return \%extras;
- }
- sub parseVideoData($) {
- my ($data) = @_;
- my %video = %VIDEO_FIELDS;
- # Sanity check
- if ( !exists($data->{'id'})
- || !exists($data->{'snippet'})
- || ref($data->{'snippet'}) ne 'HASH'
- || !exists($data->{'contentDetails'})
- || ref($data->{'contentDetails'}) ne 'HASH'
- || !exists($data->{'contentDetails'}->{'duration'}))
- {
- if ($DEBUG > 2) {
- print STDERR "Unable to parse video data:\n" . prettyPrint($data, "\t") . "\n";
- }
- return undef();
- }
- # Direct extraction
- $video{'id'} = $data->{'id'};
- $video{'title'} = $data->{'snippet'}->{'title'};
- $video{'description'} = $data->{'snippet'}->{'description'};
- $video{'creator'} = $data->{'snippet'}->{'channelTitle'};
- $video{'publishedAt'} = $data->{'snippet'}->{'publishedAt'};
- $video{'channelId'} = $data->{'snippet'}->{'channelId'};
- $video{'rawDuration'} = $data->{'contentDetails'}->{'duration'};
- # Convert dates and whatnot
- calcVideoData(\%video);
- return \%video;
- }
- sub calcVideoData($) {
- my ($video) = @_;
- if ($video->{'publishedAt'}) {
- $video->{'date'} = str2time($video->{'publishedAt'});
- }
- if ($video->{'date'}) {
- $video->{'season'} = time2str('%Y', $video->{'date'});
- } else {
- $video->{'season'} = 0;
- }
- if ($video->{'rawDuration'} =~ /PT(\d+)M(\d+)S/) {
- $video->{'duration'} = ($1 * 60) + $2;
- } elsif ($video->{'rawDuration'} =~ /PT(\d+)S/) {
- $video->{'duration'} = $1;
- }
- }
- sub dropExcludes($) {
- my ($videos) = @_;
- my $excludes = readExcludes();
- foreach my $id (keys(%{$excludes})) {
- if (!exists($videos->{$id})) {
- if ($DEBUG > 1) {
- print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
- }
- next;
- }
- delete($videos->{$id});
- }
- return $excludes;
- }
- sub addExtras($) {
- my ($videos) = @_;
- my $extras = readExtras();
- foreach my $id (keys(%{$extras})) {
- if (exists($videos->{$id})) {
- if ($DEBUG > 1) {
- print STDERR 'Skipping known extra video: ' . $id . "\n";
- }
- next;
- }
- my $video = getVideoData([$id]);
- $videos->{$id} = $video->[0];
- }
- return $extras;
- }
- sub getVideoData($) {
- my ($ids) = @_;
- my @videos = ();
- our $NAME;
- my %found = ();
- # Do a batch request for the entire batch of video data
- my $data = fetchParse('video', { 'id' => join(',', @{$ids}) });
- if ( exists($data->{'pageInfo'})
- && ref($data->{'pageInfo'}) eq 'HASH'
- && exists($data->{'pageInfo'}->{'totalResults'})
- && exists($data->{'items'})
- && ref($data->{'items'}) eq 'ARRAY')
- {
- if (scalar(@{$ids}) > 1 && $data->{'pageInfo'}->{'totalResults'} != scalar(@{$ids})) {
- die($NAME . ': Video/metadata search count mismatch: ' . $data->{'pageInfo'}->{'totalResults'} . '/' . scalar(@{$ids}) . "\n");
- }
- $data = $data->{'items'};
- } else {
- die($NAME . ": Invalid search video data\n" . prettyPrint($data, "\t") . "\n");
- }
- # Build each metadata record
- foreach my $item (@{$data}) {
- if (!exists($item->{'snippet'}) || ref($item->{'snippet'}) ne 'HASH') {
- warn($NAME . ": Skipping invalid metadata item\n");
- if ($DEBUG > 2) {
- print STDERR prettyPrint($item, "\t") . "\n";
- }
- next;
- }
- my $video = parseVideoData($item);
- if (!$video) {
- die($NAME . ": Unable to parse video data\n");
- }
- # Allow local metadata overrides
- localMetadata($video->{'id'}, $video);
- # Save
- $found{ $video->{'id'} } = 1;
- push(@videos, $video);
- }
- # Load local metadata even for videos YT doesn't know anything about
- foreach my $id (@{$ids}) {
- if ($found{$id}) {
- next;
- }
- my %video = %VIDEO_FIELDS;
- $video{'id'} = $id;
- localMetadata($id, \%video);
- # Save
- $found{$id} = 1;
- push(@videos, \%video);
- }
- # Parse the video data
- return \@videos;
- }
- sub localMetadata($$) {
- my ($id, $video) = @_;
- my $meta = $id . '.meta';
- if (!-r $meta) {
- return;
- }
- my $fh = undef();
- open($fh, '<', $meta)
- or warn('Unable to open local metadata file: ' . $meta . ': ' . $! . ". Skipping...\n");
- while (<$fh>) {
- if (/^\s*#/ || /^\s*$/) {
- next;
- }
- if (/^\s*([^\s\=][^\=]*[^\s\=])\s*=\>?\s*(\S.*\S)\s*$/) {
- if (exists($video->{$1})) {
- if ($DEBUG) {
- print STDERR 'Using local metadata line (' . $id . '): ' . $1 . ' => ' . $2 . "\n";
- }
- $video->{$1} = $2;
- } else {
- die('Invalid local metadata attribute (' . $1 . '): ' . $meta . "\n");
- }
- } else {
- die('Invalid local metdata line (' . $meta . '): ' . chomp($_) . "\n");
- }
- }
- close($fh);
- }
- sub findVideos($) {
- my ($id) = @_;
- my %videos = ();
- our $NAME;
- # Allow complete bypass
- if ($NO_SEARCH) {
- return \%videos;
- }
- # Loop through until we have all the entries
- my $totalCount = 0;
- my %params = ('channelId' => $id);
- LOOP:
- {
- # Loop status
- my $count = scalar(keys(%videos));
- if ($DEBUG && $totalCount) {
- print STDERR 'Fetching ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%)\n";
- }
- # Build, fetch, parse, check
- my $data = fetchParse('search', \%params);
- delete($params{'pageToken'});
- if ( exists($data->{'pageInfo'})
- && ref($data->{'pageInfo'}) eq 'HASH'
- && exists($data->{'pageInfo'}->{'totalResults'})
- && $data->{'pageInfo'}->{'totalResults'} > 0
- && exists($data->{'pageInfo'}->{'resultsPerPage'})
- && exists($data->{'items'})
- && ref($data->{'items'}) eq 'ARRAY')
- {
- if (!$totalCount) {
- $totalCount = $data->{'pageInfo'}->{'totalResults'};
- }
- $data = $data->{'items'};
- } else {
- die($NAME . ": Invalid search data\n");
- }
- # Sanity check
- if (scalar(@{$data}) < 1) {
- last LOOP;
- }
- # Grab each video ID
- my @ids = ();
- foreach my $item (@{$data}) {
- if (!exists($item->{'id'}) || ref($item->{'id'}) ne 'HASH' || !exists($item->{'id'}->{'videoId'})) {
- warn($NAME . ": Skipping invalid video item\n");
- if ($DEBUG > 2) {
- print STDERR prettyPrint($data, "\t") . "\n";
- }
- next;
- }
- push(@ids, $item->{'id'}->{'videoId'});
- }
- # Do a batch request for the entire batch of video data
- my $more = 0;
- my $records = getVideoData(\@ids);
- foreach my $rec (@{$records}) {
- if (exists($rec->{'duration'}) && $rec->{'duration'} == 0) {
- if ($DEBUG > 1) {
- print STDERR 'Skipping 0-length video: ' . $rec->{'id'} . "\n";
- }
- next;
- }
- $videos{ $rec->{'id'} } = $rec;
- if (!defined($params{'publishedBefore'}) || (defined($rec->{'publishedAt'}) && ($rec->{'publishedAt'} cmp $params{'publishedBefore'}) < 0)) {
- $params{'publishedBefore'} = $rec->{'publishedAt'};
- $more = 1;
- }
- }
- # Loop if there are results left to fetch
- if ($more && $params{'publishedBefore'} && $count <= $MAX_INDEX) {
- redo LOOP;
- }
- }
- # Sanity check/debug
- {
- my $count = scalar(keys(%videos));
- if ($DEBUG) {
- print STDERR 'Found ' . $count . " remote videos\n";
- if ($DEBUG > 1) {
- print STDERR prettyPrint(\%videos, "\t") . "\n";
- }
- }
- if (abs(1 - ($count / $totalCount)) > $API_ST_REL && abs($totalCount - $count) > $API_ST_ABS) {
- die($NAME . ': Found only ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%) remote videos. Aborting...\n");
- }
- }
- return \%videos;
- }
- sub updateNFOData($$$) {
- my ($file, $season, $episode) = @_;
- my $nfoData = undef();
- {
- open(my $fh, '<', $file)
- or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
- $/ = undef;
- $nfoData = <$fh>;
- close($fh);
- }
- if ($nfoData) {
- $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
- $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
- }
- return $nfoData;
- }
- sub videoNumberStr($$) {
- my ($season, $episode) = @_;
- my $str = '';
- if ($season == 0 && $episode =~ /^(20[012]\d)(\d{2,4})$/) {
- $str = sprintf('%04d%04d', $1, $2);
- } else {
- $str = sprintf('%02d', $episode);
- }
- return $str;
- }
- sub videoSE($$) {
- my ($season, $episode) = @_;
- return sprintf('S%0dE%s - ', $season, videoNumberStr($season, $episode));
- }
- sub videoPath($$$$) {
- my ($season, $episode, $id, $ext) = @_;
- return videoSE($season, $episode) . $id . '.' . $ext;
- }
- sub renameVideo($$$$$$) {
- my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
- our $NAME;
- # Bail if disabled
- if ($NO_RENAME) {
- print STDERR 'Not renaming: ' . $video . "\n";
- return;
- }
- # General sanity checks
- if (!defined($id) || !defined($season) || !defined($episode)) {
- die($NAME . ": Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
- }
- # Warning about useless calls
- if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
- warn("No NFO or video path provided in renameVideo()\n");
- return;
- }
- # Video
- if (defined($video) && defined($suffix)) {
- my $videoNew = videoPath($season, $episode, $id, $suffix);
- # Sanity checks, as we do dangerous work here
- if (!-r $video) {
- die($NAME . ': Invalid video source in rename: ' . $video . "\n");
- }
- if ($video ne $videoNew) {
- print STDERR 'Renaming ' . $NAME . '/' . $video . ' => ' . $videoNew . "\n";
- if (-e $videoNew) {
- die($NAME . ': Rename: Target exists: ' . $videoNew . "\n");
- }
- if ($SUDO_CHATTR) {
- system('sudo', 'chattr', '-i', $video);
- }
- rename($video, $videoNew)
- or die($NAME . ': Unable to rename: ' . $video . ': ' . $! . "\n");
- }
- }
- # NFO
- if (defined($nfo)) {
- my $nfoNew = videoPath($season, $episode, $id, 'nfo');
- # Sanity checks, as we do dangerous work here
- if (!-r $nfo) {
- die($NAME . ': Invalid NFO source in rename: ' . $nfo . "\n");
- }
- # Parse old NFO data
- my $nfoData = updateNFOData($nfo, $season, $episode)
- or die($NAME . ': No NFO data found, refusing to rename: ' . $id . "\n");
- # Write a new NFO and unlink the old one
- if ($SUDO_CHATTR) {
- system('sudo', 'chattr', '-i', $nfo);
- }
- saveString($nfoNew, $nfoData);
- if ($nfo ne $nfoNew) {
- print STDERR 'Renaming ' . $NAME . '/' . $nfo . ' => ' . $nfoNew . "\n";
- unlink($nfo)
- or warn('Unable to delete NFO during rename: ' . $! . "\n");
- }
- }
- }
- sub parseFilename($) {
- my ($file) = @_;
- return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
- }
- # GET the requested URL, accepting encoded data and extracting the HTTP status code
- sub fetch($$) {
- my ($url, $data) = @_;
- # Init the global LWP as needed
- if (!defined($LWP)) {
- $LWP = new LWP::UserAgent;
- $LWP->agent($HTTP_UA);
- $LWP->timeout($HTTP_TIMEOUT);
- $LWP->ssl_opts({ 'verify_hostname' => $HTTP_VERIFY });
- }
- # Make a request
- my $request = HTTP::Request->new('GET' => $url);
- $request->header('Accept-Encoding' => HTTP::Message::decodable);
- my $response = $LWP->request($request);
- # Do something useful with the response
- my $code = 400;
- if ($response->status_line =~ /^(\d+)\s/) {
- $code = $1;
- }
- ${$data} = $response->decoded_content();
- return $code;
- }
|