subscribe.pl 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::UserAgent;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use Cwd qw( abs_path );
  17. use PrettyPrint;
  18. # Paramters
  19. my %USERS = ('profplump' => 'UCkj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'UChfwMHzkPXOOFDce5hyQkTA');
  20. my $EXTRAS_FILE = 'extra_videos.ini';
  21. my $EXCLUDES_FILE = 'exclude_videos.ini';
  22. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  23. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  24. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  25. my @YTDL_DEBUG = ('--verbose');
  26. my $CHATTR = '/usr/bin/chattr';
  27. my $BATCH_SIZE = 50;
  28. my $MAX_INDEX = 25000;
  29. my $FETCH_LIMIT = 50;
  30. my $DELAY = 0.5;
  31. my $HTTP_TIMEOUT = 10;
  32. my $HTTP_UA = 'ZachBot/1.0 (Plex)';
  33. my $HTTP_VERIFY = 0;
  34. my $API_URL = 'https://www.googleapis.com/youtube/v3/';
  35. my $API_KEY = $ENV{'YT_API_KEY'};
  36. my $API_ST_REL = 0.50;
  37. my $API_ST_ABS = 5;
  38. my %VIDEO_FIELDS = (
  39. 'id' => '',
  40. 'title' => '',
  41. 'description' => '',
  42. 'creator' => '',
  43. 'publishedAt' => '',
  44. 'channelId' => '',
  45. 'rawDuration' => '',
  46. 'date', => 0,
  47. 'season' => 0,
  48. 'duration' => 0,
  49. );
  50. my %API = (
  51. #https://www.googleapis.com/youtube/v3/channels?part=id&forUsername={channelName}&key={YOUR_API_KEY}
  52. 'channelID' => {
  53. 'url' => $API_URL . 'channels',
  54. 'params' => {
  55. 'part' => 'id',
  56. 'key' => $API_KEY,
  57. },
  58. },
  59. #https://www.googleapis.com/youtube/v3/channels?part=snippet&id={channelID}&key={YOUR_API_KEY}
  60. 'channel' => {
  61. 'url' => $API_URL . 'channels',
  62. 'params' => {
  63. 'part' => 'snippet',
  64. 'key' => $API_KEY,
  65. },
  66. },
  67. #https://www.googleapis.com/youtube/v3/search?part=snippet&channelId={channelID}&maxResults={maxResults}&pageToken={foo}&safeSearch=none&type=video&key={YOUR_API_KEY}
  68. 'search' => {
  69. 'url' => $API_URL . 'search',
  70. 'params' => {
  71. 'part' => 'id',
  72. 'key' => $API_KEY,
  73. 'maxResults' => 5,
  74. 'safeSearch' => 'none',
  75. 'type' => 'video',
  76. 'order' => 'date',
  77. },
  78. },
  79. #https://www.googleapis.com/youtube/v3/videos?part=snippet&id={videoID}&key={YOUR_API_KEY}
  80. 'video' => {
  81. 'url' => $API_URL . 'videos',
  82. 'params' => {
  83. 'part' => 'snippet,contentDetails',
  84. 'key' => $API_KEY,
  85. },
  86. },
  87. #https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&channelId={channelID}&key={YOUR_API_KEY}
  88. 'subscriptions' => {
  89. 'url' => $API_URL . 'subscriptions',
  90. 'params' => {
  91. 'part' => 'snippet',
  92. 'key' => $API_KEY,
  93. 'maxResults' => 5,
  94. },
  95. },
  96. );
  97. # Prototypes
  98. sub getVideoData($);
  99. sub localMetadata($$);
  100. sub findVideos($);
  101. sub findFiles($);
  102. sub buildNFO($);
  103. sub buildSeriesNFO($);
  104. sub getSubscriptions($);
  105. sub saveSubscriptions($$);
  106. sub saveChannel($);
  107. sub getChannel($);
  108. sub fetchParse($$);
  109. sub saveString($$);
  110. sub readExcludes();
  111. sub readExtras();
  112. sub parseVideoData($);
  113. sub calcVideoData($);
  114. sub dropExcludes($);
  115. sub addExtras($);
  116. sub updateNFOData($$$);
  117. sub videoNumberStr($$);
  118. sub videoSE($$);
  119. sub videoPath($$$$);
  120. sub renameVideo($$$$$$);
  121. sub parseFilename($);
  122. sub fetch($$);
  123. # Debug
  124. my $DEBUG = 0;
  125. if ($ENV{'DEBUG'}) {
  126. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  127. $DEBUG = $1;
  128. } else {
  129. $DEBUG = 1;
  130. }
  131. $DELAY /= 2;
  132. }
  133. # Sanity check
  134. if (scalar(@ARGV) < 1) {
  135. die('Usage: ' . basename($0) . " output_directory\n");
  136. }
  137. if (!defined($API_KEY) || length($API_KEY) < 1) {
  138. die("No API key provided\n");
  139. }
  140. my $SUBSCRIPTIONS = 0;
  141. if ($0 =~ /subscription/i) {
  142. $SUBSCRIPTIONS = 1;
  143. }
  144. # Command-line parameters
  145. our ($DIR) = @ARGV;
  146. $DIR = Cwd::abs_path($DIR);
  147. $DIR =~ s/\/+$//;
  148. if (!-d $DIR) {
  149. die('Invalid output directory: ' . $DIR . "\n");
  150. }
  151. our $NAME = basename($DIR);
  152. our $ID = $NAME;
  153. if ($NAME =~ /^(.*)\s\(([\w\-]+)\)$/) {
  154. $NAME = $1;
  155. $ID = $2;
  156. if (length($ID) < 1 || !($ID =~ /^[\w\-]+$/)) {
  157. die('Invalid channel ID: ' . $ID . "\n");
  158. }
  159. } elsif (!$SUBSCRIPTIONS) {
  160. die('Invalid folder: ' . $DIR . "\n");
  161. }
  162. # Allow folders to disable themselves
  163. if (-e "${DIR}/season_done") {
  164. if ($DEBUG) {
  165. print STDERR "Folder disabled. Exiting...\n";
  166. }
  167. exit(0);
  168. }
  169. # Move to the target directory so we can use relative paths later
  170. chdir($DIR) or die('Unable to chdir to: ' . $DIR . "\n");
  171. # Environmental parameters (debug)
  172. my $NO_FETCH = 0;
  173. if ($ENV{'NO_FETCH'}) {
  174. $NO_FETCH = 1;
  175. }
  176. my $NO_NFO = 0;
  177. if ($ENV{'NO_NFO'}) {
  178. $NO_NFO = 1;
  179. }
  180. my $NO_SEARCH = 0;
  181. if ($ENV{'NO_SEARCH'}) {
  182. $NO_SEARCH = 1;
  183. }
  184. my $NO_FILES = 0;
  185. if ($ENV{'NO_FILES'}) {
  186. $NO_FILES = 1;
  187. }
  188. my $NO_CHANNEL = 0;
  189. if ($ENV{'NO_CHANNEL'}) {
  190. $NO_CHANNEL = 1;
  191. }
  192. my $NO_EXTRAS = 0;
  193. if ($ENV{'NO_EXTRAS'}) {
  194. $NO_EXTRAS = 1;
  195. }
  196. my $NO_EXCLUDES = 0;
  197. if ($ENV{'NO_EXCLUDES'}) {
  198. $NO_EXCLUDES = 1;
  199. }
  200. my $NO_RENAME = 0;
  201. if ($ENV{'NO_RENAME'}) {
  202. $NO_RENAME = 1;
  203. }
  204. # Environmental parameters (functional)
  205. my $SUDO_CHATTR = 1;
  206. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run($CHATTR)) {
  207. $SUDO_CHATTR = 0;
  208. }
  209. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  210. $MAX_INDEX = $1;
  211. }
  212. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  213. $BATCH_SIZE = $1;
  214. }
  215. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  216. $FETCH_LIMIT = $1;
  217. }
  218. # Construct globals
  219. my $LWP = undef();
  220. foreach my $key (keys(%API)) {
  221. if (exists($API{$key}{'params'}{'maxResults'})) {
  222. $API{$key}{'params'}{'maxResults'} = $BATCH_SIZE;
  223. }
  224. }
  225. if ($DEBUG > 1) {
  226. push(@YTDL_ARGS, @YTDL_DEBUG);
  227. } elsif (!$DEBUG) {
  228. push(@YTDL_ARGS, @YTDL_QUIET);
  229. }
  230. # Allow use as a subscription manager
  231. if ($SUBSCRIPTIONS) {
  232. my %subs = ();
  233. foreach my $user (keys(%USERS)) {
  234. my $tmp = getSubscriptions($USERS{$user});
  235. foreach my $sub (keys(%{$tmp})) {
  236. $subs{$sub} = $tmp->{$sub};
  237. }
  238. }
  239. saveSubscriptions($DIR, \%subs);
  240. exit(0);
  241. }
  242. # Disable channel fetching if the ID channel is "None"
  243. if ($ID =~ /^\s*None\s*$/i) {
  244. if ($DEBUG) {
  245. print STDERR 'Channel features disabled for: ' . $NAME . "\n";
  246. }
  247. $NO_CHANNEL = 1;
  248. $NO_SEARCH = 1;
  249. }
  250. # Run the local "update" script if any
  251. {
  252. my @update = ('./update', $DIR);
  253. if (can_run($update[0])) {
  254. if ($DEBUG) {
  255. print STDERR "Executing local update script...\n";
  256. }
  257. system(@update);
  258. }
  259. }
  260. # Grab the channel data
  261. my $channel = {};
  262. if (!$NO_CHANNEL) {
  263. $channel = getChannel($ID);
  264. saveChannel($channel);
  265. }
  266. # Find all the user's videos on YT
  267. my $videos = {};
  268. if (!$NO_SEARCH) {
  269. $videos = findVideos($ID);
  270. }
  271. # Find any requested "extra" videos
  272. my $extras = {};
  273. if (!$NO_EXTRAS) {
  274. $extras = addExtras($videos);
  275. }
  276. # Drop any "excludes" videos
  277. my $excludes = {};
  278. if (!$NO_EXCLUDES) {
  279. $excludes = dropExcludes($videos);
  280. }
  281. # Find all existing YT files on disk
  282. my $files = {};
  283. if (!$NO_FILES) {
  284. $files = findFiles($videos);
  285. }
  286. # Deal with unknown videos
  287. foreach my $id (keys(%{$files})) {
  288. if ((!exists($videos->{$id}) || !$videos->{$id}->{'date'}) && $files->{$id}->{'season'} > 0) {
  289. # Check the ID specifically to find things the APIv3 fuzzy search misses
  290. if ($DEBUG) {
  291. print STDERR 'Validating local video by ID: ' . $id . "\n";
  292. }
  293. my $video = getVideoData([$id]);
  294. if ($video->[0] && $video->[0]->{'season'} > 0 && $video->[0]->{'channelId'} eq $ID) {
  295. $videos->{$id} = $video->[0];
  296. next;
  297. }
  298. # Whine when videos do go away
  299. warn('Local video not known to YT channel (' . $NAME . '): ' . $id . "\n");
  300. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  301. }
  302. }
  303. # Calculate the episode number using the publish dates
  304. {
  305. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  306. my $num = undef();
  307. my $lastSeason = undef();
  308. foreach my $id (@byDate) {
  309. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  310. $num = 1;
  311. $lastSeason = $videos->{$id}->{'season'};
  312. }
  313. $videos->{$id}->{'number'} = $num++;
  314. }
  315. }
  316. # Fill in missing videos and NFOs
  317. my $fetched = 0;
  318. FETCH_LOOP: foreach my $id (keys(%{$videos})) {
  319. if ($DEBUG > 1) {
  320. print STDERR 'Checking remote video: ' . $id . "\n";
  321. if (!exists($files->{$id})) {
  322. print STDERR "\tLocal file: <missing>\n";
  323. } else {
  324. print STDERR "\tLocal media: " . $files->{$id}->{'path'} . "\n";
  325. print STDERR "\tLocal NFO: ";
  326. if ($files->{$id}->{'nfo'}) {
  327. print STDERR $files->{$id}->{'nfo'} . "\n";
  328. } else {
  329. print STDERR "<none>\n";
  330. }
  331. }
  332. }
  333. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  334. # Determine if we may or must rename
  335. my $rename = 0;
  336. if (exists($files->{$id})) {
  337. # Rename if we drift
  338. if (!$rename && $files->{$id}->{'path'}) {
  339. if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  340. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  341. {
  342. if ($DEBUG) {
  343. print STDERR "Rename due to drift\n";
  344. }
  345. $rename = 1;
  346. }
  347. }
  348. # Always rename if the NFO does not match the media
  349. if (!$rename && $files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
  350. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  351. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  352. if ($DEBUG) {
  353. print STDERR "Rename due to media-metadata mismatch\n";
  354. }
  355. $rename = 1;
  356. }
  357. }
  358. }
  359. # Warn (and optionally rename) as selected
  360. if ($rename) {
  361. print STDERR 'Video ' . $id . ' had number ' . $files->{$id}->{'number'} . ' but now has number ' . $videos->{$id}->{'number'} . "\n";
  362. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  363. }
  364. # If we haven't heard of the file, or don't have an NFO for it
  365. # Checking for the NFO allows use to resume failed downloads
  366. if (!exists($files->{$id}) || !-e $nfo) {
  367. if ($DEBUG) {
  368. print STDERR 'Fetching video: ' . $id . "\n";
  369. }
  370. # Let youtube-dl handle the URLs and downloading
  371. {
  372. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  373. my @name = ($YTDL_BIN);
  374. push(@name, @YTDL_ARGS);
  375. my @fetch = @name;
  376. push(@name, '--get-filename');
  377. push(@fetch, @args);
  378. push(@name, @args);
  379. if ($NO_FETCH) {
  380. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  381. } else {
  382. # Count fetch attempts (even if they fail later)
  383. $fetched++;
  384. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  385. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $NAME . "\n";
  386. if ($DEBUG) {
  387. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  388. }
  389. exit 1;
  390. }
  391. # Find the output file name
  392. if ($DEBUG > 1) {
  393. print STDERR join(' ', @name) . "\n";
  394. }
  395. sleep($DELAY);
  396. my $file = capture(EXIT_ANY, @name);
  397. if ($EXITVAL != 0) {
  398. warn('Error executing youtube-dl for name: ' . $NAME . '/' . $id . ' (' . $EXITVAL . ")\n");
  399. next FETCH_LOOP;
  400. }
  401. if ($file =~ /\n\S/) {
  402. $file =~ s/^.*\n//;
  403. }
  404. $file =~ s/^\s+//;
  405. $file =~ s/\s+$//;
  406. # Sanity check
  407. if (!$file) {
  408. warn('No file name available for video: ' . $NAME . '/' . $id . "\n");
  409. next FETCH_LOOP;
  410. } elsif ($DEBUG > 0) {
  411. print STDERR 'Output video file: ' . $file . "\n";
  412. }
  413. # Download
  414. if ($DEBUG > 1) {
  415. print STDERR join(' ', @fetch) . "\n";
  416. }
  417. sleep($DELAY);
  418. my $exit = run(EXIT_ANY, @fetch);
  419. if ($exit != 0) {
  420. if ($DEBUG) {
  421. warn('Error executing youtube-dl for video: ' . $NAME . '/' . $id . "\n");
  422. }
  423. next FETCH_LOOP;
  424. }
  425. # Ensure we found something useful
  426. if (-e $file . '.part') {
  427. warn('Partial download detected: ' . $NAME . '/' . $file . "\n");
  428. next FETCH_LOOP;
  429. }
  430. # Sometimes the downloader lies about file extensions
  431. if (!-s $file) {
  432. $file =~ s/\.webm$/.mp4/;
  433. }
  434. if (!-s $file) {
  435. $file =~ s/\.mp4$/.webm/;
  436. }
  437. # But we still need *a* file
  438. if (!-s $file) {
  439. warn('No output video file: ' . $NAME . '/' . $file . "\n");
  440. next FETCH_LOOP;
  441. }
  442. # Touch the file to reflect the download time rather than the upload time
  443. touch($file);
  444. }
  445. }
  446. # Build and save the XML document
  447. my $xml = buildNFO($videos->{$id});
  448. if ($DEBUG > 1) {
  449. print STDERR 'Saving NFO: ' . $xml . "\n";
  450. }
  451. if ($NO_NFO) {
  452. print STDERR "Not saving NFO\n";
  453. } else {
  454. saveString($nfo, $xml);
  455. }
  456. }
  457. }
  458. sub saveString($$) {
  459. my ($path, $str) = @_;
  460. my $fh = undef();
  461. if (!open($fh, '>', $path)) {
  462. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  463. return undef();
  464. }
  465. print $fh $str;
  466. close($fh);
  467. return 1;
  468. }
  469. sub buildSeriesNFO($) {
  470. my ($channel) = @_;
  471. # Create an XML tree
  472. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  473. $doc->setStandalone(1);
  474. my $show = $doc->createElement('tvshow');
  475. $doc->setDocumentElement($show);
  476. my $elm;
  477. # Add data
  478. $elm = $doc->createElement('title');
  479. $elm->appendText($channel->{'title'});
  480. $show->appendChild($elm);
  481. $elm = $doc->createElement('premiered');
  482. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  483. $show->appendChild($elm);
  484. $elm = $doc->createElement('plot');
  485. $elm->appendText($channel->{'description'});
  486. $show->appendChild($elm);
  487. # Return the string
  488. return $doc->toString();
  489. }
  490. sub buildNFO($) {
  491. my ($video) = @_;
  492. # Create an XML tree
  493. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  494. $doc->setStandalone(1);
  495. my $show = $doc->createElement('episodedetails');
  496. $doc->setDocumentElement($show);
  497. my $file = $doc->createElement('fileinfo');
  498. $show->appendChild($file);
  499. my $stream = $doc->createElement('streamdetails');
  500. $file->appendChild($stream);
  501. my $stream_video = $doc->createElement('video');
  502. $stream->appendChild($stream_video);
  503. my $elm;
  504. # Add data
  505. $elm = $doc->createElement('season');
  506. $elm->appendText($video->{'season'});
  507. $show->appendChild($elm);
  508. $elm = $doc->createElement('episode');
  509. $elm->appendText(videoNumberStr($video->{'season'}, $video->{'number'}));
  510. $show->appendChild($elm);
  511. if (!defined($video->{'title'})) {
  512. $video->{'title'} = 'Episode ' . $video->{'number'};
  513. }
  514. $elm = $doc->createElement('title');
  515. $elm->appendText($video->{'title'});
  516. $show->appendChild($elm);
  517. if (!defined($video->{'date'})) {
  518. $video->{'date'} = time();
  519. }
  520. $elm = $doc->createElement('aired');
  521. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  522. $show->appendChild($elm);
  523. if (defined($video->{'description'})) {
  524. $elm = $doc->createElement('plot');
  525. $elm->appendText($video->{'description'});
  526. $show->appendChild($elm);
  527. }
  528. if (defined($video->{'duration'})) {
  529. $elm = $doc->createElement('runtime');
  530. $elm->appendText($video->{'duration'});
  531. $show->appendChild($elm);
  532. $elm = $doc->createElement('durationinseconds');
  533. $elm->appendText($video->{'duration'});
  534. $stream_video->appendChild($elm);
  535. }
  536. if (defined($video->{'creator'})) {
  537. $elm = $doc->createElement('director');
  538. $elm->appendText($video->{'creator'});
  539. $show->appendChild($elm);
  540. }
  541. # Return the string
  542. return $doc->toString();
  543. }
  544. sub findFiles($) {
  545. my ($videos) = @_;
  546. my %files = ();
  547. our $NAME;
  548. # Allow complete bypass
  549. if ($NO_FILES) {
  550. return \%files;
  551. }
  552. # Read the output directory
  553. my $fh = undef();
  554. opendir($fh, '.')
  555. or die($NAME . ': Unable to open files directory: ' . $! . "\n");
  556. while (my $file = readdir($fh)) {
  557. my ($season, $number, $id, $suffix) = parseFilename($file);
  558. if (defined($id) && length($id) > 0) {
  559. # Determine if there is another video with the same season/number
  560. my $exists = 0;
  561. if ( exists($videos->{$id})
  562. && exists($videos->{$id}->{'season'})
  563. && exists($videos->{$id}->{'number'})
  564. && $season == $videos->{$id}->{'season'}
  565. && $number == $videos->{$id}->{'number'})
  566. {
  567. $exists = 1;
  568. }
  569. # Create the record as needed
  570. if (!exists($files{$id})) {
  571. my %tmp = (
  572. 'season' => $season,
  573. 'number' => $number,
  574. );
  575. $files{$id} = \%tmp;
  576. }
  577. # Video or NFO?
  578. my $type = 'path';
  579. if ($suffix eq 'nfo') {
  580. $type = 'nfo';
  581. }
  582. # Deal with duplicates
  583. if (exists($files{$id}->{$type})) {
  584. if ($type eq 'nfo') {
  585. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  586. if (!$NO_RENAME) {
  587. my $del = $file;
  588. if ($exists) {
  589. $del = $files{$id}->{'nfo'};
  590. }
  591. warn("\tDeleting: " . $file . "\n");
  592. if ($SUDO_CHATTR) {
  593. system('sudo', $CHATTR, '-i', Cwd::abs_path($file));
  594. }
  595. unlink($file);
  596. next;
  597. }
  598. } else {
  599. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  600. if (!$NO_RENAME) {
  601. my $del = $file;
  602. if ($exists) {
  603. $del = $files{$id}->{'path'};
  604. } elsif ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  605. $del = $files{$id}->{'path'};
  606. } elsif ($suffix ne 'webm' && $files{$id}->{'suffix'} eq 'webm') {
  607. $del = $files{$id}->{'path'};
  608. }
  609. warn("\tDeleting: " . $del . "\n");
  610. if ($SUDO_CHATTR) {
  611. system('sudo', $CHATTR, '-i', Cwd::abs_path($del));
  612. }
  613. unlink($del);
  614. next;
  615. }
  616. }
  617. }
  618. # Assign the component paths
  619. $files{$id}->{$type} = $file;
  620. if ($type ne 'nfo') {
  621. $files{$id}->{'suffix'} = $suffix;
  622. }
  623. }
  624. }
  625. close($fh);
  626. if ($DEBUG) {
  627. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  628. if ($DEBUG > 1) {
  629. print STDERR prettyPrint(\%files, "\t") . "\n";
  630. }
  631. }
  632. return \%files;
  633. }
  634. sub fetchParse($$) {
  635. my ($name, $params) = @_;
  636. our $NAME;
  637. my $url = $API{$name}{'url'} . '?';
  638. foreach my $key (keys(%{ $API{$name}{'params'} }), keys(%{$params})) {
  639. $url .= '&' . uri_escape($key) . '=';
  640. if (exists($params->{$key})) {
  641. $url .= uri_escape($params->{$key});
  642. } else {
  643. $url .= uri_escape($API{$name}{'params'}{$key});
  644. }
  645. }
  646. # Fetch
  647. if ($DEBUG > 1) {
  648. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  649. }
  650. sleep($DELAY);
  651. my $content;
  652. my $code = fetch($url, \$content);
  653. if ($code != 200 || !defined($content) || length($content) < 10) {
  654. my $msg = $NAME . ': Invalid content from URL (' . $code . '): ' . $url . "\n" . $content . "\n";
  655. if ($code >= 500 && $code < 600) {
  656. if ($DEBUG) {
  657. print STDERR $msg;
  658. }
  659. exit(0);
  660. }
  661. if ($code >= 400 && $code < 500) {
  662. if ($DEBUG) {
  663. print STDERR "Wainting 30 seconds after 400 error\n";
  664. }
  665. sleep(30);
  666. }
  667. die($msg);
  668. }
  669. # Parse
  670. if ($DEBUG > 2) {
  671. print STDERR "Raw JSON data:\n" . $content . "\n";
  672. }
  673. my $data = decode_json($content);
  674. if (!defined($data) || ref($data) ne 'HASH') {
  675. die($NAME . ': Invalid JSON: ' . $content . "\n");
  676. }
  677. return $data;
  678. }
  679. sub getSubscriptions($) {
  680. my ($id) = @_;
  681. our $NAME;
  682. # Loop through until we have all the entries
  683. my $pageToken = '';
  684. my %subs = ();
  685. SUBS_LOOP:
  686. {
  687. # Build, fetch, parse, check
  688. my $data = fetchParse('subscriptions', { 'channelId' => $id, 'pageToken' => $pageToken });
  689. $pageToken = '';
  690. if ( exists($data->{'pageInfo'})
  691. && ref($data->{'pageInfo'}) eq 'HASH'
  692. && exists($data->{'pageInfo'}->{'totalResults'})
  693. && $data->{'pageInfo'}->{'totalResults'} > 0
  694. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  695. && exists($data->{'items'})
  696. && ref($data->{'items'}) eq 'ARRAY')
  697. {
  698. if (exists($data->{'nextPageToken'})) {
  699. $pageToken = $data->{'nextPageToken'};
  700. }
  701. $data = $data->{'items'};
  702. } else {
  703. die($NAME . ": Invalid subscription data\n");
  704. }
  705. foreach my $item (@{$data}) {
  706. if ( !exists($item->{'snippet'})
  707. || ref($item->{'snippet'}) ne 'HASH'
  708. || !exists($item->{'snippet'}->{'title'})
  709. || !exists($item->{'snippet'}->{'resourceId'})
  710. || ref($item->{'snippet'}->{'resourceId'}) ne 'HASH'
  711. || !exists($item->{'snippet'}->{'resourceId'}->{'channelId'}))
  712. {
  713. warn($NAME . ": Skipping invalid subscription\n");
  714. if ($DEBUG > 2) {
  715. print STDERR prettyPrint($item, "\t") . "\n";
  716. }
  717. next;
  718. }
  719. if ($DEBUG) {
  720. print STDERR $item->{'snippet'}->{'title'} . ' => ' . $item->{'snippet'}->{'resourceId'}->{'channelId'} . "\n";
  721. }
  722. $subs{ $item->{'snippet'}->{'resourceId'}->{'channelId'} } = $item->{'snippet'}->{'title'};
  723. }
  724. # Loop if there are results left to fetch
  725. if ($pageToken) {
  726. redo SUBS_LOOP;
  727. }
  728. }
  729. # Return the list of subscribed channel IDs
  730. return \%subs;
  731. }
  732. sub saveSubscriptions($$) {
  733. my ($folder, $subs) = @_;
  734. our $NAME;
  735. # Check for local subscriptions missing from YT
  736. my %locals = ();
  737. my $fh = undef();
  738. opendir($fh, '.')
  739. or die($NAME . ': Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  740. while (my $file = readdir($fh)) {
  741. # Skip dotfiles
  742. if ($file =~ /^\./) {
  743. next;
  744. }
  745. # Skip non-directories
  746. if (!-d $file) {
  747. next;
  748. }
  749. # Extract the YTID
  750. my ($id) = $file =~ /\(([\w\-]+)\)$/;
  751. if (!$id) {
  752. warn('Skipping invalid local subscription: ' . $file . "\n");
  753. next;
  754. }
  755. # Skip "None" directories (i.e. non-channel subscriptions)
  756. if ($id eq 'None') {
  757. if ($DEBUG) {
  758. print STDERR 'Ignoring non-channel subscription: ' . $file . "\n";
  759. }
  760. next;
  761. }
  762. # Anything else should be in the list
  763. if (!$subs->{$id}) {
  764. my (undef(), undef(), $hour) = localtime(time);
  765. if ($hour == 0 || $DEBUG) {
  766. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  767. }
  768. }
  769. # Note local subscriptions
  770. $locals{$id} = 1;
  771. }
  772. closedir($fh);
  773. # Check for YT subscriptions missing locally
  774. foreach my $id (keys(%{$subs})) {
  775. if (!exists($locals{$id})) {
  776. my $dir = $subs->{$id} . ' (' . $id . ')';
  777. print STDERR 'Adding local subscription for: ' . $dir . "\n";
  778. mkdir($folder . '/' . $dir);
  779. }
  780. }
  781. }
  782. sub saveChannel($) {
  783. my ($channel) = @_;
  784. my $nfo = 'tvshow.nfo';
  785. if (!-e $nfo) {
  786. if ($DEBUG) {
  787. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  788. }
  789. # Save the poster
  790. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  791. my $jpg;
  792. if (fetch($channel->{'thumbnail'}, \$jpg) == 200) {
  793. saveString('poster.jpg', $jpg);
  794. }
  795. }
  796. # Save the series NFO
  797. my $xml = buildSeriesNFO($channel);
  798. if ($DEBUG > 1) {
  799. print STDERR 'Saving NFO: ' . $xml . "\n";
  800. }
  801. saveString($nfo, $xml);
  802. }
  803. }
  804. sub getChannel($) {
  805. my ($id) = @_;
  806. our $NAME;
  807. # Build, fetch, parse, check
  808. my $data = fetchParse('channel', { 'id' => $id });
  809. if ( exists($data->{'pageInfo'})
  810. && ref($data->{'pageInfo'}) eq 'HASH'
  811. && exists($data->{'pageInfo'}->{'totalResults'})
  812. && $data->{'pageInfo'}->{'totalResults'} == 1
  813. && exists($data->{'items'})
  814. && ref($data->{'items'}) eq 'ARRAY'
  815. && scalar(@{ $data->{'items'} }) > 0
  816. && ref($data->{'items'}[0]) eq 'HASH'
  817. && exists($data->{'items'}[0]->{'snippet'})
  818. && ref($data->{'items'}[0]->{'snippet'}) eq 'HASH')
  819. {
  820. $data = $data->{'items'}[0]->{'snippet'};
  821. } else {
  822. die($NAME . ": Invalid channel data\n");
  823. }
  824. # Extract the data we want
  825. my %channel = (
  826. 'id' => $id,
  827. 'title' => $data->{'title'},
  828. 'date' => str2time($data->{'publishedAt'}),
  829. 'description' => $data->{'description'},
  830. 'thumbnail' => $data->{'thumbnails'}->{'high'}->{'url'},
  831. );
  832. return \%channel;
  833. }
  834. sub readExcludes() {
  835. my %excludes = ();
  836. our $NAME;
  837. # Read and parse the excludes videos file, if it exists
  838. if (-e $EXCLUDES_FILE) {
  839. my $fh;
  840. open($fh, $EXCLUDES_FILE)
  841. or die($NAME . ': Unable to open excludes videos file: ' . $! . "\n");
  842. while (<$fh>) {
  843. # Skip blank lines and comments
  844. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  845. next;
  846. }
  847. # Match our specific format or whine
  848. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  849. if ($DEBUG > 1) {
  850. print STDERR 'Adding exclude video: ' . $1 . "\n";
  851. }
  852. $excludes{$1} = 1;
  853. } else {
  854. print STDERR 'Skipped exclude video line: ' . $_;
  855. }
  856. }
  857. close($fh);
  858. }
  859. return \%excludes;
  860. }
  861. sub readExtras() {
  862. my %extras = ();
  863. our $NAME;
  864. # Read and parse the extra videos file, if it exists
  865. if (-e $EXTRAS_FILE) {
  866. my $fh;
  867. open($fh, $EXTRAS_FILE)
  868. or die($NAME . ': Unable to open extra videos file: ' . $! . "\n");
  869. while (<$fh>) {
  870. # Skip blank lines and comments
  871. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  872. next;
  873. }
  874. # Match our specific format or whine
  875. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  876. if ($DEBUG > 1) {
  877. print STDERR 'Adding extra video: ' . $1 . "\n";
  878. }
  879. $extras{$1} = 1;
  880. } else {
  881. print STDERR 'Skipped extra video line: ' . $_;
  882. }
  883. }
  884. close($fh);
  885. }
  886. return \%extras;
  887. }
  888. sub parseVideoData($) {
  889. my ($data) = @_;
  890. my %video = %VIDEO_FIELDS;
  891. # Sanity check
  892. if ( !exists($data->{'id'})
  893. || !exists($data->{'snippet'})
  894. || ref($data->{'snippet'}) ne 'HASH'
  895. || !exists($data->{'contentDetails'})
  896. || ref($data->{'contentDetails'}) ne 'HASH'
  897. || !exists($data->{'contentDetails'}->{'duration'}))
  898. {
  899. if ($DEBUG > 2) {
  900. print STDERR "Unable to parse video data:\n" . prettyPrint($data, "\t") . "\n";
  901. }
  902. return undef();
  903. }
  904. # Direct extraction
  905. $video{'id'} = $data->{'id'};
  906. $video{'title'} = $data->{'snippet'}->{'title'};
  907. $video{'description'} = $data->{'snippet'}->{'description'};
  908. $video{'creator'} = $data->{'snippet'}->{'channelTitle'};
  909. $video{'publishedAt'} = $data->{'snippet'}->{'publishedAt'};
  910. $video{'channelId'} = $data->{'snippet'}->{'channelId'};
  911. $video{'rawDuration'} = $data->{'contentDetails'}->{'duration'};
  912. # Convert dates and whatnot
  913. calcVideoData(\%video);
  914. return \%video;
  915. }
  916. sub calcVideoData($) {
  917. my ($video) = @_;
  918. if ($video->{'publishedAt'} && !$video->{'date'}) {
  919. $video->{'date'} = str2time($video->{'publishedAt'});
  920. }
  921. if ($video->{'date'}) {
  922. $video->{'season'} = time2str('%Y', $video->{'date'});
  923. } else {
  924. $video->{'season'} = 0;
  925. }
  926. if ($video->{'rawDuration'} =~ /PT(\d+)H(\d+)M(\d+)S/) {
  927. $video->{'duration'} = ($1 * 3600) + ($2 * 60) + $2;
  928. } elsif ($video->{'rawDuration'} =~ /PT(\d+)M(\d+)S/) {
  929. $video->{'duration'} = ($1 * 60) + $2;
  930. } elsif ($video->{'rawDuration'} =~ /PT(\d+)S/) {
  931. $video->{'duration'} = $1;
  932. }
  933. }
  934. sub dropExcludes($) {
  935. my ($videos) = @_;
  936. my $excludes = readExcludes();
  937. foreach my $id (keys(%{$excludes})) {
  938. if (!exists($videos->{$id})) {
  939. if ($DEBUG > 1) {
  940. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  941. }
  942. next;
  943. }
  944. delete($videos->{$id});
  945. }
  946. return $excludes;
  947. }
  948. sub addExtras($) {
  949. my ($videos) = @_;
  950. my $extras = readExtras();
  951. foreach my $id (keys(%{$extras})) {
  952. if (exists($videos->{$id})) {
  953. if ($DEBUG > 1) {
  954. print STDERR 'Skipping known extra video: ' . $id . "\n";
  955. }
  956. next;
  957. }
  958. my $video = getVideoData([$id]);
  959. $videos->{$id} = $video->[0];
  960. }
  961. return $extras;
  962. }
  963. sub getVideoData($) {
  964. my ($ids) = @_;
  965. my @videos = ();
  966. our $NAME;
  967. my %found = ();
  968. # Do a batch request for the entire batch of video data
  969. my $data = fetchParse('video', { 'id' => join(',', @{$ids}) });
  970. if ( exists($data->{'pageInfo'})
  971. && ref($data->{'pageInfo'}) eq 'HASH'
  972. && exists($data->{'pageInfo'}->{'totalResults'})
  973. && exists($data->{'items'})
  974. && ref($data->{'items'}) eq 'ARRAY')
  975. {
  976. if (scalar(@{$ids}) > 1 && $data->{'pageInfo'}->{'totalResults'} != scalar(@{$ids})) {
  977. die($NAME . ': Video/metadata search count mismatch: ' . $data->{'pageInfo'}->{'totalResults'} . '/' . scalar(@{$ids}) . "\n");
  978. }
  979. $data = $data->{'items'};
  980. } else {
  981. die($NAME . ": Invalid search video data\n" . prettyPrint($data, "\t") . "\n");
  982. }
  983. # Build each metadata record
  984. foreach my $item (@{$data}) {
  985. if (!exists($item->{'snippet'}) || ref($item->{'snippet'}) ne 'HASH') {
  986. warn($NAME . ": Skipping invalid metadata item\n");
  987. if ($DEBUG > 2) {
  988. print STDERR prettyPrint($item, "\t") . "\n";
  989. }
  990. next;
  991. }
  992. my $video = parseVideoData($item);
  993. if (!$video) {
  994. die($NAME . ": Unable to parse video data\n");
  995. }
  996. # Allow local metadata overrides
  997. localMetadata($video->{'id'}, $video);
  998. # Save
  999. $found{ $video->{'id'} } = 1;
  1000. push(@videos, $video);
  1001. }
  1002. # Load local metadata even for videos YT doesn't know anything about
  1003. foreach my $id (@{$ids}) {
  1004. if ($found{$id}) {
  1005. next;
  1006. }
  1007. my %video = %VIDEO_FIELDS;
  1008. $video{'id'} = $id;
  1009. localMetadata($id, \%video);
  1010. # Save
  1011. $found{$id} = 1;
  1012. push(@videos, \%video);
  1013. }
  1014. # Parse the video data
  1015. return \@videos;
  1016. }
  1017. sub localMetadata($$) {
  1018. my ($id, $video) = @_;
  1019. my $meta = $id . '.meta';
  1020. if (!-r $meta) {
  1021. return;
  1022. }
  1023. my $fh = undef();
  1024. open($fh, '<', $meta)
  1025. or warn('Unable to open local metadata file: ' . $meta . ': ' . $! . ". Skipping...\n");
  1026. while (<$fh>) {
  1027. if (/^\s*#/ || /^\s*$/) {
  1028. next;
  1029. }
  1030. if (/^\s*([^\s\=][^\=]*[^\s\=])\s*=\>?\s*(\S.*\S)\s*$/) {
  1031. if (exists($video->{$1})) {
  1032. if ($DEBUG) {
  1033. print STDERR 'Using local metadata line (' . $id . '): ' . $1 . ' => ' . $2 . "\n";
  1034. }
  1035. $video->{$1} = $2;
  1036. } else {
  1037. die('Invalid local metadata attribute (' . $1 . '): ' . $meta . "\n");
  1038. }
  1039. } else {
  1040. die('Invalid local metdata line (' . $meta . '): ' . chomp($_) . "\n");
  1041. }
  1042. }
  1043. close($fh);
  1044. calcVideoData($video);
  1045. }
  1046. sub findVideos($) {
  1047. my ($id) = @_;
  1048. my %videos = ();
  1049. our $NAME;
  1050. # Allow complete bypass
  1051. if ($NO_SEARCH) {
  1052. return \%videos;
  1053. }
  1054. # Loop through until we have all the entries
  1055. my $totalCount = 0;
  1056. my %params = ('channelId' => $id);
  1057. LOOP:
  1058. {
  1059. # Loop status
  1060. my $count = scalar(keys(%videos));
  1061. if ($DEBUG && $totalCount) {
  1062. print STDERR 'Fetching ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%)\n";
  1063. }
  1064. # Build, fetch, parse, check
  1065. my $data = fetchParse('search', \%params);
  1066. delete($params{'pageToken'});
  1067. if ( exists($data->{'pageInfo'})
  1068. && ref($data->{'pageInfo'}) eq 'HASH'
  1069. && exists($data->{'pageInfo'}->{'totalResults'})
  1070. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  1071. && exists($data->{'items'})
  1072. && ref($data->{'items'}) eq 'ARRAY')
  1073. {
  1074. if (!$totalCount) {
  1075. $totalCount = $data->{'pageInfo'}->{'totalResults'};
  1076. }
  1077. $data = $data->{'items'};
  1078. } else {
  1079. die($NAME . ": Invalid search data\n");
  1080. }
  1081. # Sanity check
  1082. if (scalar(@{$data}) < 1) {
  1083. last LOOP;
  1084. }
  1085. # Grab each video ID
  1086. my @ids = ();
  1087. foreach my $item (@{$data}) {
  1088. if (!exists($item->{'id'}) || ref($item->{'id'}) ne 'HASH' || !exists($item->{'id'}->{'videoId'})) {
  1089. warn($NAME . ": Skipping invalid video item\n");
  1090. if ($DEBUG > 2) {
  1091. print STDERR prettyPrint($data, "\t") . "\n";
  1092. }
  1093. next;
  1094. }
  1095. push(@ids, $item->{'id'}->{'videoId'});
  1096. }
  1097. # Do a batch request for the entire batch of video data
  1098. my $more = 0;
  1099. my $records = getVideoData(\@ids);
  1100. foreach my $rec (@{$records}) {
  1101. if (exists($rec->{'duration'}) && $rec->{'duration'} == 0) {
  1102. if ($DEBUG > 1) {
  1103. print STDERR 'Skipping 0-length video: ' . $rec->{'id'} . "\n";
  1104. }
  1105. next;
  1106. }
  1107. $videos{ $rec->{'id'} } = $rec;
  1108. if (!defined($params{'publishedBefore'}) || (defined($rec->{'publishedAt'}) && ($rec->{'publishedAt'} cmp $params{'publishedBefore'}) < 0)) {
  1109. $params{'publishedBefore'} = $rec->{'publishedAt'};
  1110. $more = 1;
  1111. }
  1112. }
  1113. # Loop if there are results left to fetch
  1114. if ($more && $params{'publishedBefore'} && $count <= $MAX_INDEX) {
  1115. redo LOOP;
  1116. }
  1117. }
  1118. # Sanity check/debug
  1119. {
  1120. my $count = scalar(keys(%videos));
  1121. if ($DEBUG) {
  1122. print STDERR 'Found ' . $count . " remote videos\n";
  1123. if (abs(1 - ($count / $totalCount)) > $API_ST_REL && abs($totalCount - $count) > $API_ST_ABS) {
  1124. print STDERR 'Found only ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%) channel-API-reported remote videos.\n";
  1125. }
  1126. if ($DEBUG > 1) {
  1127. print STDERR prettyPrint(\%videos, "\t") . "\n";
  1128. }
  1129. }
  1130. }
  1131. return \%videos;
  1132. }
  1133. sub updateNFOData($$$) {
  1134. my ($file, $season, $episode) = @_;
  1135. my $nfoData = undef();
  1136. {
  1137. open(my $fh, '<', $file)
  1138. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  1139. $/ = undef;
  1140. $nfoData = <$fh>;
  1141. close($fh);
  1142. }
  1143. if ($nfoData) {
  1144. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  1145. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  1146. }
  1147. return $nfoData;
  1148. }
  1149. sub videoNumberStr($$) {
  1150. my ($season, $episode) = @_;
  1151. my $str = '';
  1152. if ($season == 0 && $episode =~ /^(20[012]\d)(\d{2,4})$/) {
  1153. $str = sprintf('%04d%04d', $1, $2);
  1154. } else {
  1155. $str = sprintf('%02d', $episode);
  1156. }
  1157. return $str;
  1158. }
  1159. sub videoSE($$) {
  1160. my ($season, $episode) = @_;
  1161. return sprintf('S%0dE%s - ', $season, videoNumberStr($season, $episode));
  1162. }
  1163. sub videoPath($$$$) {
  1164. my ($season, $episode, $id, $ext) = @_;
  1165. return videoSE($season, $episode) . $id . '.' . $ext;
  1166. }
  1167. sub renameVideo($$$$$$) {
  1168. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  1169. our $NAME;
  1170. # Bail if disabled
  1171. if ($NO_RENAME) {
  1172. print STDERR 'Not renaming: ' . $video . "\n";
  1173. return;
  1174. }
  1175. # General sanity checks
  1176. if (!defined($id) || !defined($season) || !defined($episode)) {
  1177. die($NAME . ": Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  1178. }
  1179. # Warning about useless calls
  1180. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  1181. warn("No NFO or video path provided in renameVideo()\n");
  1182. return;
  1183. }
  1184. # Video
  1185. if (defined($video) && defined($suffix)) {
  1186. my $videoNew = videoPath($season, $episode, $id, $suffix);
  1187. # Sanity checks, as we do dangerous work here
  1188. if (!-r $video) {
  1189. die($NAME . ': Invalid video source in rename: ' . $video . "\n");
  1190. }
  1191. if ($video ne $videoNew) {
  1192. print STDERR 'Renaming ' . $NAME . '/' . $video . ' => ' . $videoNew . "\n";
  1193. if (-e $videoNew) {
  1194. die($NAME . ': Rename: Target exists: ' . $videoNew . "\n");
  1195. }
  1196. if ($SUDO_CHATTR) {
  1197. system('sudo', $CHATTR, '-i', Cwd::abs_path($video));
  1198. }
  1199. rename($video, $videoNew)
  1200. or die($NAME . ': Unable to rename: ' . $video . ': ' . $! . "\n");
  1201. }
  1202. }
  1203. # NFO
  1204. if (defined($nfo)) {
  1205. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  1206. # Sanity checks, as we do dangerous work here
  1207. if (!-r $nfo) {
  1208. die($NAME . ': Invalid NFO source in rename: ' . $nfo . "\n");
  1209. }
  1210. # Parse old NFO data
  1211. my $nfoData = updateNFOData($nfo, $season, $episode)
  1212. or die($NAME . ': No NFO data found, refusing to rename: ' . $id . "\n");
  1213. # Write a new NFO and unlink the old one
  1214. if ($SUDO_CHATTR) {
  1215. system('sudo', $CHATTR, '-i', Cwd::abs_path($nfo));
  1216. }
  1217. saveString($nfoNew, $nfoData);
  1218. if ($nfo ne $nfoNew) {
  1219. print STDERR 'Renaming ' . $NAME . '/' . $nfo . ' => ' . $nfoNew . "\n";
  1220. unlink($nfo)
  1221. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  1222. }
  1223. }
  1224. }
  1225. sub parseFilename($) {
  1226. my ($file) = @_;
  1227. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)(?:\-recode)?\.(\w\w\w)$/i;
  1228. }
  1229. # GET the requested URL, accepting encoded data and extracting the HTTP status code
  1230. sub fetch($$) {
  1231. my ($url, $data) = @_;
  1232. # Init the global LWP as needed
  1233. if (!defined($LWP)) {
  1234. $LWP = new LWP::UserAgent;
  1235. $LWP->agent($HTTP_UA);
  1236. $LWP->timeout($HTTP_TIMEOUT);
  1237. $LWP->ssl_opts({ 'verify_hostname' => $HTTP_VERIFY });
  1238. }
  1239. # Make a request
  1240. my $request = HTTP::Request->new('GET' => $url);
  1241. $request->header('Accept-Encoding' => HTTP::Message::decodable);
  1242. my $response = $LWP->request($request);
  1243. # Do something useful with the response
  1244. my $code = 400;
  1245. if ($response->status_line =~ /^(\d+)\s/) {
  1246. $code = $1;
  1247. }
  1248. ${$data} = $response->decoded_content();
  1249. return $code;
  1250. }