subscribe.pl 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::UserAgent;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use Cwd qw( abs_path );
  17. use PrettyPrint;
  18. # Paramters
  19. my %USERS = ('profplump' => 'UCkj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'UChfwMHzkPXOOFDce5hyQkTA');
  20. my $EXTRAS_FILE = 'extra_videos.ini';
  21. my $EXCLUDES_FILE = 'exclude_videos.ini';
  22. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  23. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--retries', '1',
  24. '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  25. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  26. my @YTDL_DEBUG = ('--verbose');
  27. my $CHATTR = '/usr/bin/chattr';
  28. my $BATCH_SIZE = int(rand(32));
  29. my $MAX_INDEX = 25000;
  30. my $FETCH_LIMIT = 50;
  31. my $DELAY = 5;
  32. my $MIN_DELAY = 0.75;
  33. my $FORK_DELAY = 5;
  34. my $HTTP_TIMEOUT = 12;
  35. my $HTTP_UA = 'ZachBot/1.1 (Firewall)';
  36. my $HTTP_VERIFY = 0;
  37. my $API_HTML = $ENV{'API_HTML'};
  38. my $API_URL = 'https://www.googleapis.com/youtube/v3/';
  39. my $API_KEY = $ENV{'YT_API_KEY'};
  40. my $API_ST_REL = 0.50;
  41. my $API_ST_ABS = 5;
  42. my %VIDEO_FIELDS = (
  43. 'id' => '',
  44. 'title' => '',
  45. 'description' => '',
  46. 'creator' => '',
  47. 'publishedAt' => '',
  48. 'channelId' => '',
  49. 'rawDuration' => '',
  50. 'date', => 0,
  51. 'season' => 0,
  52. 'duration' => 0,
  53. );
  54. my %API = (
  55. #https://www.googleapis.com/youtube/v3/channels?part=id&forUsername={channelName}&key={YOUR_API_KEY}
  56. 'channelID' => {
  57. 'url' => $API_URL . 'channels',
  58. 'params' => {
  59. 'part' => 'id',
  60. 'key' => $API_KEY,
  61. },
  62. },
  63. #https://www.googleapis.com/youtube/v3/channels?part=snippet&id={channelID}&key={YOUR_API_KEY}
  64. 'channel' => {
  65. 'url' => $API_URL . 'channels',
  66. 'params' => {
  67. 'part' => 'snippet',
  68. 'key' => $API_KEY,
  69. },
  70. },
  71. #https://www.googleapis.com/youtube/v3/search?part=snippet&channelId={channelID}&maxResults={maxResults}&pageToken={foo}&safeSearch=none&type=video&key={YOUR_API_KEY}
  72. 'search' => {
  73. 'url' => $API_URL . 'search',
  74. 'params' => {
  75. 'part' => 'id',
  76. 'key' => $API_KEY,
  77. 'maxResults' => 5,
  78. 'safeSearch' => 'none',
  79. 'type' => 'video',
  80. 'order' => 'date',
  81. },
  82. },
  83. #https://www.googleapis.com/youtube/v3/videos?part=snippet&id={videoID}&key={YOUR_API_KEY}
  84. 'video' => {
  85. 'url' => $API_URL . 'videos',
  86. 'params' => {
  87. 'part' => 'snippet,contentDetails',
  88. 'key' => $API_KEY,
  89. },
  90. },
  91. #https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&channelId={channelID}&key={YOUR_API_KEY}
  92. 'subscriptions' => {
  93. 'url' => $API_URL . 'subscriptions',
  94. 'params' => {
  95. 'part' => 'snippet',
  96. 'key' => $API_KEY,
  97. 'maxResults' => 5,
  98. },
  99. },
  100. );
  101. # Prototypes
  102. sub getVideoData($);
  103. sub localMetadata($$);
  104. sub findVideos($);
  105. sub findFiles($);
  106. sub buildNFO($);
  107. sub buildSeriesNFO($);
  108. sub getSubscriptions($);
  109. sub saveSubscriptions($$);
  110. sub saveChannel($);
  111. sub getChannel($);
  112. sub fetchParse($$);
  113. sub saveString($$);
  114. sub readExcludes();
  115. sub readExtras();
  116. sub parseVideoData($);
  117. sub calcVideoData($);
  118. sub dropExcludes($);
  119. sub addExtras($);
  120. sub updateNFOData($$$);
  121. sub videoNumberStr($$);
  122. sub videoSE($$);
  123. sub videoPath($$$$);
  124. sub renameVideo($$$$$$);
  125. sub parseFilename($);
  126. sub fetch($$);
  127. sub delay(;$);
  128. # Debug
  129. my $DEBUG = 0;
  130. if ($ENV{'DEBUG'}) {
  131. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  132. $DEBUG = $1;
  133. } else {
  134. $DEBUG = 1;
  135. }
  136. $DELAY = 0;
  137. }
  138. # Sanity check
  139. if (scalar(@ARGV) < 1) {
  140. die('Usage: ' . basename($0) . " output_directory\n");
  141. }
  142. if (!defined($API_KEY) || length($API_KEY) < 1) {
  143. die("No API key provided\n");
  144. }
  145. my $SUBSCRIPTIONS = 0;
  146. if ($0 =~ /subscription/i) {
  147. $SUBSCRIPTIONS = 1;
  148. }
  149. # Command-line parameters
  150. our ($DIR) = @ARGV;
  151. $DIR = Cwd::abs_path($DIR);
  152. $DIR =~ s/\/+$//;
  153. if (!-d $DIR) {
  154. die('Invalid output directory: ' . $DIR . "\n");
  155. }
  156. our $NAME = basename($DIR);
  157. our $ID = $NAME;
  158. if ($NAME =~ /^(.*)\s\(([\w\-]+)\)$/) {
  159. $NAME = $1;
  160. $ID = $2;
  161. if (length($ID) < 1 || !($ID =~ /^[\w\-]+$/)) {
  162. die('Invalid channel ID: ' . $ID . "\n");
  163. }
  164. } elsif (!$SUBSCRIPTIONS) {
  165. die('Invalid folder: ' . $DIR . "\n");
  166. }
  167. # Allow folders to disable themselves
  168. if (-e "${DIR}/season_done") {
  169. if ($DEBUG) {
  170. print STDERR "Folder disabled. Exiting...\n";
  171. }
  172. exit(0);
  173. }
  174. # Move to the target directory so we can use relative paths later
  175. chdir($DIR) or die('Unable to chdir to: ' . $DIR . "\n");
  176. # Environmental parameters (debug)
  177. my $NO_FETCH = 0;
  178. if ($ENV{'NO_FETCH'}) {
  179. $NO_FETCH = 1;
  180. }
  181. my $NO_NFO = 0;
  182. if ($ENV{'NO_NFO'}) {
  183. $NO_NFO = 1;
  184. }
  185. my $NO_SEARCH = 0;
  186. if ($ENV{'NO_SEARCH'}) {
  187. $NO_SEARCH = 1;
  188. }
  189. my $NO_FILES = 0;
  190. if ($ENV{'NO_FILES'}) {
  191. $NO_FILES = 1;
  192. }
  193. my $NO_CHANNEL = 0;
  194. if ($ENV{'NO_CHANNEL'}) {
  195. $NO_CHANNEL = 1;
  196. }
  197. my $NO_EXTRAS = 0;
  198. if ($ENV{'NO_EXTRAS'}) {
  199. $NO_EXTRAS = 1;
  200. }
  201. my $NO_EXCLUDES = 0;
  202. if ($ENV{'NO_EXCLUDES'}) {
  203. $NO_EXCLUDES = 1;
  204. }
  205. my $NO_RENAME = 0;
  206. if ($ENV{'NO_RENAME'}) {
  207. $NO_RENAME = 1;
  208. }
  209. # Environmental parameters (functional)
  210. my $SUDO_CHATTR = 1;
  211. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run($CHATTR)) {
  212. $SUDO_CHATTR = 0;
  213. }
  214. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  215. $MAX_INDEX = $1;
  216. }
  217. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  218. $BATCH_SIZE = $1;
  219. }
  220. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  221. $FETCH_LIMIT = $1;
  222. }
  223. # Construct globals
  224. my $LWP = undef();
  225. foreach my $key (keys(%API)) {
  226. if (exists($API{$key}{'params'}{'maxResults'})) {
  227. $API{$key}{'params'}{'maxResults'} = $BATCH_SIZE;
  228. }
  229. }
  230. if ($DEBUG > 1) {
  231. push(@YTDL_ARGS, @YTDL_DEBUG);
  232. } elsif (!$DEBUG) {
  233. push(@YTDL_ARGS, @YTDL_QUIET);
  234. }
  235. # Allow use as a subscription manager
  236. if ($SUBSCRIPTIONS) {
  237. my %subs = ();
  238. foreach my $user (keys(%USERS)) {
  239. my $tmp = getSubscriptions($USERS{$user});
  240. foreach my $sub (keys(%{$tmp})) {
  241. $subs{$sub} = $tmp->{$sub};
  242. }
  243. }
  244. saveSubscriptions($DIR, \%subs);
  245. exit(0);
  246. }
  247. # Disable channel fetching if the ID channel is "None"
  248. if ($ID =~ /^\s*None\s*$/i) {
  249. if ($DEBUG) {
  250. print STDERR 'Channel features disabled for: ' . $NAME . "\n";
  251. }
  252. $NO_CHANNEL = 1;
  253. $NO_SEARCH = 1;
  254. }
  255. # Run the local "update" script if any
  256. {
  257. my @update = ('./update', $DIR);
  258. if (can_run($update[0])) {
  259. if ($DEBUG) {
  260. print STDERR "Executing local update script...\n";
  261. }
  262. system(@update);
  263. }
  264. }
  265. # Grab the channel data
  266. my $channel = {};
  267. if (!$NO_CHANNEL) {
  268. $channel = getChannel($ID);
  269. saveChannel($channel);
  270. }
  271. # Find all the user's videos on YT
  272. my $videos = {};
  273. if (!$NO_SEARCH) {
  274. $videos = findVideos($ID);
  275. }
  276. # Find any requested "extra" videos
  277. my $extras = {};
  278. if (!$NO_EXTRAS) {
  279. $extras = addExtras($videos);
  280. }
  281. # Drop any "excludes" videos
  282. my $excludes = {};
  283. if (!$NO_EXCLUDES) {
  284. $excludes = dropExcludes($videos);
  285. }
  286. # Find all existing YT files on disk
  287. my $files = {};
  288. if (!$NO_FILES) {
  289. $files = findFiles($videos);
  290. }
  291. # Deal with unknown videos
  292. foreach my $id (keys(%{$files})) {
  293. if ((!exists($videos->{$id}) || !$videos->{$id}->{'date'}) && $files->{$id}->{'season'} > 0) {
  294. # Check the ID specifically to find things the APIv3 fuzzy search misses
  295. if ($DEBUG) {
  296. print STDERR 'Validating local video by ID: ' . $id . "\n";
  297. }
  298. my $video = getVideoData([$id]);
  299. if ($video->[0] && $video->[0]->{'season'} > 0 && $video->[0]->{'channelId'} eq $ID) {
  300. $videos->{$id} = $video->[0];
  301. next;
  302. }
  303. # Whine when videos do go away
  304. if ($DEBUG) {
  305. warn('Local video not known to YT channel (' . $NAME . '): ' . $id . "\n");
  306. }
  307. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  308. }
  309. }
  310. # Calculate the episode number using the publish dates
  311. {
  312. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  313. my $num = undef();
  314. my $lastSeason = undef();
  315. foreach my $id (@byDate) {
  316. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  317. $num = 1;
  318. $lastSeason = $videos->{$id}->{'season'};
  319. }
  320. $videos->{$id}->{'number'} = $num++;
  321. }
  322. }
  323. # HTML output
  324. if ($API_HTML) {
  325. my $html = undef();
  326. my $hdate = time() - (86400 * 28);
  327. open($html, '>', $API_HTML) or warn('Could not open file "' . $API_HTML . '": ' . $!);
  328. if (!$html) {
  329. goto END_HTML;
  330. }
  331. print $html '<html><head><title>' . $NAME . '(' . $ID . ')' . '</title></head>' . "\n";
  332. print $html '<body><form method="post" action="https://firewall.local/yt/update.py">' . "\n";
  333. print $html '<h1>' . $NAME . '</h1><table>' . "\n";
  334. my @byDate = sort { $videos->{$b}->{'date'} <=> $videos->{$a}->{'date'} || $a cmp $b } keys %{$videos};
  335. foreach my $id (@byDate) {
  336. my $val = 'Block';
  337. if ($videos->{$id}->{'date'} > $hdate) {
  338. $val = 'Allow';
  339. }
  340. print $html '<tr>';
  341. print $html '<td>' . time2str('%Y-%m-%d', $videos->{$id}->{'date'}) . '</td>';
  342. print $html '<td><a target="_blank" rel="noopener" href="https://www.youtube.com/watch?v='
  343. . $id . '">' . $videos->{$id}->{'title'} . '</a></td>';
  344. print $html '<td><input type="submit" name="ytid|' . $id . '" value="' . $val . '" /></td>';
  345. print $html "</tr>\n";
  346. }
  347. print $html '</table></form></body></html>' . "\n";
  348. close($html);
  349. END_HTML:
  350. }
  351. # Fill in missing videos and NFOs
  352. my $fetched = 0;
  353. FETCH_LOOP: foreach my $id (keys(%{$videos})) {
  354. if ($DEBUG > 1) {
  355. print STDERR 'Checking remote video: ' . $id . "\n";
  356. if (!exists($files->{$id})) {
  357. print STDERR "\tLocal file: <missing>\n";
  358. } else {
  359. print STDERR "\tLocal media: " . $files->{$id}->{'path'} . "\n";
  360. print STDERR "\tLocal NFO: ";
  361. if ($files->{$id}->{'nfo'}) {
  362. print STDERR $files->{$id}->{'nfo'} . "\n";
  363. } else {
  364. print STDERR "<none>\n";
  365. }
  366. }
  367. }
  368. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  369. # Determine if we may or must rename
  370. my $rename = 0;
  371. if (exists($files->{$id})) {
  372. # Rename if we drift
  373. if (!$rename && $files->{$id}->{'path'}) {
  374. if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  375. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  376. {
  377. if ($DEBUG) {
  378. print STDERR "Rename due to drift\n";
  379. }
  380. $rename = 1;
  381. }
  382. }
  383. # Always rename if the NFO does not match the media
  384. if (!$rename && $files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
  385. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  386. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  387. if ($DEBUG) {
  388. print STDERR "Rename due to media-metadata mismatch\n";
  389. }
  390. $rename = 1;
  391. }
  392. }
  393. }
  394. # Warn (and optionally rename) as selected
  395. if ($rename) {
  396. if ($DEBUG) {
  397. warn('Video ' . $id . ' had number ' . $files->{$id}->{'number'} . ' but now has number ' . $videos->{$id}->{'number'} . "\n");
  398. }
  399. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  400. }
  401. # If we haven't heard of the file, or don't have an NFO for it
  402. # Checking for the NFO allows use to resume failed downloads
  403. if (!exists($files->{$id}) || !-e $nfo) {
  404. if ($DEBUG) {
  405. print STDERR 'Fetching video: ' . $id . "\n";
  406. }
  407. # Let youtube-dl handle the URLs and downloading
  408. {
  409. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  410. my @name = ($YTDL_BIN);
  411. push(@name, @YTDL_ARGS);
  412. my @fetch = @name;
  413. push(@name, '--get-filename');
  414. push(@fetch, @args);
  415. push(@name, @args);
  416. if ($NO_FETCH) {
  417. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  418. } else {
  419. # Count fetch attempts (even if they fail later)
  420. $fetched++;
  421. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  422. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $NAME . "\n";
  423. if ($DEBUG) {
  424. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  425. }
  426. exit 1;
  427. }
  428. # Find the output file name
  429. if ($DEBUG > 1) {
  430. print STDERR join(' ', @name) . "\n";
  431. }
  432. delay();
  433. my $file = capture(EXIT_ANY, @name);
  434. if ($EXITVAL != 0) {
  435. warn('Error executing youtube-dl for name: ' . $NAME . '/' . $id . ' (' . $EXITVAL . ")\n");
  436. next FETCH_LOOP;
  437. }
  438. if ($file =~ /\n\S/) {
  439. $file =~ s/^.*\n//;
  440. }
  441. $file =~ s/^\s+//;
  442. $file =~ s/\s+$//;
  443. # Sanity check
  444. if (!$file) {
  445. warn('No file name available for video: ' . $NAME . '/' . $id . "\n");
  446. next FETCH_LOOP;
  447. } elsif ($DEBUG > 0) {
  448. print STDERR 'Output video file: ' . $file . "\n";
  449. }
  450. # Download
  451. if ($DEBUG > 1) {
  452. print STDERR join(' ', @fetch) . "\n";
  453. }
  454. delay();
  455. my $exit = run(EXIT_ANY, @fetch);
  456. if ($exit != 0) {
  457. if ($DEBUG) {
  458. warn('Error executing youtube-dl for video: ' . $NAME . '/' . $id . "\n");
  459. }
  460. next FETCH_LOOP;
  461. }
  462. # Ensure we found something useful
  463. if (-e $file . '.part') {
  464. warn('Partial download detected: ' . $NAME . '/' . $file . "\n");
  465. next FETCH_LOOP;
  466. }
  467. # Sometimes the downloader lies about file extensions
  468. if (!-s $file) {
  469. $file =~ s/\.webm$/.mp4/;
  470. }
  471. if (!-s $file) {
  472. $file =~ s/\.mp4$/.webm/;
  473. }
  474. # But we still need *a* file
  475. if (!-s $file) {
  476. warn('No output video file: ' . $NAME . '/' . $file . "\n");
  477. next FETCH_LOOP;
  478. }
  479. # Touch the file to reflect the download time rather than the upload time
  480. touch($file);
  481. }
  482. }
  483. # Build and save the XML document
  484. my $xml = buildNFO($videos->{$id});
  485. if ($DEBUG > 1) {
  486. print STDERR 'Saving NFO: ' . $xml . "\n";
  487. }
  488. if ($NO_NFO) {
  489. print STDERR "Not saving NFO\n";
  490. } else {
  491. saveString($nfo, $xml);
  492. }
  493. }
  494. }
  495. sub saveString($$) {
  496. my ($path, $str) = @_;
  497. my $fh = undef();
  498. if (!open($fh, '>', $path)) {
  499. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  500. return undef();
  501. }
  502. print $fh $str;
  503. close($fh);
  504. return 1;
  505. }
  506. sub buildSeriesNFO($) {
  507. my ($channel) = @_;
  508. # Create an XML tree
  509. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  510. $doc->setStandalone(1);
  511. my $show = $doc->createElement('tvshow');
  512. $doc->setDocumentElement($show);
  513. my $elm;
  514. # Add data
  515. $elm = $doc->createElement('title');
  516. $elm->appendText($channel->{'title'});
  517. $show->appendChild($elm);
  518. $elm = $doc->createElement('premiered');
  519. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  520. $show->appendChild($elm);
  521. $elm = $doc->createElement('plot');
  522. $elm->appendText($channel->{'description'});
  523. $show->appendChild($elm);
  524. # Return the string
  525. return $doc->toString();
  526. }
  527. sub buildNFO($) {
  528. my ($video) = @_;
  529. # Create an XML tree
  530. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  531. $doc->setStandalone(1);
  532. my $show = $doc->createElement('episodedetails');
  533. $doc->setDocumentElement($show);
  534. my $file = $doc->createElement('fileinfo');
  535. $show->appendChild($file);
  536. my $stream = $doc->createElement('streamdetails');
  537. $file->appendChild($stream);
  538. my $stream_video = $doc->createElement('video');
  539. $stream->appendChild($stream_video);
  540. my $elm;
  541. # Add data
  542. $elm = $doc->createElement('season');
  543. $elm->appendText($video->{'season'});
  544. $show->appendChild($elm);
  545. $elm = $doc->createElement('episode');
  546. $elm->appendText(videoNumberStr($video->{'season'}, $video->{'number'}));
  547. $show->appendChild($elm);
  548. if (!defined($video->{'title'})) {
  549. $video->{'title'} = 'Episode ' . $video->{'number'};
  550. }
  551. $elm = $doc->createElement('title');
  552. $elm->appendText($video->{'title'});
  553. $show->appendChild($elm);
  554. if (!defined($video->{'date'})) {
  555. $video->{'date'} = time();
  556. }
  557. $elm = $doc->createElement('aired');
  558. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  559. $show->appendChild($elm);
  560. if (defined($video->{'description'})) {
  561. $elm = $doc->createElement('plot');
  562. $elm->appendText($video->{'description'});
  563. $show->appendChild($elm);
  564. }
  565. if (defined($video->{'duration'})) {
  566. $elm = $doc->createElement('runtime');
  567. $elm->appendText($video->{'duration'});
  568. $show->appendChild($elm);
  569. $elm = $doc->createElement('durationinseconds');
  570. $elm->appendText($video->{'duration'});
  571. $stream_video->appendChild($elm);
  572. }
  573. if (defined($video->{'creator'})) {
  574. $elm = $doc->createElement('director');
  575. $elm->appendText($video->{'creator'});
  576. $show->appendChild($elm);
  577. }
  578. # Return the string
  579. return $doc->toString();
  580. }
  581. sub findFiles($) {
  582. my ($videos) = @_;
  583. my %files = ();
  584. our $NAME;
  585. # Allow complete bypass
  586. if ($NO_FILES) {
  587. return \%files;
  588. }
  589. # Read the output directory
  590. my $fh = undef();
  591. opendir($fh, '.')
  592. or die($NAME . ': Unable to open files directory: ' . $! . "\n");
  593. while (my $file = readdir($fh)) {
  594. my ($season, $number, $id, $suffix) = parseFilename($file);
  595. if (defined($id) && length($id) > 0) {
  596. # Determine if there is another video with the same season/number
  597. my $exists = 0;
  598. if ( exists($videos->{$id})
  599. && exists($videos->{$id}->{'season'})
  600. && exists($videos->{$id}->{'number'})
  601. && $season == $videos->{$id}->{'season'}
  602. && $number == $videos->{$id}->{'number'})
  603. {
  604. $exists = 1;
  605. }
  606. # Create the record as needed
  607. if (!exists($files{$id})) {
  608. my %tmp = (
  609. 'season' => $season,
  610. 'number' => $number,
  611. );
  612. $files{$id} = \%tmp;
  613. }
  614. # Video or NFO?
  615. my $type = 'path';
  616. if ($suffix eq 'nfo') {
  617. $type = 'nfo';
  618. }
  619. # Deal with duplicates
  620. if (exists($files{$id}->{$type})) {
  621. if ($type eq 'nfo') {
  622. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  623. if (!$NO_RENAME) {
  624. my $del = $file;
  625. if ($exists) {
  626. $del = $files{$id}->{'nfo'};
  627. }
  628. warn("\tDeleting: " . $file . "\n");
  629. if ($SUDO_CHATTR) {
  630. system('sudo', $CHATTR, '-i', Cwd::abs_path($file));
  631. }
  632. unlink($file);
  633. next;
  634. }
  635. } else {
  636. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  637. if (!$NO_RENAME) {
  638. my $del = $file;
  639. if ($exists) {
  640. $del = $files{$id}->{'path'};
  641. } elsif ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  642. $del = $files{$id}->{'path'};
  643. } elsif ($suffix ne 'webm' && $files{$id}->{'suffix'} eq 'webm') {
  644. $del = $files{$id}->{'path'};
  645. }
  646. warn("\tDeleting: " . $del . "\n");
  647. if ($SUDO_CHATTR) {
  648. system('sudo', $CHATTR, '-i', Cwd::abs_path($del));
  649. }
  650. unlink($del);
  651. next;
  652. }
  653. }
  654. }
  655. # Assign the component paths
  656. $files{$id}->{$type} = $file;
  657. if ($type ne 'nfo') {
  658. $files{$id}->{'suffix'} = $suffix;
  659. }
  660. }
  661. }
  662. close($fh);
  663. if ($DEBUG) {
  664. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  665. if ($DEBUG > 1) {
  666. print STDERR prettyPrint(\%files, "\t") . "\n";
  667. }
  668. }
  669. return \%files;
  670. }
  671. sub fetchParse($$) {
  672. my ($name, $params) = @_;
  673. our $NAME;
  674. my $url = $API{$name}{'url'} . '?';
  675. foreach my $key (keys(%{ $API{$name}{'params'} }), keys(%{$params})) {
  676. $url .= '&' . uri_escape($key) . '=';
  677. if (exists($params->{$key})) {
  678. $url .= uri_escape($params->{$key});
  679. } else {
  680. $url .= uri_escape($API{$name}{'params'}{$key});
  681. }
  682. }
  683. # Fetch
  684. if ($DEBUG > 1) {
  685. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  686. }
  687. delay();
  688. my $content;
  689. my $code = fetch($url, \$content);
  690. if ($code != 200 || !defined($content) || length($content) < 10) {
  691. my $msg = $NAME . ': Invalid content from URL (' . $code . '): ' . $url . "\n" . $content . "\n";
  692. if ($code >= 500 && $code < 600) {
  693. if ($DEBUG) {
  694. print STDERR $msg;
  695. }
  696. exit(0);
  697. }
  698. if ($code >= 400 && $code < 500) {
  699. if ($DEBUG) {
  700. print STDERR "Waiting 30 seconds after 400 error\n";
  701. }
  702. delay(30);
  703. }
  704. die($msg);
  705. }
  706. # Parse
  707. if ($DEBUG > 2) {
  708. print STDERR "Raw JSON data:\n" . $content . "\n";
  709. }
  710. my $data = decode_json($content);
  711. if (!defined($data) || ref($data) ne 'HASH') {
  712. die($NAME . ': Invalid JSON: ' . $content . "\n");
  713. }
  714. return $data;
  715. }
  716. sub getSubscriptions($) {
  717. my ($id) = @_;
  718. our $NAME;
  719. # Loop through until we have all the entries
  720. my $pageToken = '';
  721. my %subs = ();
  722. SUBS_LOOP:
  723. {
  724. # Build, fetch, parse, check
  725. my $data = fetchParse('subscriptions', { 'channelId' => $id, 'pageToken' => $pageToken });
  726. $pageToken = '';
  727. if ( exists($data->{'pageInfo'})
  728. && ref($data->{'pageInfo'}) eq 'HASH'
  729. && exists($data->{'pageInfo'}->{'totalResults'})
  730. && $data->{'pageInfo'}->{'totalResults'} > 0
  731. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  732. && exists($data->{'items'})
  733. && ref($data->{'items'}) eq 'ARRAY')
  734. {
  735. if (exists($data->{'nextPageToken'})) {
  736. $pageToken = $data->{'nextPageToken'};
  737. }
  738. $data = $data->{'items'};
  739. } else {
  740. die($NAME . ": Invalid subscription data\n");
  741. }
  742. foreach my $item (@{$data}) {
  743. if ( !exists($item->{'snippet'})
  744. || ref($item->{'snippet'}) ne 'HASH'
  745. || !exists($item->{'snippet'}->{'title'})
  746. || !exists($item->{'snippet'}->{'resourceId'})
  747. || ref($item->{'snippet'}->{'resourceId'}) ne 'HASH'
  748. || !exists($item->{'snippet'}->{'resourceId'}->{'channelId'}))
  749. {
  750. if ($DEBUG > 2) {
  751. warn($NAME . ": Skipping invalid subscription\n");
  752. print STDERR prettyPrint($item, "\t") . "\n";
  753. }
  754. next;
  755. }
  756. if ($DEBUG) {
  757. print STDERR $item->{'snippet'}->{'title'} . ' => ' . $item->{'snippet'}->{'resourceId'}->{'channelId'} . "\n";
  758. }
  759. $subs{ $item->{'snippet'}->{'resourceId'}->{'channelId'} } = $item->{'snippet'}->{'title'};
  760. }
  761. # Loop if there are results left to fetch
  762. if ($pageToken) {
  763. redo SUBS_LOOP;
  764. }
  765. }
  766. # Return the list of subscribed channel IDs
  767. return \%subs;
  768. }
  769. sub saveSubscriptions($$) {
  770. my ($folder, $subs) = @_;
  771. our $NAME;
  772. # Check for local subscriptions missing from YT
  773. my %locals = ();
  774. my $fh = undef();
  775. opendir($fh, '.')
  776. or die($NAME . ': Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  777. while (my $file = readdir($fh)) {
  778. # Skip dotfiles
  779. if ($file =~ /^\./) {
  780. next;
  781. }
  782. # Skip non-directories
  783. if (!-d $file) {
  784. next;
  785. }
  786. # Extract the YTID
  787. my ($id) = $file =~ /\(([\w\-]+)\)$/;
  788. if (!$id) {
  789. warn('Skipping invalid local subscription: ' . $file . "\n");
  790. next;
  791. }
  792. # Skip "None" directories (i.e. non-channel subscriptions)
  793. if ($id eq 'None') {
  794. if ($DEBUG) {
  795. print STDERR 'Ignoring non-channel subscription: ' . $file . "\n";
  796. }
  797. next;
  798. }
  799. # Skip the weird dual Ask a Ninja thing, which vascilates
  800. if ($id eq 'UCpYQ53cR1p2VzFijrXYz4WQ') {
  801. if ($DEBUG) {
  802. print STDERR 'Ignoring weird Ask a Ninja subscription: ' . $file . "\n";
  803. }
  804. next;
  805. }
  806. # Anything else should be in the list
  807. if (!$subs->{$id}) {
  808. my (undef(), undef(), $hour) = localtime(time);
  809. if ($hour == 0 || $DEBUG) {
  810. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  811. }
  812. }
  813. # Note local subscriptions
  814. $locals{$id} = 1;
  815. }
  816. closedir($fh);
  817. # Check for YT subscriptions missing locally
  818. foreach my $id (keys(%{$subs})) {
  819. if (!exists($locals{$id})) {
  820. my $dir = $subs->{$id} . ' (' . $id . ')';
  821. print STDERR 'Adding local subscription for: ' . $dir . "\n";
  822. mkdir($folder . '/' . $dir);
  823. }
  824. }
  825. }
  826. sub saveChannel($) {
  827. my ($channel) = @_;
  828. my $nfo = 'tvshow.nfo';
  829. if (!-e $nfo) {
  830. if ($DEBUG) {
  831. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  832. }
  833. # Save the poster
  834. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  835. my $jpg;
  836. if (fetch($channel->{'thumbnail'}, \$jpg) == 200) {
  837. saveString('poster.jpg', $jpg);
  838. }
  839. }
  840. # Save the series NFO
  841. my $xml = buildSeriesNFO($channel);
  842. if ($DEBUG > 1) {
  843. print STDERR 'Saving NFO: ' . $xml . "\n";
  844. }
  845. saveString($nfo, $xml);
  846. }
  847. }
  848. sub getChannel($) {
  849. my ($id) = @_;
  850. our $NAME;
  851. # Build, fetch, parse, check
  852. my $data = fetchParse('channel', { 'id' => $id });
  853. if ( exists($data->{'pageInfo'})
  854. && ref($data->{'pageInfo'}) eq 'HASH'
  855. && exists($data->{'pageInfo'}->{'totalResults'})
  856. && $data->{'pageInfo'}->{'totalResults'} == 1
  857. && exists($data->{'items'})
  858. && ref($data->{'items'}) eq 'ARRAY'
  859. && scalar(@{ $data->{'items'} }) > 0
  860. && ref($data->{'items'}[0]) eq 'HASH'
  861. && exists($data->{'items'}[0]->{'snippet'})
  862. && ref($data->{'items'}[0]->{'snippet'}) eq 'HASH')
  863. {
  864. $data = $data->{'items'}[0]->{'snippet'};
  865. } else {
  866. die($NAME . ": Invalid channel data\n");
  867. }
  868. # Extract the data we want
  869. my %channel = (
  870. 'id' => $id,
  871. 'title' => $data->{'title'},
  872. 'date' => str2time($data->{'publishedAt'}),
  873. 'description' => $data->{'description'},
  874. 'thumbnail' => $data->{'thumbnails'}->{'high'}->{'url'},
  875. );
  876. return \%channel;
  877. }
  878. sub readExcludes() {
  879. my %excludes = ();
  880. our $NAME;
  881. # Read and parse the excludes videos file, if it exists
  882. if (-e $EXCLUDES_FILE) {
  883. my $fh;
  884. open($fh, $EXCLUDES_FILE)
  885. or die($NAME . ': Unable to open excludes videos file: ' . $! . "\n");
  886. while (<$fh>) {
  887. # Skip blank lines and comments
  888. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  889. next;
  890. }
  891. # Match our specific format or whine
  892. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  893. if ($DEBUG > 1) {
  894. print STDERR 'Adding exclude video: ' . $1 . "\n";
  895. }
  896. $excludes{$1} = 1;
  897. } else {
  898. print STDERR 'Skipped exclude video line: ' . $_;
  899. }
  900. }
  901. close($fh);
  902. }
  903. return \%excludes;
  904. }
  905. sub readExtras() {
  906. my %extras = ();
  907. our $NAME;
  908. # Read and parse the extra videos file, if it exists
  909. if (-e $EXTRAS_FILE) {
  910. my $fh;
  911. open($fh, $EXTRAS_FILE)
  912. or die($NAME . ': Unable to open extra videos file: ' . $! . "\n");
  913. while (<$fh>) {
  914. # Skip blank lines and comments
  915. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  916. next;
  917. }
  918. # Match our specific format or whine
  919. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  920. if ($DEBUG > 1) {
  921. print STDERR 'Adding extra video: ' . $1 . "\n";
  922. }
  923. $extras{$1} = 1;
  924. } else {
  925. print STDERR 'Skipped extra video line: ' . $_;
  926. }
  927. }
  928. close($fh);
  929. }
  930. return \%extras;
  931. }
  932. sub parseVideoData($) {
  933. my ($data) = @_;
  934. my %video = %VIDEO_FIELDS;
  935. # Sanity check
  936. if ( !exists($data->{'id'})
  937. || !exists($data->{'snippet'})
  938. || ref($data->{'snippet'}) ne 'HASH'
  939. || !exists($data->{'contentDetails'})
  940. || ref($data->{'contentDetails'}) ne 'HASH'
  941. || !exists($data->{'contentDetails'}->{'duration'}))
  942. {
  943. if ($DEBUG > 2) {
  944. print STDERR "Unable to parse video data:\n" . prettyPrint($data, "\t") . "\n";
  945. }
  946. return undef();
  947. }
  948. # Direct extraction
  949. $video{'id'} = $data->{'id'};
  950. $video{'title'} = $data->{'snippet'}->{'title'};
  951. $video{'description'} = $data->{'snippet'}->{'description'};
  952. $video{'creator'} = $data->{'snippet'}->{'channelTitle'};
  953. $video{'publishedAt'} = $data->{'snippet'}->{'publishedAt'};
  954. $video{'channelId'} = $data->{'snippet'}->{'channelId'};
  955. $video{'rawDuration'} = $data->{'contentDetails'}->{'duration'};
  956. # Convert dates and whatnot
  957. calcVideoData(\%video);
  958. return \%video;
  959. }
  960. sub calcVideoData($) {
  961. my ($video) = @_;
  962. if ($video->{'publishedAt'} && !$video->{'date'}) {
  963. $video->{'date'} = str2time($video->{'publishedAt'});
  964. }
  965. if ($video->{'date'}) {
  966. $video->{'season'} = time2str('%Y', $video->{'date'});
  967. } else {
  968. $video->{'season'} = 0;
  969. }
  970. if ($video->{'rawDuration'} =~ /PT(\d+)H(\d+)M(\d+)S/) {
  971. $video->{'duration'} = ($1 * 3600) + ($2 * 60) + $2;
  972. } elsif ($video->{'rawDuration'} =~ /PT(\d+)M(\d+)S/) {
  973. $video->{'duration'} = ($1 * 60) + $2;
  974. } elsif ($video->{'rawDuration'} =~ /PT(\d+)S/) {
  975. $video->{'duration'} = $1;
  976. }
  977. }
  978. sub dropExcludes($) {
  979. my ($videos) = @_;
  980. my $excludes = readExcludes();
  981. foreach my $id (keys(%{$excludes})) {
  982. if (!exists($videos->{$id})) {
  983. if ($DEBUG > 1) {
  984. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  985. }
  986. next;
  987. }
  988. delete($videos->{$id});
  989. }
  990. return $excludes;
  991. }
  992. sub addExtras($) {
  993. my ($videos) = @_;
  994. my $extras = readExtras();
  995. foreach my $id (keys(%{$extras})) {
  996. if (exists($videos->{$id})) {
  997. if ($DEBUG > 1) {
  998. print STDERR 'Skipping known extra video: ' . $id . "\n";
  999. }
  1000. next;
  1001. }
  1002. my $video = getVideoData([$id]);
  1003. $videos->{$id} = $video->[0];
  1004. }
  1005. return $extras;
  1006. }
  1007. sub getVideoData($) {
  1008. my ($ids) = @_;
  1009. my @videos = ();
  1010. our $NAME;
  1011. my %found = ();
  1012. # Do a batch request for the entire batch of video data
  1013. my $data = fetchParse('video', { 'id' => join(',', @{$ids}) });
  1014. if ( exists($data->{'pageInfo'})
  1015. && ref($data->{'pageInfo'}) eq 'HASH'
  1016. && exists($data->{'pageInfo'}->{'totalResults'})
  1017. && exists($data->{'items'})
  1018. && ref($data->{'items'}) eq 'ARRAY')
  1019. {
  1020. if (scalar(@{$ids}) > 1 && $data->{'pageInfo'}->{'totalResults'} != scalar(@{$ids})) {
  1021. if ($DEBUG) {
  1022. die($NAME . ': Video/metadata search count mismatch: ' . $data->{'pageInfo'}->{'totalResults'} . '/' . scalar(@{$ids}) . "\n");
  1023. }
  1024. exit(0);
  1025. }
  1026. $data = $data->{'items'};
  1027. } else {
  1028. die($NAME . ": Invalid search video data\n" . prettyPrint($data, "\t") . "\n");
  1029. }
  1030. # Build each metadata record
  1031. foreach my $item (@{$data}) {
  1032. if (!exists($item->{'snippet'}) || ref($item->{'snippet'}) ne 'HASH') {
  1033. warn($NAME . ": Skipping invalid metadata item\n");
  1034. if ($DEBUG > 2) {
  1035. print STDERR prettyPrint($item, "\t") . "\n";
  1036. }
  1037. next;
  1038. }
  1039. my $video = parseVideoData($item);
  1040. if (!$video) {
  1041. die($NAME . ": Unable to parse video data\n");
  1042. }
  1043. # Allow local metadata overrides
  1044. localMetadata($video->{'id'}, $video);
  1045. # Save
  1046. $found{ $video->{'id'} } = 1;
  1047. push(@videos, $video);
  1048. }
  1049. # Load local metadata even for videos YT doesn't know anything about
  1050. foreach my $id (@{$ids}) {
  1051. if ($found{$id}) {
  1052. next;
  1053. }
  1054. my %video = %VIDEO_FIELDS;
  1055. $video{'id'} = $id;
  1056. localMetadata($id, \%video);
  1057. # Save
  1058. $found{$id} = 1;
  1059. push(@videos, \%video);
  1060. }
  1061. # Parse the video data
  1062. return \@videos;
  1063. }
  1064. sub localMetadata($$) {
  1065. my ($id, $video) = @_;
  1066. my $meta = $id . '.meta';
  1067. if (!-r $meta) {
  1068. return;
  1069. }
  1070. my $fh = undef();
  1071. open($fh, '<', $meta)
  1072. or warn('Unable to open local metadata file: ' . $meta . ': ' . $! . ". Skipping...\n");
  1073. while (<$fh>) {
  1074. if (/^\s*#/ || /^\s*$/) {
  1075. next;
  1076. }
  1077. if (/^\s*([^\s\=][^\=]*[^\s\=])\s*=\>?\s*(\S.*\S)\s*$/) {
  1078. if (exists($video->{$1})) {
  1079. if ($DEBUG) {
  1080. print STDERR 'Using local metadata line (' . $id . '): ' . $1 . ' => ' . $2 . "\n";
  1081. }
  1082. $video->{$1} = $2;
  1083. } else {
  1084. die('Invalid local metadata attribute (' . $1 . '): ' . $meta . "\n");
  1085. }
  1086. } else {
  1087. die('Invalid local metdata line (' . $meta . '): ' . chomp($_) . "\n");
  1088. }
  1089. }
  1090. close($fh);
  1091. calcVideoData($video);
  1092. }
  1093. sub findVideos($) {
  1094. my ($id) = @_;
  1095. my %videos = ();
  1096. our $NAME;
  1097. # Allow complete bypass
  1098. if ($NO_SEARCH) {
  1099. return \%videos;
  1100. }
  1101. # Loop through until we have all the entries
  1102. my $totalCount = 0;
  1103. my %params = ('channelId' => $id);
  1104. LOOP:
  1105. {
  1106. # Loop status
  1107. my $count = scalar(keys(%videos));
  1108. if ($DEBUG && $totalCount) {
  1109. my $date = $params{'publishedBefore'};
  1110. $date =~ s/T.*$//;
  1111. print STDERR 'Fetching ' . $count . ' of ' . $totalCount .
  1112. "\t" . $date .
  1113. "\t(" . int(100 * $count / $totalCount) . "%)\n";
  1114. }
  1115. # Build, fetch, parse, check
  1116. my $data = fetchParse('search', \%params);
  1117. delete($params{'pageToken'});
  1118. if ( exists($data->{'pageInfo'})
  1119. && ref($data->{'pageInfo'}) eq 'HASH'
  1120. && exists($data->{'pageInfo'}->{'totalResults'})
  1121. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  1122. && exists($data->{'items'})
  1123. && ref($data->{'items'}) eq 'ARRAY')
  1124. {
  1125. if (!$totalCount) {
  1126. $totalCount = $data->{'pageInfo'}->{'totalResults'};
  1127. }
  1128. $data = $data->{'items'};
  1129. } else {
  1130. die($NAME . ": Invalid search data\n");
  1131. }
  1132. # Sanity check
  1133. if (scalar(@{$data}) < 1) {
  1134. last LOOP;
  1135. }
  1136. # Grab each video ID
  1137. my @ids = ();
  1138. foreach my $item (@{$data}) {
  1139. if (!exists($item->{'id'}) || ref($item->{'id'}) ne 'HASH' || !exists($item->{'id'}->{'videoId'})) {
  1140. warn($NAME . ": Skipping invalid video item\n");
  1141. if ($DEBUG > 2) {
  1142. print STDERR prettyPrint($data, "\t") . "\n";
  1143. }
  1144. next;
  1145. }
  1146. push(@ids, $item->{'id'}->{'videoId'});
  1147. }
  1148. # Do a batch request for the entire batch of video data
  1149. my $more = 0;
  1150. my $records = getVideoData(\@ids);
  1151. foreach my $rec (@{$records}) {
  1152. if (exists($rec->{'duration'}) && $rec->{'duration'} == 0) {
  1153. if ($DEBUG > 1) {
  1154. print STDERR 'Skipping 0-length video: ' . $rec->{'id'} . "\n";
  1155. }
  1156. next;
  1157. }
  1158. $videos{ $rec->{'id'} } = $rec;
  1159. if (!defined($params{'publishedBefore'}) || (defined($rec->{'publishedAt'}) && ($rec->{'publishedAt'} cmp $params{'publishedBefore'}) < 0)) {
  1160. $params{'publishedBefore'} = $rec->{'publishedAt'};
  1161. $more = 1;
  1162. }
  1163. }
  1164. # Loop if there are results left to fetch
  1165. if ($DEBUG > 1) {
  1166. print STDERR "more \t => ${more}\n";
  1167. print STDERR "count \t => ${count}\n";
  1168. }
  1169. if ($more && $params{'publishedBefore'} && $count <= $MAX_INDEX) {
  1170. redo LOOP;
  1171. }
  1172. }
  1173. # Sanity check/debug
  1174. {
  1175. my $count = scalar(keys(%videos));
  1176. if ($DEBUG) {
  1177. print STDERR 'Found ' . $count . " remote videos\n";
  1178. if (abs(1 - ($count / $totalCount)) > $API_ST_REL && abs($totalCount - $count) > $API_ST_ABS) {
  1179. print STDERR 'Found only ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%) channel-API-reported remote videos.\n";
  1180. }
  1181. if ($DEBUG > 1) {
  1182. print STDERR prettyPrint(\%videos, "\t") . "\n";
  1183. }
  1184. }
  1185. }
  1186. return \%videos;
  1187. }
  1188. sub updateNFOData($$$) {
  1189. my ($file, $season, $episode) = @_;
  1190. my $nfoData = undef();
  1191. {
  1192. open(my $fh, '<', $file)
  1193. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  1194. $/ = undef;
  1195. $nfoData = <$fh>;
  1196. close($fh);
  1197. }
  1198. if ($nfoData) {
  1199. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  1200. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  1201. }
  1202. return $nfoData;
  1203. }
  1204. sub videoNumberStr($$) {
  1205. my ($season, $episode) = @_;
  1206. my $str = '';
  1207. if ($season == 0 && $episode =~ /^(20[012]\d)(\d{2,4})$/) {
  1208. $str = sprintf('%04d%04d', $1, $2);
  1209. } else {
  1210. $str = sprintf('%02d', $episode);
  1211. }
  1212. return $str;
  1213. }
  1214. sub videoSE($$) {
  1215. my ($season, $episode) = @_;
  1216. return sprintf('S%0dE%s - ', $season, videoNumberStr($season, $episode));
  1217. }
  1218. sub videoPath($$$$) {
  1219. my ($season, $episode, $id, $ext) = @_;
  1220. return videoSE($season, $episode) . $id . '.' . $ext;
  1221. }
  1222. sub renameVideo($$$$$$) {
  1223. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  1224. our $NAME;
  1225. # Bail if disabled
  1226. if ($NO_RENAME) {
  1227. print STDERR 'Not renaming: ' . $video . "\n";
  1228. return;
  1229. }
  1230. # General sanity checks
  1231. if (!defined($id) || !defined($season) || !defined($episode)) {
  1232. die($NAME . ": Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  1233. }
  1234. # Warning about useless calls
  1235. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  1236. warn("No NFO or video path provided in renameVideo()\n");
  1237. return;
  1238. }
  1239. # Video
  1240. if (defined($video) && defined($suffix)) {
  1241. my $videoNew = videoPath($season, $episode, $id, $suffix);
  1242. # Sanity checks, as we do dangerous work here
  1243. if (!-r $video) {
  1244. die($NAME . ': Invalid video source in rename: ' . $video . "\n");
  1245. }
  1246. if ($video ne $videoNew) {
  1247. if ($DEBUG) {
  1248. warn('Renaming ' . $NAME . '/' . $video . ' => ' . $videoNew . "\n");
  1249. }
  1250. if (-e $videoNew) {
  1251. die($NAME . ': Rename: Target exists: ' . $videoNew . "\n");
  1252. }
  1253. if ($SUDO_CHATTR) {
  1254. system('sudo', $CHATTR, '-i', Cwd::abs_path($video));
  1255. }
  1256. rename($video, $videoNew)
  1257. or die($NAME . ': Unable to rename: ' . $video . ': ' . $! . "\n");
  1258. }
  1259. }
  1260. # NFO
  1261. if (defined($nfo)) {
  1262. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  1263. # Sanity checks, as we do dangerous work here
  1264. if (!-r $nfo) {
  1265. die($NAME . ': Invalid NFO source in rename: ' . $nfo . "\n");
  1266. }
  1267. # Parse old NFO data
  1268. my $nfoData = updateNFOData($nfo, $season, $episode)
  1269. or die($NAME . ': No NFO data found, refusing to rename: ' . $id . "\n");
  1270. # Write a new NFO and unlink the old one
  1271. if ($SUDO_CHATTR) {
  1272. system('sudo', $CHATTR, '-i', Cwd::abs_path($nfo));
  1273. }
  1274. saveString($nfoNew, $nfoData);
  1275. if ($nfo ne $nfoNew) {
  1276. if ($DEBUG) {
  1277. warn('Renaming ' . $NAME . '/' . $nfo . ' => ' . $nfoNew . "\n");
  1278. }
  1279. unlink($nfo)
  1280. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  1281. }
  1282. }
  1283. }
  1284. sub parseFilename($) {
  1285. my ($file) = @_;
  1286. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)(?:\-recode)?\.(\w\w\w)$/i;
  1287. }
  1288. # GET the requested URL, accepting encoded data and extracting the HTTP status code
  1289. sub fetch($$) {
  1290. my ($url, $data) = @_;
  1291. # Init the global LWP as needed
  1292. if (!defined($LWP)) {
  1293. $LWP = new LWP::UserAgent;
  1294. $LWP->agent($HTTP_UA);
  1295. $LWP->timeout($HTTP_TIMEOUT);
  1296. $LWP->ssl_opts({ 'verify_hostname' => $HTTP_VERIFY });
  1297. }
  1298. # Make a request
  1299. my $request = HTTP::Request->new('GET' => $url);
  1300. $request->header('Accept-Encoding' => HTTP::Message::decodable);
  1301. my $response = $LWP->request($request);
  1302. # Do something useful with the response
  1303. my $code = 400;
  1304. if ($response->status_line =~ /^(\d+)\s/) {
  1305. $code = $1;
  1306. }
  1307. ${$data} = $response->decoded_content();
  1308. return $code;
  1309. }
  1310. # Inter-operation delay
  1311. sub delay(;$) {
  1312. my $retval = 0;
  1313. my ($extra) = ($@);
  1314. if (!Scalar::Util::looks_like_number($extra)) {
  1315. $extra = 0;
  1316. }
  1317. my $sleep = rand($DELAY) + $MIN_DELAY + $extra;
  1318. if ($sleep > $FORK_DELAY) {
  1319. # Fork, signal the loop to exit in the child, and resume the parent
  1320. $retval = 1;
  1321. }
  1322. sleep($sleep);
  1323. return($retval);
  1324. }