subscribe.pl 25 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::Simple;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my @YTDL_DEBUG = ('--verbose');
  25. my $BATCH_SIZE = 50;
  26. my $MAX_INDEX = 25000;
  27. my $FETCH_LIMIT = 50;
  28. my $DRIFT_TOLERANCE = 2;
  29. my $DRIFT_FACTOR = 100.0;
  30. my $DELAY = 5;
  31. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  32. my %API = (
  33. 'search' => {
  34. 'prefix' => $API_URL . 'users/',
  35. 'suffix' => '/uploads',
  36. 'params' => {
  37. 'start-index' => 1,
  38. 'max-results' => 1,
  39. 'strict' => 1,
  40. 'v' => 2,
  41. 'alt' => 'jsonc',
  42. },
  43. },
  44. 'subscriptions' => {
  45. 'prefix' => $API_URL . 'users/',
  46. 'suffix' => '/subscriptions',
  47. 'params' => {
  48. 'start-index' => 1,
  49. 'max-results' => 1,
  50. 'strict' => 1,
  51. 'v' => 2,
  52. 'alt' => 'json',
  53. },
  54. },
  55. 'video' => {
  56. 'prefix' => $API_URL . 'videos/',
  57. 'suffix' => '',
  58. 'params' => {
  59. 'strict' => 1,
  60. 'v' => 2,
  61. 'alt' => 'jsonc'
  62. },
  63. },
  64. 'channel' => {
  65. 'prefix' => $API_URL . 'users/',
  66. 'suffix' => '',
  67. 'params' => {
  68. 'strict' => 1,
  69. 'v' => 2,
  70. 'alt' => 'json'
  71. },
  72. },
  73. );
  74. # Prototypes
  75. sub findVideos($);
  76. sub findFiles();
  77. sub findVideo($);
  78. sub buildNFO($);
  79. sub buildSeriesNFO($);
  80. sub getSubscriptions($$);
  81. sub saveSubscriptions($$);
  82. sub saveChannel($);
  83. sub getChannel($);
  84. sub fetchParse($$);
  85. sub saveString($$);
  86. sub readExcludes();
  87. sub readExtras();
  88. sub parseVideoData($);
  89. sub dropExcludes($);
  90. sub addExtras($);
  91. sub updateNFOData($$$);
  92. sub videoSE($$);
  93. sub videoPath($$$$);
  94. sub renameVideo($$$$$$);
  95. sub parseFilename($);
  96. # Sanity check
  97. if (scalar(@ARGV) < 1) {
  98. die('Usage: ' . basename($0) . " output_directory\n");
  99. }
  100. # Command-line parameters
  101. my ($dir) = @ARGV;
  102. $dir =~ s/\/+$//;
  103. if (!-d $dir) {
  104. die('Invalid output directory: ' . $dir . "\n");
  105. }
  106. my $user = basename($dir);
  107. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  108. die('Invalid user: ' . $user . "\n");
  109. }
  110. # Move to the target directory so we can use relative paths later
  111. chdir($dir);
  112. # Environmental parameters (debug)
  113. my $DEBUG = 0;
  114. if ($ENV{'DEBUG'}) {
  115. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  116. $DEBUG = $1;
  117. } else {
  118. $DEBUG = 1;
  119. }
  120. }
  121. my $NO_FETCH = 0;
  122. if ($ENV{'NO_FETCH'}) {
  123. $NO_FETCH = 1;
  124. }
  125. my $NO_NFO = 0;
  126. if ($ENV{'NO_NFO'}) {
  127. $NO_NFO = 1;
  128. }
  129. my $NO_SEARCH = 0;
  130. if ($ENV{'NO_SEARCH'}) {
  131. $NO_SEARCH = 1;
  132. }
  133. my $NO_FILES = 0;
  134. if ($ENV{'NO_FILES'}) {
  135. $NO_FILES = 1;
  136. }
  137. my $NO_CHANNEL = 0;
  138. if ($ENV{'NO_CHANNEL'}) {
  139. $NO_CHANNEL = 1;
  140. }
  141. my $NO_EXTRAS = 0;
  142. if ($ENV{'NO_EXTRAS'}) {
  143. $NO_EXTRAS = 1;
  144. }
  145. my $NO_EXCLUDES = 0;
  146. if ($ENV{'NO_EXCLUDES'}) {
  147. $NO_EXCLUDES = 1;
  148. }
  149. my $NO_RENAME = 0;
  150. if ($ENV{'NO_RENAME'}) {
  151. $NO_RENAME = 1;
  152. }
  153. # Environmental parameters (functional)
  154. my $FORCE_RENAME = 0;
  155. if ($ENV{'FORCE_RENAME'}) {
  156. $FORCE_RENAME = 1;
  157. }
  158. my $SUDO_CHATTR = 1;
  159. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  160. $SUDO_CHATTR = 0;
  161. }
  162. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  163. $MAX_INDEX = $1;
  164. }
  165. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  166. $BATCH_SIZE = $1;
  167. }
  168. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  169. $FETCH_LIMIT = $1;
  170. }
  171. # Construct globals
  172. foreach my $key (keys(%API)) {
  173. if (exists($API{$key}{'params'}{'max-results'})) {
  174. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  175. }
  176. }
  177. if ($DEBUG > 1) {
  178. push(@YTDL_ARGS, @YTDL_DEBUG);
  179. } elsif (!$DEBUG) {
  180. push(@YTDL_ARGS, @YTDL_QUIET);
  181. }
  182. # Allow use as a subscription manager
  183. if ($0 =~ /subscription/i) {
  184. my %subs = ();
  185. foreach my $user (keys(%USERS)) {
  186. my $tmp = getSubscriptions($user, $USERS{$user});
  187. foreach my $sub (keys(%{$tmp})) {
  188. if (exists($subs{$sub})) {
  189. $subs{$sub} .= ', ' . $tmp->{$sub};
  190. } else {
  191. $subs{$sub} = $tmp->{$sub};
  192. }
  193. }
  194. }
  195. saveSubscriptions($dir, \%subs);
  196. exit(0);
  197. }
  198. # Grab the channel data
  199. my $channel = {};
  200. if (!$NO_CHANNEL) {
  201. $channel = getChannel($user);
  202. saveChannel($channel);
  203. }
  204. # Find all the user's videos on YT
  205. my $videos = {};
  206. if (!$NO_SEARCH) {
  207. $videos = findVideos($user);
  208. }
  209. # Find any requested "extra" videos
  210. my $extras = {};
  211. if (!$NO_EXTRAS) {
  212. $extras = addExtras($videos);
  213. }
  214. # Drop any "excludes" videos
  215. my $excludes = {};
  216. if (!$NO_EXCLUDES) {
  217. $excludes = dropExcludes($videos);
  218. }
  219. # Calculate the episode number using the publish dates
  220. {
  221. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  222. my $num = undef();
  223. my $lastSeason = undef();
  224. foreach my $id (@byDate) {
  225. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  226. $num = 1;
  227. $lastSeason = $videos->{$id}->{'season'};
  228. }
  229. $videos->{$id}->{'number'} = $num++;
  230. }
  231. }
  232. # Find all existing YT files on disk
  233. my $files = {};
  234. if (!$NO_FILES) {
  235. $files = findFiles();
  236. }
  237. # Whine about unknown videos
  238. foreach my $id (keys(%{$files})) {
  239. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  240. print STDERR 'Local video not known to YT channel (' . $user . '): ' . $id . "\n";
  241. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  242. }
  243. }
  244. # Fill in missing videos and NFOs
  245. my $fetched = 0;
  246. foreach my $id (keys(%{$videos})) {
  247. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  248. # Determine if we may or must rename
  249. my $rename = 0;
  250. # Skip remote-only videos
  251. if (exists($files->{$id})) {
  252. if ($files->{$id}->{'nfo'}) {
  253. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  254. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  255. $rename = -1;
  256. }
  257. }
  258. if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  259. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  260. {
  261. $rename = 1;
  262. }
  263. }
  264. # Warn (and optionally rename) if the video numbers drift
  265. if ($rename != 0) {
  266. if ($rename < 0 || $FORCE_RENAME) {
  267. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  268. } else {
  269. # Find the old NFO to avoid re-fetching
  270. $nfo = $files->{$id}->{'nfo'};
  271. # Calculate the drift magnitude
  272. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  273. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  274. if ($tolerance < $DRIFT_TOLERANCE) {
  275. $tolerance = $DRIFT_TOLERANCE;
  276. }
  277. # Ignore small changes
  278. my $rename = 0;
  279. if ($FORCE_RENAME || $delta > $tolerance) {
  280. $rename = 1;
  281. }
  282. # Execute
  283. if ($DEBUG || $rename) {
  284. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  285. if ($rename) {
  286. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  287. }
  288. }
  289. }
  290. }
  291. # If we haven't heard of the file, or don't have an NFO for it
  292. # Checking for the NFO allows use to resume failed downloads
  293. if (!exists($files->{$id}) || !-e $nfo) {
  294. if ($DEBUG) {
  295. print STDERR 'Fetching video: ' . $id . "\n";
  296. }
  297. # Let youtube-dl handle the URLs and downloading
  298. {
  299. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  300. my @name = ($YTDL_BIN);
  301. push(@name, @YTDL_ARGS);
  302. my @fetch = @name;
  303. push(@name, '--get-filename');
  304. push(@fetch, @args);
  305. push(@name, @args);
  306. if ($NO_FETCH) {
  307. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  308. } else {
  309. # Count fetch attempts (even if they fail later)
  310. $fetched++;
  311. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  312. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $user . "\n";
  313. if ($DEBUG) {
  314. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  315. }
  316. exit 1;
  317. }
  318. # Find the output file name
  319. if ($DEBUG > 1) {
  320. print STDERR join(' ', @name) . "\n";
  321. }
  322. sleep($DELAY);
  323. my $file = capture(EXIT_ANY, @name);
  324. if ($EXITVAL != 0) {
  325. warn('Error executing youtube-dl (name): ' . $EXITVAL . "\n");
  326. next;
  327. }
  328. $file =~ s/^\s+//;
  329. $file =~ s/\s+$//;
  330. # Sanity check
  331. if (!$file) {
  332. warn('No file name available for video: ' . $id . "\n");
  333. next;
  334. }
  335. # Download
  336. if ($DEBUG > 1) {
  337. print STDERR join(' ', @fetch) . "\n";
  338. }
  339. sleep($DELAY);
  340. my $exit = run(EXIT_ANY, @fetch);
  341. if ($exit != 0) {
  342. warn('Error executing youtube-dl for video: ' . $file . "\n");
  343. next;
  344. }
  345. # Ensure we found something useful
  346. if (-e $file . '.part') {
  347. warn('Partial download detected: ' . $file . "\n");
  348. next;
  349. }
  350. if (!-s $file) {
  351. warn('No output video file: ' . $file . "\n");
  352. next;
  353. }
  354. # Touch the file to reflect the download time rather than the upload time
  355. touch($file);
  356. }
  357. }
  358. # Build and save the XML document
  359. my $xml = buildNFO($videos->{$id});
  360. if ($DEBUG > 1) {
  361. print STDERR 'Saving NFO: ' . $xml . "\n";
  362. }
  363. if ($NO_NFO) {
  364. print STDERR "Not saving NFO\n";
  365. } else {
  366. saveString($nfo, $xml);
  367. }
  368. }
  369. }
  370. sub saveString($$) {
  371. my ($path, $str) = @_;
  372. my $fh = undef();
  373. if (!open($fh, '>', $path)) {
  374. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  375. return undef();
  376. }
  377. print $fh $str;
  378. close($fh);
  379. return 1;
  380. }
  381. sub buildSeriesNFO($) {
  382. my ($channel) = @_;
  383. # Create an XML tree
  384. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  385. $doc->setStandalone(1);
  386. my $show = $doc->createElement('tvshow');
  387. $doc->setDocumentElement($show);
  388. my $elm;
  389. # Add data
  390. $elm = $doc->createElement('title');
  391. $elm->appendText($channel->{'title'});
  392. $show->appendChild($elm);
  393. $elm = $doc->createElement('premiered');
  394. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  395. $show->appendChild($elm);
  396. $elm = $doc->createElement('plot');
  397. $elm->appendText($channel->{'description'});
  398. $show->appendChild($elm);
  399. # Return the string
  400. return $doc->toString();
  401. }
  402. sub buildNFO($) {
  403. my ($video) = @_;
  404. # Create an XML tree
  405. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  406. $doc->setStandalone(1);
  407. my $show = $doc->createElement('episodedetails');
  408. $doc->setDocumentElement($show);
  409. my $file = $doc->createElement('fileinfo');
  410. $show->appendChild($file);
  411. my $stream = $doc->createElement('streamdetails');
  412. $file->appendChild($stream);
  413. my $stream_video = $doc->createElement('video');
  414. $stream->appendChild($stream_video);
  415. my $elm;
  416. # Add data
  417. $elm = $doc->createElement('season');
  418. $elm->appendText($video->{'season'});
  419. $show->appendChild($elm);
  420. $elm = $doc->createElement('episode');
  421. $elm->appendText($video->{'number'});
  422. $show->appendChild($elm);
  423. if (!defined($video->{'title'})) {
  424. $video->{'title'} = 'Episode ' . $video->{'number'};
  425. }
  426. $elm = $doc->createElement('title');
  427. $elm->appendText($video->{'title'});
  428. $show->appendChild($elm);
  429. if (!defined($video->{'date'})) {
  430. $video->{'date'} = time();
  431. }
  432. $elm = $doc->createElement('aired');
  433. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  434. $show->appendChild($elm);
  435. if (defined($video->{'description'})) {
  436. $elm = $doc->createElement('plot');
  437. $elm->appendText($video->{'description'});
  438. $show->appendChild($elm);
  439. }
  440. if (defined($video->{'duration'})) {
  441. $elm = $doc->createElement('runtime');
  442. $elm->appendText($video->{'duration'});
  443. $show->appendChild($elm);
  444. $elm = $doc->createElement('durationinseconds');
  445. $elm->appendText($video->{'duration'});
  446. $stream_video->appendChild($elm);
  447. }
  448. if (defined($video->{'rating'})) {
  449. $elm = $doc->createElement('rating');
  450. $elm->appendText($video->{'rating'});
  451. $show->appendChild($elm);
  452. }
  453. if (defined($video->{'creator'})) {
  454. $elm = $doc->createElement('director');
  455. $elm->appendText($video->{'creator'});
  456. $show->appendChild($elm);
  457. }
  458. # Return the string
  459. return $doc->toString();
  460. }
  461. sub findFiles() {
  462. my %files = ();
  463. # Allow complete bypass
  464. if ($NO_FILES) {
  465. return \%files;
  466. }
  467. # Read the output directory
  468. my $fh = undef();
  469. opendir($fh, '.')
  470. or die('Unable to open files directory: ' . $! . "\n");
  471. while (my $file = readdir($fh)) {
  472. my ($season, $num, $id, $suffix) = parseFilename($file);
  473. if (defined($id) && length($id) > 0) {
  474. # Create the record as needed
  475. if (!exists($files{$id})) {
  476. my %tmp = (
  477. 'season' => $season,
  478. 'number' => $num,
  479. );
  480. $files{$id} = \%tmp;
  481. }
  482. # Video or NFO?
  483. my $type = 'path';
  484. if ($suffix eq 'nfo') {
  485. $type = 'nfo';
  486. }
  487. # Deal with duplicates
  488. if (exists($files{$id}->{$type})) {
  489. if ($type eq 'nfo') {
  490. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  491. if (!$NO_RENAME) {
  492. warn("\tDeleting: " . $file . "\n");
  493. if ($SUDO_CHATTR) {
  494. system('sudo', 'chattr', '-i', $file);
  495. }
  496. unlink($file);
  497. next;
  498. }
  499. } else {
  500. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  501. if (!$NO_RENAME) {
  502. my $del = $file;
  503. if ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  504. $del = $files{$id}->{'path'};
  505. }
  506. warn("\tDeleting: " . $del . "\n");
  507. if ($SUDO_CHATTR) {
  508. system('sudo', 'chattr', '-i', $del);
  509. }
  510. unlink($del);
  511. next;
  512. }
  513. }
  514. }
  515. # Assign the component paths
  516. $files{$id}->{$type} = $file;
  517. if ($type ne 'nfo') {
  518. $files{$id}->{'suffix'} = $suffix;
  519. }
  520. }
  521. }
  522. close($fh);
  523. if ($DEBUG) {
  524. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  525. if ($DEBUG > 1) {
  526. print STDERR prettyPrint(\%files, "\t") . "\n";
  527. }
  528. }
  529. return \%files;
  530. }
  531. sub fetchParse($$) {
  532. my ($name, $id) = @_;
  533. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  534. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  535. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  536. }
  537. # Fetch
  538. if ($DEBUG) {
  539. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  540. }
  541. sleep($DELAY);
  542. my $content = get($url);
  543. if (!defined($content) || length($content) < 10) {
  544. die('Invalid content from URL: ' . $url . "\n");
  545. }
  546. # Parse
  547. my $data = decode_json($content);
  548. if (!defined($data) || ref($data) ne 'HASH') {
  549. die('Invalid JSON: ' . $content . "\n");
  550. }
  551. if ($DEBUG > 2) {
  552. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  553. }
  554. return $data;
  555. }
  556. sub getSubscriptions($$) {
  557. my ($user, $id) = @_;
  558. my $index = 1;
  559. my $itemCount = undef();
  560. my %subs = ();
  561. SUBS_LOOP:
  562. {
  563. # Build, fetch, parse
  564. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  565. my $data = fetchParse('subscriptions', $id);
  566. # It's all in the feed
  567. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  568. die("Invalid subscription data\n");
  569. }
  570. $data = $data->{'feed'};
  571. # Grab the total count, so we know when to stop
  572. if (!defined($itemCount)) {
  573. if ( !exists($data->{'openSearch$totalResults'})
  574. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  575. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  576. {
  577. die("Invalid subscription feed metadata\n");
  578. }
  579. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  580. }
  581. # Process each item
  582. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  583. die("Invalid subscription entries\n");
  584. }
  585. my $items = $data->{'entry'};
  586. foreach my $item (@{$items}) {
  587. if ( ref($item) ne 'HASH'
  588. || !exists($item->{'yt$username'})
  589. || ref($item->{'yt$username'}) ne 'HASH'
  590. || !exists($item->{'yt$username'}->{'$t'}))
  591. {
  592. next;
  593. }
  594. if ($DEBUG) {
  595. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  596. }
  597. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  598. }
  599. # Loop if there are results left to fetch
  600. $index += $BATCH_SIZE;
  601. if (defined($itemCount) && $itemCount >= $index) {
  602. # But don't go past the max supported index
  603. if ($index <= $MAX_INDEX) {
  604. redo SUBS_LOOP;
  605. }
  606. }
  607. }
  608. # Return the list of subscribed usernames
  609. return \%subs;
  610. }
  611. sub saveSubscriptions($$) {
  612. my ($folder, $subs) = @_;
  613. # Check for local subscriptions missing from YT
  614. my %locals = ();
  615. my $fh = undef();
  616. opendir($fh, '.')
  617. or die('Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  618. while (my $file = readdir($fh)) {
  619. # Skip dotfiles
  620. if ($file =~ /^\./) {
  621. next;
  622. }
  623. # Skip non-directories
  624. if (!-d $file) {
  625. next;
  626. }
  627. # YT has some trouble with case
  628. my $lcFile = lc($file);
  629. # Anything else should be in the list
  630. if (!$subs->{$file} && !$subs->{$lcFile}) {
  631. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  632. }
  633. # Note local subscriptions
  634. $locals{$lcFile} = 1;
  635. }
  636. closedir($fh);
  637. # Check for YT subscriptions missing locally
  638. foreach my $sub (keys(%{$subs})) {
  639. if (!exists($locals{ lc($sub) })) {
  640. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  641. mkdir($folder . '/' . $sub);
  642. }
  643. }
  644. }
  645. sub saveChannel($) {
  646. my ($channel) = @_;
  647. my $nfo = 'tvshow.nfo';
  648. if (!-e $nfo) {
  649. if ($DEBUG) {
  650. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  651. }
  652. # Save the poster
  653. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  654. my $jpg = get($channel->{'thumbnail'});
  655. saveString('poster.jpg', $jpg);
  656. }
  657. # Save the series NFO
  658. my $xml = buildSeriesNFO($channel);
  659. if ($DEBUG > 1) {
  660. print STDERR 'Saving NFO: ' . $xml . "\n";
  661. }
  662. saveString($nfo, $xml);
  663. }
  664. }
  665. sub getChannel($) {
  666. my ($user) = @_;
  667. # Build, fetch, parse
  668. my $data = fetchParse('channel', $user);
  669. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  670. die("Invalid channel data\n");
  671. }
  672. $data = $data->{'entry'};
  673. # Extract the data we want
  674. my %channel = (
  675. 'id' => $data->{'yt$channelId'}->{'$t'},
  676. 'title' => $data->{'title'}->{'$t'},
  677. 'date' => str2time($data->{'published'}->{'$t'}),
  678. 'description' => $data->{'summary'}->{'$t'},
  679. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  680. );
  681. return \%channel;
  682. }
  683. sub readExcludes() {
  684. my %excludes = ();
  685. # Read and parse the excludes videos file, if it exists
  686. if (-e $EXCLUDES_FILE) {
  687. my $fh;
  688. open($fh, $EXCLUDES_FILE)
  689. or die('Unable to open excludes videos file: ' . $! . "\n");
  690. while (<$fh>) {
  691. # Skip blank lines and comments
  692. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  693. next;
  694. }
  695. # Match our specific format or whine
  696. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  697. if ($DEBUG > 1) {
  698. print STDERR 'Adding exclude video: ' . $1 . "\n";
  699. }
  700. $excludes{$1} = 1;
  701. } else {
  702. print STDERR 'Skipped exclude video line: ' . $_;
  703. }
  704. }
  705. close($fh);
  706. }
  707. return \%excludes;
  708. }
  709. sub readExtras() {
  710. my %extras = ();
  711. # Read and parse the extra videos file, if it exists
  712. if (-e $EXTRAS_FILE) {
  713. my $fh;
  714. open($fh, $EXTRAS_FILE)
  715. or die('Unable to open extra videos file: ' . $! . "\n");
  716. while (<$fh>) {
  717. # Skip blank lines and comments
  718. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  719. next;
  720. }
  721. # Match our specific format or whine
  722. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  723. if ($DEBUG > 1) {
  724. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  725. }
  726. $extras{$2} = $1;
  727. } else {
  728. print STDERR 'Skipped extra video line: ' . $_;
  729. }
  730. }
  731. close($fh);
  732. }
  733. return \%extras;
  734. }
  735. sub parseVideoData($) {
  736. my ($data) = @_;
  737. my %video = (
  738. 'title' => $data->{'title'},
  739. 'date' => str2time($data->{'uploaded'}),
  740. 'description' => $data->{'description'},
  741. 'duration' => $data->{'duration'},
  742. 'rating' => $data->{'rating'},
  743. 'creator' => $data->{'uploader'},
  744. );
  745. $video{'season'} = time2str('%Y', $video{'date'});
  746. return \%video;
  747. }
  748. sub findVideo($) {
  749. my ($id) = @_;
  750. # Build, fetch, parse
  751. my $data = fetchParse('video', $id);
  752. # Validate
  753. if (!exists($data->{'data'})) {
  754. die("Invalid video list\n");
  755. }
  756. $data = $data->{'data'};
  757. # Parse the video data
  758. return parseVideoData($data);
  759. }
  760. sub dropExcludes($) {
  761. my ($videos) = @_;
  762. my $excludes = readExcludes();
  763. foreach my $id (keys(%{$excludes})) {
  764. if (!exists($videos->{$id})) {
  765. if ($DEBUG > 1) {
  766. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  767. }
  768. next;
  769. }
  770. delete($videos->{$id});
  771. }
  772. return $excludes;
  773. }
  774. sub addExtras($) {
  775. my ($videos) = @_;
  776. my $extras = readExtras();
  777. foreach my $id (keys(%{$extras})) {
  778. if (exists($videos->{$id})) {
  779. if ($DEBUG > 1) {
  780. print STDERR 'Skipping known extra video: ' . $id . "\n";
  781. }
  782. next;
  783. }
  784. my $video = findVideo($id);
  785. $videos->{$id} = $video;
  786. }
  787. return $extras;
  788. }
  789. sub findVideos($) {
  790. my ($user) = @_;
  791. my %videos = ();
  792. # Allow complete bypass
  793. if ($NO_SEARCH) {
  794. return \%videos;
  795. }
  796. # Loop through until we have all the entries
  797. my $index = 1;
  798. my $itemCount = undef();
  799. LOOP:
  800. {
  801. # Build, fetch, parse
  802. $API{'search'}{'params'}{'start-index'} = $index;
  803. my $data = fetchParse('search', $user);
  804. # Grab the total count, so we know when to stop
  805. if (!exists($data->{'data'})) {
  806. die("Invalid video list\n");
  807. }
  808. $data = $data->{'data'};
  809. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  810. $itemCount = $data->{'totalItems'};
  811. }
  812. # Process each item
  813. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  814. die("Invalid video list\n");
  815. }
  816. my $items = $data->{'items'};
  817. my $offset = 0;
  818. foreach my $item (@{$items}) {
  819. my $video = parseVideoData($item);
  820. $videos{ $item->{'id'} } = $video;
  821. $offset++;
  822. }
  823. # Loop if there are results left to fetch
  824. $index += $BATCH_SIZE;
  825. if (defined($itemCount) && $itemCount >= $index) {
  826. # But don't go past the max supported index
  827. if ($index <= $MAX_INDEX) {
  828. redo LOOP;
  829. }
  830. }
  831. }
  832. if ($DEBUG) {
  833. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  834. if ($DEBUG > 1) {
  835. print STDERR prettyPrint(\%videos, "\t") . "\n";
  836. }
  837. }
  838. return \%videos;
  839. }
  840. sub updateNFOData($$$) {
  841. my ($file, $season, $episode) = @_;
  842. my $nfoData = undef();
  843. {
  844. open(my $fh, '<', $file)
  845. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  846. $/ = undef;
  847. $nfoData = <$fh>;
  848. close($fh);
  849. }
  850. if ($nfoData) {
  851. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  852. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  853. }
  854. return $nfoData;
  855. }
  856. sub videoSE($$) {
  857. my ($season, $episode) = @_;
  858. return sprintf('S%02dE%02d - ', $season, $episode);
  859. }
  860. sub videoPath($$$$) {
  861. my ($season, $episode, $id, $ext) = @_;
  862. return videoSE($season, $episode) . $id . '.' . $ext;
  863. }
  864. sub renameVideo($$$$$$) {
  865. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  866. # Bail if disabled
  867. if ($NO_RENAME) {
  868. print STDERR 'Not renaming: ' . $video . "\n";
  869. return;
  870. }
  871. # General sanity checks
  872. if (!defined($id) || !defined($season) || !defined($episode)) {
  873. die("Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  874. }
  875. # Warning about useless calls
  876. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  877. warn("No NFO or video path provided in renameVideo()\n");
  878. return;
  879. }
  880. # Video
  881. if (defined($video) && defined($suffix)) {
  882. my $videoNew = videoPath($season, $episode, $id, $suffix);
  883. # Sanity checks, as we do dangerous work here
  884. if (!-r $video) {
  885. die('Invalid video source in rename: ' . $video . "\n");
  886. }
  887. if ($video ne $videoNew) {
  888. print STDERR 'Renaming ' . $video . ' => ' . $videoNew . "\n";
  889. if (-e $videoNew) {
  890. die('Rename: Target exists: ' . $videoNew . "\n");
  891. }
  892. if ($SUDO_CHATTR) {
  893. system('sudo', 'chattr', '-i', $video);
  894. }
  895. rename($video, $videoNew)
  896. or die('Unable to rename: ' . $video . ': ' . $! . "\n");
  897. }
  898. }
  899. # NFO
  900. if (defined($nfo)) {
  901. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  902. # Sanity checks, as we do dangerous work here
  903. if (!-r $nfo) {
  904. die('Invalid NFO source in rename: ' . $nfo . "\n");
  905. }
  906. # Parse old NFO data
  907. my $nfoData = updateNFOData($nfo, $season, $episode)
  908. or die('No NFO data found, refusing to rename: ' . $id . "\n");
  909. # Write a new NFO and unlink the old one
  910. if ($SUDO_CHATTR) {
  911. system('sudo', 'chattr', '-i', $nfo);
  912. }
  913. saveString($nfoNew, $nfoData);
  914. if ($nfo ne $nfoNew) {
  915. print STDERR 'Renaming ' . $nfo . ' => ' . $nfoNew . "\n";
  916. unlink($nfo)
  917. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  918. }
  919. }
  920. }
  921. sub parseFilename($) {
  922. my ($file) = @_;
  923. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  924. }