subscribe.pl 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::Simple;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my @YTDL_DEBUG = ('--verbose');
  25. my $BATCH_SIZE = 50;
  26. my $MAX_INDEX = 25000;
  27. my $FETCH_LIMIT = 50;
  28. my $DELAY = 5;
  29. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  30. my %API = (
  31. 'search' => {
  32. 'prefix' => $API_URL . 'users/',
  33. 'suffix' => '/uploads',
  34. 'params' => {
  35. 'start-index' => 1,
  36. 'max-results' => 1,
  37. 'strict' => 1,
  38. 'v' => 2,
  39. 'alt' => 'jsonc',
  40. },
  41. },
  42. 'subscriptions' => {
  43. 'prefix' => $API_URL . 'users/',
  44. 'suffix' => '/subscriptions',
  45. 'params' => {
  46. 'start-index' => 1,
  47. 'max-results' => 1,
  48. 'strict' => 1,
  49. 'v' => 2,
  50. 'alt' => 'json',
  51. },
  52. },
  53. 'video' => {
  54. 'prefix' => $API_URL . 'videos/',
  55. 'suffix' => '',
  56. 'params' => {
  57. 'strict' => 1,
  58. 'v' => 2,
  59. 'alt' => 'jsonc'
  60. },
  61. },
  62. 'channel' => {
  63. 'prefix' => $API_URL . 'users/',
  64. 'suffix' => '',
  65. 'params' => {
  66. 'strict' => 1,
  67. 'v' => 2,
  68. 'alt' => 'json'
  69. },
  70. },
  71. );
  72. # Prototypes
  73. sub findVideos($);
  74. sub findFiles($);
  75. sub findVideo($);
  76. sub buildNFO($);
  77. sub buildSeriesNFO($);
  78. sub getSubscriptions($$);
  79. sub saveSubscriptions($$);
  80. sub saveChannel($);
  81. sub getChannel($);
  82. sub fetchParse($$);
  83. sub saveString($$);
  84. sub readExcludes();
  85. sub readExtras();
  86. sub parseVideoData($);
  87. sub dropExcludes($);
  88. sub addExtras($);
  89. sub updateNFOData($$$);
  90. sub videoNumberStr($$);
  91. sub videoSE($$);
  92. sub videoPath($$$$);
  93. sub renameVideo($$$$$$);
  94. sub parseFilename($);
  95. # Sanity check
  96. if (scalar(@ARGV) < 1) {
  97. die('Usage: ' . basename($0) . " output_directory\n");
  98. }
  99. # Command-line parameters
  100. my ($dir) = @ARGV;
  101. $dir =~ s/\/+$//;
  102. if (!-d $dir) {
  103. die('Invalid output directory: ' . $dir . "\n");
  104. }
  105. my $user = basename($dir);
  106. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  107. die('Invalid user: ' . $user . "\n");
  108. }
  109. # Move to the target directory so we can use relative paths later
  110. chdir($dir);
  111. # Environmental parameters (debug)
  112. my $DEBUG = 0;
  113. if ($ENV{'DEBUG'}) {
  114. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  115. $DEBUG = $1;
  116. } else {
  117. $DEBUG = 1;
  118. }
  119. }
  120. my $NO_FETCH = 0;
  121. if ($ENV{'NO_FETCH'}) {
  122. $NO_FETCH = 1;
  123. }
  124. my $NO_NFO = 0;
  125. if ($ENV{'NO_NFO'}) {
  126. $NO_NFO = 1;
  127. }
  128. my $NO_SEARCH = 0;
  129. if ($ENV{'NO_SEARCH'}) {
  130. $NO_SEARCH = 1;
  131. }
  132. my $NO_FILES = 0;
  133. if ($ENV{'NO_FILES'}) {
  134. $NO_FILES = 1;
  135. }
  136. my $NO_CHANNEL = 0;
  137. if ($ENV{'NO_CHANNEL'}) {
  138. $NO_CHANNEL = 1;
  139. }
  140. my $NO_EXTRAS = 0;
  141. if ($ENV{'NO_EXTRAS'}) {
  142. $NO_EXTRAS = 1;
  143. }
  144. my $NO_EXCLUDES = 0;
  145. if ($ENV{'NO_EXCLUDES'}) {
  146. $NO_EXCLUDES = 1;
  147. }
  148. my $NO_RENAME = 0;
  149. if ($ENV{'NO_RENAME'}) {
  150. $NO_RENAME = 1;
  151. }
  152. # Environmental parameters (functional)
  153. my $SUDO_CHATTR = 1;
  154. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  155. $SUDO_CHATTR = 0;
  156. }
  157. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  158. $MAX_INDEX = $1;
  159. }
  160. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  161. $BATCH_SIZE = $1;
  162. }
  163. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  164. $FETCH_LIMIT = $1;
  165. }
  166. # Construct globals
  167. foreach my $key (keys(%API)) {
  168. if (exists($API{$key}{'params'}{'max-results'})) {
  169. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  170. }
  171. }
  172. if ($DEBUG > 1) {
  173. push(@YTDL_ARGS, @YTDL_DEBUG);
  174. } elsif (!$DEBUG) {
  175. push(@YTDL_ARGS, @YTDL_QUIET);
  176. }
  177. # Allow use as a subscription manager
  178. if ($0 =~ /subscription/i) {
  179. my %subs = ();
  180. foreach my $user (keys(%USERS)) {
  181. my $tmp = getSubscriptions($user, $USERS{$user});
  182. foreach my $sub (keys(%{$tmp})) {
  183. if (exists($subs{$sub})) {
  184. $subs{$sub} .= ', ' . $tmp->{$sub};
  185. } else {
  186. $subs{$sub} = $tmp->{$sub};
  187. }
  188. }
  189. }
  190. saveSubscriptions($dir, \%subs);
  191. exit(0);
  192. }
  193. # Grab the channel data
  194. my $channel = {};
  195. if (!$NO_CHANNEL) {
  196. $channel = getChannel($user);
  197. saveChannel($channel);
  198. }
  199. # Find all the user's videos on YT
  200. my $videos = {};
  201. if (!$NO_SEARCH) {
  202. $videos = findVideos($user);
  203. }
  204. # Find any requested "extra" videos
  205. my $extras = {};
  206. if (!$NO_EXTRAS) {
  207. $extras = addExtras($videos);
  208. }
  209. # Drop any "excludes" videos
  210. my $excludes = {};
  211. if (!$NO_EXCLUDES) {
  212. $excludes = dropExcludes($videos);
  213. }
  214. # Calculate the episode number using the publish dates
  215. {
  216. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  217. my $num = undef();
  218. my $lastSeason = undef();
  219. foreach my $id (@byDate) {
  220. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  221. $num = 1;
  222. $lastSeason = $videos->{$id}->{'season'};
  223. }
  224. $videos->{$id}->{'number'} = $num++;
  225. }
  226. }
  227. # Find all existing YT files on disk
  228. my $files = {};
  229. if (!$NO_FILES) {
  230. $files = findFiles($videos);
  231. }
  232. # Whine about unknown videos
  233. foreach my $id (keys(%{$files})) {
  234. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  235. print STDERR 'Local video not known to YT channel (' . $user . '): ' . $id . "\n";
  236. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  237. }
  238. }
  239. # Fill in missing videos and NFOs
  240. my $fetched = 0;
  241. foreach my $id (keys(%{$videos})) {
  242. if ($DEBUG > 1) {
  243. print STDERR 'Checking remote video: ' . $id . "\n";
  244. if (!exists($files->{$id})) {
  245. print STDERR "\tLocal file: <missing>\n";
  246. } else {
  247. print STDERR "\tLocal media: " . $files->{$id}->{'path'} ."\n";
  248. print STDERR "\tLocal NFO: " . $files->{$id}->{'nfo'} ."\n";
  249. }
  250. }
  251. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  252. # Determine if we may or must rename
  253. my $rename = 0;
  254. if (exists($files->{$id})) {
  255. # Rename if we drift
  256. if (!$rename && $files->{$id}->{'path'}) {
  257. if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  258. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  259. {
  260. if ($DEBUG) {
  261. print STDERR "Rename due to drift\n";
  262. }
  263. $rename = 1;
  264. }
  265. }
  266. # Always rename if the NFO does not match the media
  267. if (!$rename && $files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
  268. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  269. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  270. if ($DEBUG) {
  271. print STDERR "Rename due to media-metadata mismatch\n";
  272. }
  273. $rename = 1;
  274. }
  275. }
  276. }
  277. # Warn (and optionally rename) as selected
  278. if ($rename) {
  279. print STDERR 'Video ' . $id . ' had number ' . $files->{$id}->{'number'} . ' but now has number ' . $videos->{$id}->{'number'} . "\n";
  280. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  281. }
  282. # If we haven't heard of the file, or don't have an NFO for it
  283. # Checking for the NFO allows use to resume failed downloads
  284. if (!exists($files->{$id}) || !-e $nfo) {
  285. if ($DEBUG) {
  286. print STDERR 'Fetching video: ' . $id . "\n";
  287. }
  288. # Let youtube-dl handle the URLs and downloading
  289. {
  290. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  291. my @name = ($YTDL_BIN);
  292. push(@name, @YTDL_ARGS);
  293. my @fetch = @name;
  294. push(@name, '--get-filename');
  295. push(@fetch, @args);
  296. push(@name, @args);
  297. if ($NO_FETCH) {
  298. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  299. } else {
  300. # Count fetch attempts (even if they fail later)
  301. $fetched++;
  302. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  303. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $user . "\n";
  304. if ($DEBUG) {
  305. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  306. }
  307. exit 1;
  308. }
  309. # Find the output file name
  310. if ($DEBUG > 1) {
  311. print STDERR join(' ', @name) . "\n";
  312. }
  313. sleep($DELAY);
  314. my $file = capture(EXIT_ANY, @name);
  315. if ($EXITVAL != 0) {
  316. warn('Error executing youtube-dl for name: ' . $id . ' (' . $EXITVAL . ")\n");
  317. next;
  318. }
  319. $file =~ s/^\s+//;
  320. $file =~ s/\s+$//;
  321. # Sanity check
  322. if (!$file) {
  323. warn('No file name available for video: ' . $id . "\n");
  324. next;
  325. }
  326. # Download
  327. if ($DEBUG > 1) {
  328. print STDERR join(' ', @fetch) . "\n";
  329. }
  330. sleep($DELAY);
  331. my $exit = run(EXIT_ANY, @fetch);
  332. if ($exit != 0) {
  333. warn('Error executing youtube-dl for video: ' . $file . "\n");
  334. next;
  335. }
  336. # Ensure we found something useful
  337. if (-e $file . '.part') {
  338. warn('Partial download detected: ' . $file . "\n");
  339. next;
  340. }
  341. if (!-s $file) {
  342. warn('No output video file: ' . $file . "\n");
  343. next;
  344. }
  345. # Touch the file to reflect the download time rather than the upload time
  346. touch($file);
  347. }
  348. }
  349. # Build and save the XML document
  350. my $xml = buildNFO($videos->{$id});
  351. if ($DEBUG > 1) {
  352. print STDERR 'Saving NFO: ' . $xml . "\n";
  353. }
  354. if ($NO_NFO) {
  355. print STDERR "Not saving NFO\n";
  356. } else {
  357. saveString($nfo, $xml);
  358. }
  359. }
  360. }
  361. sub saveString($$) {
  362. my ($path, $str) = @_;
  363. my $fh = undef();
  364. if (!open($fh, '>', $path)) {
  365. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  366. return undef();
  367. }
  368. print $fh $str;
  369. close($fh);
  370. return 1;
  371. }
  372. sub buildSeriesNFO($) {
  373. my ($channel) = @_;
  374. # Create an XML tree
  375. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  376. $doc->setStandalone(1);
  377. my $show = $doc->createElement('tvshow');
  378. $doc->setDocumentElement($show);
  379. my $elm;
  380. # Add data
  381. $elm = $doc->createElement('title');
  382. $elm->appendText($channel->{'title'});
  383. $show->appendChild($elm);
  384. $elm = $doc->createElement('premiered');
  385. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  386. $show->appendChild($elm);
  387. $elm = $doc->createElement('plot');
  388. $elm->appendText($channel->{'description'});
  389. $show->appendChild($elm);
  390. # Return the string
  391. return $doc->toString();
  392. }
  393. sub buildNFO($) {
  394. my ($video) = @_;
  395. # Create an XML tree
  396. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  397. $doc->setStandalone(1);
  398. my $show = $doc->createElement('episodedetails');
  399. $doc->setDocumentElement($show);
  400. my $file = $doc->createElement('fileinfo');
  401. $show->appendChild($file);
  402. my $stream = $doc->createElement('streamdetails');
  403. $file->appendChild($stream);
  404. my $stream_video = $doc->createElement('video');
  405. $stream->appendChild($stream_video);
  406. my $elm;
  407. # Add data
  408. $elm = $doc->createElement('season');
  409. $elm->appendText($video->{'season'});
  410. $show->appendChild($elm);
  411. $elm = $doc->createElement('episode');
  412. $elm->appendText(videoNumberStr($video->{'season'}, $video->{'number'}));
  413. $show->appendChild($elm);
  414. if (!defined($video->{'title'})) {
  415. $video->{'title'} = 'Episode ' . $video->{'number'};
  416. }
  417. $elm = $doc->createElement('title');
  418. $elm->appendText($video->{'title'});
  419. $show->appendChild($elm);
  420. if (!defined($video->{'date'})) {
  421. $video->{'date'} = time();
  422. }
  423. $elm = $doc->createElement('aired');
  424. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  425. $show->appendChild($elm);
  426. if (defined($video->{'description'})) {
  427. $elm = $doc->createElement('plot');
  428. $elm->appendText($video->{'description'});
  429. $show->appendChild($elm);
  430. }
  431. if (defined($video->{'duration'})) {
  432. $elm = $doc->createElement('runtime');
  433. $elm->appendText($video->{'duration'});
  434. $show->appendChild($elm);
  435. $elm = $doc->createElement('durationinseconds');
  436. $elm->appendText($video->{'duration'});
  437. $stream_video->appendChild($elm);
  438. }
  439. if (defined($video->{'rating'})) {
  440. $elm = $doc->createElement('rating');
  441. $elm->appendText($video->{'rating'});
  442. $show->appendChild($elm);
  443. }
  444. if (defined($video->{'creator'})) {
  445. $elm = $doc->createElement('director');
  446. $elm->appendText($video->{'creator'});
  447. $show->appendChild($elm);
  448. }
  449. # Return the string
  450. return $doc->toString();
  451. }
  452. sub findFiles($) {
  453. my ($videos) = @_;
  454. my %files = ();
  455. # Allow complete bypass
  456. if ($NO_FILES) {
  457. return \%files;
  458. }
  459. # Read the output directory
  460. my $fh = undef();
  461. opendir($fh, '.')
  462. or die('Unable to open files directory: ' . $! . "\n");
  463. while (my $file = readdir($fh)) {
  464. my ($season, $number, $id, $suffix) = parseFilename($file);
  465. if (defined($id) && length($id) > 0) {
  466. # Create the record as needed
  467. if (!exists($files{$id})) {
  468. my %tmp = (
  469. 'season' => $season,
  470. 'number' => $number,
  471. );
  472. $files{$id} = \%tmp;
  473. }
  474. # Video or NFO?
  475. my $type = 'path';
  476. if ($suffix eq 'nfo') {
  477. $type = 'nfo';
  478. }
  479. # Deal with duplicates
  480. if (exists($files{$id}->{$type})) {
  481. if ($type eq 'nfo') {
  482. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  483. if (!$NO_RENAME) {
  484. my $del = $file;
  485. if ( exists($videos->{$id})
  486. && $season == $videos->{$id}->{'season'}
  487. && $number == $videos->{$id}->{'number'})
  488. {
  489. $del = $files{$id}->{'nfo'};
  490. }
  491. warn("\tDeleting: " . $file . "\n");
  492. if ($SUDO_CHATTR) {
  493. system('sudo', 'chattr', '-i', $file);
  494. }
  495. unlink($file);
  496. next;
  497. }
  498. } else {
  499. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  500. if (!$NO_RENAME) {
  501. my $del = $file;
  502. if ( exists($videos->{$id})
  503. && $season == $videos->{$id}->{'season'}
  504. && $number == $videos->{$id}->{'number'})
  505. {
  506. $del = $files{$id}->{'path'};
  507. } elsif ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  508. $del = $files{$id}->{'path'};
  509. }
  510. warn("\tDeleting: " . $del . "\n");
  511. if ($SUDO_CHATTR) {
  512. system('sudo', 'chattr', '-i', $del);
  513. }
  514. unlink($del);
  515. next;
  516. }
  517. }
  518. }
  519. # Assign the component paths
  520. $files{$id}->{$type} = $file;
  521. if ($type ne 'nfo') {
  522. $files{$id}->{'suffix'} = $suffix;
  523. }
  524. }
  525. }
  526. close($fh);
  527. if ($DEBUG) {
  528. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  529. if ($DEBUG > 1) {
  530. print STDERR prettyPrint(\%files, "\t") . "\n";
  531. }
  532. }
  533. return \%files;
  534. }
  535. sub fetchParse($$) {
  536. my ($name, $id) = @_;
  537. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  538. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  539. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  540. }
  541. # Fetch
  542. if ($DEBUG) {
  543. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  544. }
  545. sleep($DELAY);
  546. my $content = get($url);
  547. if (!defined($content) || length($content) < 10) {
  548. die('Invalid content from URL: ' . $url . "\n");
  549. }
  550. # Parse
  551. my $data = decode_json($content);
  552. if (!defined($data) || ref($data) ne 'HASH') {
  553. die('Invalid JSON: ' . $content . "\n");
  554. }
  555. if ($DEBUG > 2) {
  556. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  557. }
  558. return $data;
  559. }
  560. sub getSubscriptions($$) {
  561. my ($user, $id) = @_;
  562. my $index = 1;
  563. my $itemCount = undef();
  564. my %subs = ();
  565. SUBS_LOOP:
  566. {
  567. # Build, fetch, parse
  568. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  569. my $data = fetchParse('subscriptions', $id);
  570. # It's all in the feed
  571. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  572. die("Invalid subscription data\n");
  573. }
  574. $data = $data->{'feed'};
  575. # Grab the total count, so we know when to stop
  576. if (!defined($itemCount)) {
  577. if ( !exists($data->{'openSearch$totalResults'})
  578. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  579. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  580. {
  581. die("Invalid subscription feed metadata\n");
  582. }
  583. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  584. }
  585. # Process each item
  586. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  587. die("Invalid subscription entries\n");
  588. }
  589. my $items = $data->{'entry'};
  590. foreach my $item (@{$items}) {
  591. if ( ref($item) ne 'HASH'
  592. || !exists($item->{'yt$username'})
  593. || ref($item->{'yt$username'}) ne 'HASH'
  594. || !exists($item->{'yt$username'}->{'$t'}))
  595. {
  596. next;
  597. }
  598. if ($DEBUG) {
  599. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  600. }
  601. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  602. }
  603. # Loop if there are results left to fetch
  604. $index += $BATCH_SIZE;
  605. if (defined($itemCount) && $itemCount >= $index) {
  606. # But don't go past the max supported index
  607. if ($index <= $MAX_INDEX) {
  608. redo SUBS_LOOP;
  609. }
  610. }
  611. }
  612. # Return the list of subscribed usernames
  613. return \%subs;
  614. }
  615. sub saveSubscriptions($$) {
  616. my ($folder, $subs) = @_;
  617. # Check for local subscriptions missing from YT
  618. my %locals = ();
  619. my $fh = undef();
  620. opendir($fh, '.')
  621. or die('Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  622. while (my $file = readdir($fh)) {
  623. # Skip dotfiles
  624. if ($file =~ /^\./) {
  625. next;
  626. }
  627. # Skip non-directories
  628. if (!-d $file) {
  629. next;
  630. }
  631. # YT has some trouble with case
  632. my $lcFile = lc($file);
  633. # Anything else should be in the list
  634. if (!$subs->{$file} && !$subs->{$lcFile}) {
  635. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  636. }
  637. # Note local subscriptions
  638. $locals{$lcFile} = 1;
  639. }
  640. closedir($fh);
  641. # Check for YT subscriptions missing locally
  642. foreach my $sub (keys(%{$subs})) {
  643. if (!exists($locals{ lc($sub) })) {
  644. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  645. mkdir($folder . '/' . $sub);
  646. }
  647. }
  648. }
  649. sub saveChannel($) {
  650. my ($channel) = @_;
  651. my $nfo = 'tvshow.nfo';
  652. if (!-e $nfo) {
  653. if ($DEBUG) {
  654. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  655. }
  656. # Save the poster
  657. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  658. my $jpg = get($channel->{'thumbnail'});
  659. saveString('poster.jpg', $jpg);
  660. }
  661. # Save the series NFO
  662. my $xml = buildSeriesNFO($channel);
  663. if ($DEBUG > 1) {
  664. print STDERR 'Saving NFO: ' . $xml . "\n";
  665. }
  666. saveString($nfo, $xml);
  667. }
  668. }
  669. sub getChannel($) {
  670. my ($user) = @_;
  671. # Build, fetch, parse
  672. my $data = fetchParse('channel', $user);
  673. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  674. die("Invalid channel data\n");
  675. }
  676. $data = $data->{'entry'};
  677. # Extract the data we want
  678. my %channel = (
  679. 'id' => $data->{'yt$channelId'}->{'$t'},
  680. 'title' => $data->{'title'}->{'$t'},
  681. 'date' => str2time($data->{'published'}->{'$t'}),
  682. 'description' => $data->{'summary'}->{'$t'},
  683. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  684. );
  685. return \%channel;
  686. }
  687. sub readExcludes() {
  688. my %excludes = ();
  689. # Read and parse the excludes videos file, if it exists
  690. if (-e $EXCLUDES_FILE) {
  691. my $fh;
  692. open($fh, $EXCLUDES_FILE)
  693. or die('Unable to open excludes videos file: ' . $! . "\n");
  694. while (<$fh>) {
  695. # Skip blank lines and comments
  696. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  697. next;
  698. }
  699. # Match our specific format or whine
  700. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  701. if ($DEBUG > 1) {
  702. print STDERR 'Adding exclude video: ' . $1 . "\n";
  703. }
  704. $excludes{$1} = 1;
  705. } else {
  706. print STDERR 'Skipped exclude video line: ' . $_;
  707. }
  708. }
  709. close($fh);
  710. }
  711. return \%excludes;
  712. }
  713. sub readExtras() {
  714. my %extras = ();
  715. # Read and parse the extra videos file, if it exists
  716. if (-e $EXTRAS_FILE) {
  717. my $fh;
  718. open($fh, $EXTRAS_FILE)
  719. or die('Unable to open extra videos file: ' . $! . "\n");
  720. while (<$fh>) {
  721. # Skip blank lines and comments
  722. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  723. next;
  724. }
  725. # Match our specific format or whine
  726. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  727. if ($DEBUG > 1) {
  728. print STDERR 'Adding extra video: ' . $1 . "\n";
  729. }
  730. $extras{$1} = 1;
  731. } else {
  732. print STDERR 'Skipped extra video line: ' . $_;
  733. }
  734. }
  735. close($fh);
  736. }
  737. return \%extras;
  738. }
  739. sub parseVideoData($) {
  740. my ($data) = @_;
  741. my %video = (
  742. 'title' => $data->{'title'},
  743. 'date' => str2time($data->{'uploaded'}),
  744. 'description' => $data->{'description'},
  745. 'duration' => $data->{'duration'},
  746. 'rating' => $data->{'rating'},
  747. 'creator' => $data->{'uploader'},
  748. );
  749. $video{'season'} = time2str('%Y', $video{'date'});
  750. return \%video;
  751. }
  752. sub findVideo($) {
  753. my ($id) = @_;
  754. # Build, fetch, parse
  755. my $data = fetchParse('video', $id);
  756. # Validate
  757. if (!exists($data->{'data'})) {
  758. die("Invalid video list\n");
  759. }
  760. $data = $data->{'data'};
  761. # Parse the video data
  762. return parseVideoData($data);
  763. }
  764. sub dropExcludes($) {
  765. my ($videos) = @_;
  766. my $excludes = readExcludes();
  767. foreach my $id (keys(%{$excludes})) {
  768. if (!exists($videos->{$id})) {
  769. if ($DEBUG > 1) {
  770. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  771. }
  772. next;
  773. }
  774. delete($videos->{$id});
  775. }
  776. return $excludes;
  777. }
  778. sub addExtras($) {
  779. my ($videos) = @_;
  780. my $extras = readExtras();
  781. foreach my $id (keys(%{$extras})) {
  782. if (exists($videos->{$id})) {
  783. if ($DEBUG > 1) {
  784. print STDERR 'Skipping known extra video: ' . $id . "\n";
  785. }
  786. next;
  787. }
  788. my $video = findVideo($id);
  789. $videos->{$id} = $video;
  790. }
  791. return $extras;
  792. }
  793. sub findVideos($) {
  794. my ($user) = @_;
  795. my %videos = ();
  796. # Allow complete bypass
  797. if ($NO_SEARCH) {
  798. return \%videos;
  799. }
  800. # Loop through until we have all the entries
  801. my $index = 1;
  802. my $itemCount = undef();
  803. LOOP:
  804. {
  805. # Build, fetch, parse
  806. $API{'search'}{'params'}{'start-index'} = $index;
  807. my $data = fetchParse('search', $user);
  808. # Grab the total count, so we know when to stop
  809. if (!exists($data->{'data'})) {
  810. die("Invalid video list\n");
  811. }
  812. $data = $data->{'data'};
  813. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  814. $itemCount = $data->{'totalItems'};
  815. }
  816. # Process each item
  817. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  818. die("Invalid video list\n");
  819. }
  820. my $items = $data->{'items'};
  821. my $offset = 0;
  822. foreach my $item (@{$items}) {
  823. my $video = parseVideoData($item);
  824. $videos{ $item->{'id'} } = $video;
  825. $offset++;
  826. }
  827. # Loop if there are results left to fetch
  828. $index += $BATCH_SIZE;
  829. if (defined($itemCount) && $itemCount >= $index) {
  830. # But don't go past the max supported index
  831. if ($index <= $MAX_INDEX) {
  832. redo LOOP;
  833. }
  834. }
  835. }
  836. if ($DEBUG) {
  837. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  838. if ($DEBUG > 1) {
  839. print STDERR prettyPrint(\%videos, "\t") . "\n";
  840. }
  841. }
  842. return \%videos;
  843. }
  844. sub updateNFOData($$$) {
  845. my ($file, $season, $episode) = @_;
  846. my $nfoData = undef();
  847. {
  848. open(my $fh, '<', $file)
  849. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  850. $/ = undef;
  851. $nfoData = <$fh>;
  852. close($fh);
  853. }
  854. if ($nfoData) {
  855. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  856. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  857. }
  858. return $nfoData;
  859. }
  860. sub videoNumberStr($$) {
  861. my ($season, $episode) = @_;
  862. my $str = '';
  863. if ($season == 0 && $episode =~ /^(20[012]\d)(\d{2,4})$/) {
  864. $str = sprintf('%04d%04d', $1, $2);
  865. } else {
  866. $str = sprintf('%02d', $episode);
  867. }
  868. return $str;
  869. }
  870. sub videoSE($$) {
  871. my ($season, $episode) = @_;
  872. return sprintf('S%0dE%s - ', $season, videoNumberStr($season, $episode));
  873. }
  874. sub videoPath($$$$) {
  875. my ($season, $episode, $id, $ext) = @_;
  876. return videoSE($season, $episode) . $id . '.' . $ext;
  877. }
  878. sub renameVideo($$$$$$) {
  879. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  880. # Bail if disabled
  881. if ($NO_RENAME) {
  882. print STDERR 'Not renaming: ' . $video . "\n";
  883. return;
  884. }
  885. # General sanity checks
  886. if (!defined($id) || !defined($season) || !defined($episode)) {
  887. die("Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  888. }
  889. # Warning about useless calls
  890. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  891. warn("No NFO or video path provided in renameVideo()\n");
  892. return;
  893. }
  894. # Video
  895. if (defined($video) && defined($suffix)) {
  896. my $videoNew = videoPath($season, $episode, $id, $suffix);
  897. # Sanity checks, as we do dangerous work here
  898. if (!-r $video) {
  899. die('Invalid video source in rename: ' . $video . "\n");
  900. }
  901. if ($video ne $videoNew) {
  902. print STDERR 'Renaming ' . $video . ' => ' . $videoNew . "\n";
  903. if (-e $videoNew) {
  904. die('Rename: Target exists: ' . $videoNew . "\n");
  905. }
  906. if ($SUDO_CHATTR) {
  907. system('sudo', 'chattr', '-i', $video);
  908. }
  909. rename($video, $videoNew)
  910. or die('Unable to rename: ' . $video . ': ' . $! . "\n");
  911. }
  912. }
  913. # NFO
  914. if (defined($nfo)) {
  915. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  916. # Sanity checks, as we do dangerous work here
  917. if (!-r $nfo) {
  918. die('Invalid NFO source in rename: ' . $nfo . "\n");
  919. }
  920. # Parse old NFO data
  921. my $nfoData = updateNFOData($nfo, $season, $episode)
  922. or die('No NFO data found, refusing to rename: ' . $id . "\n");
  923. # Write a new NFO and unlink the old one
  924. if ($SUDO_CHATTR) {
  925. system('sudo', 'chattr', '-i', $nfo);
  926. }
  927. saveString($nfoNew, $nfoData);
  928. if ($nfo ne $nfoNew) {
  929. print STDERR 'Renaming ' . $nfo . ' => ' . $nfoNew . "\n";
  930. unlink($nfo)
  931. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  932. }
  933. }
  934. }
  935. sub parseFilename($) {
  936. my ($file) = @_;
  937. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  938. }