subscribe.pl 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::Simple;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my @YTDL_DEBUG = ('--verbose');
  25. my $BATCH_SIZE = 50;
  26. my $MAX_INDEX = 25000;
  27. my $FETCH_LIMIT = 50;
  28. my $DRIFT_TOLERANCE = 2;
  29. my $DRIFT_FACTOR = 100.0;
  30. my $DELAY = 5;
  31. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  32. my %API = (
  33. 'search' => {
  34. 'prefix' => $API_URL . 'users/',
  35. 'suffix' => '/uploads',
  36. 'params' => {
  37. 'start-index' => 1,
  38. 'max-results' => 1,
  39. 'strict' => 1,
  40. 'v' => 2,
  41. 'alt' => 'jsonc',
  42. },
  43. },
  44. 'subscriptions' => {
  45. 'prefix' => $API_URL . 'users/',
  46. 'suffix' => '/subscriptions',
  47. 'params' => {
  48. 'start-index' => 1,
  49. 'max-results' => 1,
  50. 'strict' => 1,
  51. 'v' => 2,
  52. 'alt' => 'json',
  53. },
  54. },
  55. 'video' => {
  56. 'prefix' => $API_URL . 'videos/',
  57. 'suffix' => '',
  58. 'params' => {
  59. 'strict' => 1,
  60. 'v' => 2,
  61. 'alt' => 'jsonc'
  62. },
  63. },
  64. 'channel' => {
  65. 'prefix' => $API_URL . 'users/',
  66. 'suffix' => '',
  67. 'params' => {
  68. 'strict' => 1,
  69. 'v' => 2,
  70. 'alt' => 'json'
  71. },
  72. },
  73. );
  74. # Prototypes
  75. sub findVideos($);
  76. sub findFiles();
  77. sub findVideo($);
  78. sub buildNFO($);
  79. sub buildSeriesNFO($);
  80. sub getSubscriptions($$);
  81. sub saveSubscriptions($$);
  82. sub saveChannel($);
  83. sub getChannel($);
  84. sub fetchParse($$);
  85. sub saveString($$);
  86. sub readExcludes();
  87. sub readExtras();
  88. sub parseVideoData($);
  89. sub dropExcludes($);
  90. sub addExtras($);
  91. sub updateNFOData($$$);
  92. sub videoSE($$);
  93. sub videoPath($$$$);
  94. sub renameVideo($$$$$$);
  95. sub parseFilename($);
  96. # Sanity check
  97. if (scalar(@ARGV) < 1) {
  98. die('Usage: ' . basename($0) . " output_directory\n");
  99. }
  100. # Command-line parameters
  101. my ($dir) = @ARGV;
  102. $dir =~ s/\/+$//;
  103. if (!-d $dir) {
  104. die('Invalid output directory: ' . $dir . "\n");
  105. }
  106. my $user = basename($dir);
  107. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  108. die('Invalid user: ' . $user . "\n");
  109. }
  110. # Move to the target directory so we can use relative paths later
  111. chdir($dir);
  112. # Environmental parameters (debug)
  113. my $DEBUG = 0;
  114. if ($ENV{'DEBUG'}) {
  115. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  116. $DEBUG = $1;
  117. } else {
  118. $DEBUG = 1;
  119. }
  120. }
  121. my $NO_FETCH = 0;
  122. if ($ENV{'NO_FETCH'}) {
  123. $NO_FETCH = 1;
  124. }
  125. my $NO_NFO = 0;
  126. if ($ENV{'NO_NFO'}) {
  127. $NO_NFO = 1;
  128. }
  129. my $NO_SEARCH = 0;
  130. if ($ENV{'NO_SEARCH'}) {
  131. $NO_SEARCH = 1;
  132. }
  133. my $NO_FILES = 0;
  134. if ($ENV{'NO_FILES'}) {
  135. $NO_FILES = 1;
  136. }
  137. my $NO_CHANNEL = 0;
  138. if ($ENV{'NO_CHANNEL'}) {
  139. $NO_CHANNEL = 1;
  140. }
  141. my $NO_EXTRAS = 0;
  142. if ($ENV{'NO_EXTRAS'}) {
  143. $NO_EXTRAS = 1;
  144. }
  145. my $NO_EXCLUDES = 0;
  146. if ($ENV{'NO_EXCLUDES'}) {
  147. $NO_EXCLUDES = 1;
  148. }
  149. my $NO_RENAME = 0;
  150. if ($ENV{'NO_RENAME'}) {
  151. $NO_RENAME = 1;
  152. }
  153. # Environmental parameters (functional)
  154. my $FORCE_RENAME = 0;
  155. if ($ENV{'FORCE_RENAME'}) {
  156. $FORCE_RENAME = 1;
  157. }
  158. my $SUDO_CHATTR = 1;
  159. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  160. $SUDO_CHATTR = 0;
  161. }
  162. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  163. $MAX_INDEX = $1;
  164. }
  165. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  166. $BATCH_SIZE = $1;
  167. }
  168. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  169. $FETCH_LIMIT = $1;
  170. }
  171. # Construct globals
  172. foreach my $key (keys(%API)) {
  173. if (exists($API{$key}{'params'}{'max-results'})) {
  174. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  175. }
  176. }
  177. if ($DEBUG > 1) {
  178. push(@YTDL_ARGS, @YTDL_DEBUG);
  179. } elsif (!$DEBUG) {
  180. push(@YTDL_ARGS, @YTDL_QUIET);
  181. }
  182. # Allow use as a subscription manager
  183. if ($0 =~ /subscription/i) {
  184. my %subs = ();
  185. foreach my $user (keys(%USERS)) {
  186. my $tmp = getSubscriptions($user, $USERS{$user});
  187. foreach my $sub (keys(%{$tmp})) {
  188. if (exists($subs{$sub})) {
  189. $subs{$sub} .= ', ' . $tmp->{$sub};
  190. } else {
  191. $subs{$sub} = $tmp->{$sub};
  192. }
  193. }
  194. }
  195. saveSubscriptions($dir, \%subs);
  196. exit(0);
  197. }
  198. # Grab the channel data
  199. my $channel = {};
  200. if (!$NO_CHANNEL) {
  201. $channel = getChannel($user);
  202. saveChannel($channel);
  203. }
  204. # Find all the user's videos on YT
  205. my $videos = {};
  206. if (!$NO_SEARCH) {
  207. $videos = findVideos($user);
  208. }
  209. # Find any requested "extra" videos
  210. my $extras = {};
  211. if (!$NO_EXTRAS) {
  212. $extras = addExtras($videos);
  213. }
  214. # Drop any "excludes" videos
  215. my $excludes = {};
  216. if (!$NO_EXCLUDES) {
  217. $excludes = dropExcludes($videos);
  218. }
  219. # Calculate the episode number using the publish dates
  220. {
  221. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  222. my $num = undef();
  223. my $lastSeason = undef();
  224. foreach my $id (@byDate) {
  225. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  226. $num = 1;
  227. $lastSeason = $videos->{$id}->{'season'};
  228. }
  229. $videos->{$id}->{'number'} = $num++;
  230. }
  231. }
  232. # Find all existing YT files on disk
  233. my $files = {};
  234. if (!$NO_FILES) {
  235. $files = findFiles();
  236. }
  237. # Whine about unknown videos
  238. foreach my $id (keys(%{$files})) {
  239. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  240. print STDERR 'Local video not known to YT channel (' . $user . '): ' . $id . "\n";
  241. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  242. }
  243. }
  244. # Fill in missing videos and NFOs
  245. my $fetched = 0;
  246. foreach my $id (keys(%{$videos})) {
  247. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  248. # Determine if we may or must rename
  249. my $rename = 0;
  250. if (exists($files->{$id})) {
  251. # Renumber if we drift a lot
  252. if ($files->{$id}->{'path'}) {
  253. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  254. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  255. if ($FORCE_RENAME) {
  256. $tolerance = 0;
  257. } elsif ($tolerance < $DRIFT_TOLERANCE) {
  258. $tolerance = $DRIFT_TOLERANCE;
  259. }
  260. if ($delta > $tolerance) {
  261. if ($DEBUG) {
  262. print STDERR 'Rename due to drift (' . $delta . '/' . $tolerance . ")\n"
  263. }
  264. $rename = 1;
  265. }
  266. }
  267. # Always rename if the NFO does not match the media
  268. if ($files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
  269. my ($mseason, $mnumber) = parseFilename($files->{$id}->{'path'});
  270. my ($nseason, $nnumber) = parseFilename($files->{$id}->{'nfo'});
  271. if ($mseason != $nseason || $mnumber != $nnumber) {
  272. if ($DEBUG) {
  273. print STDERR "Rename due to media-metadata mismatch\n"
  274. }
  275. $rename = 1;
  276. }
  277. }
  278. }
  279. # Warn (and optionally rename) as selected
  280. if ($rename) {
  281. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  282. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  283. }
  284. # If we haven't heard of the file, or don't have an NFO for it
  285. # Checking for the NFO allows use to resume failed downloads
  286. if (!exists($files->{$id}) || !-e $nfo) {
  287. if ($DEBUG) {
  288. print STDERR 'Fetching video: ' . $id . "\n";
  289. }
  290. # Let youtube-dl handle the URLs and downloading
  291. {
  292. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  293. my @name = ($YTDL_BIN);
  294. push(@name, @YTDL_ARGS);
  295. my @fetch = @name;
  296. push(@name, '--get-filename');
  297. push(@fetch, @args);
  298. push(@name, @args);
  299. if ($NO_FETCH) {
  300. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  301. } else {
  302. # Count fetch attempts (even if they fail later)
  303. $fetched++;
  304. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  305. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $user . "\n";
  306. if ($DEBUG) {
  307. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  308. }
  309. exit 1;
  310. }
  311. # Find the output file name
  312. if ($DEBUG > 1) {
  313. print STDERR join(' ', @name) . "\n";
  314. }
  315. sleep($DELAY);
  316. my $file = capture(EXIT_ANY, @name);
  317. if ($EXITVAL != 0) {
  318. warn('Error executing youtube-dl (name): ' . $EXITVAL . "\n");
  319. next;
  320. }
  321. $file =~ s/^\s+//;
  322. $file =~ s/\s+$//;
  323. # Sanity check
  324. if (!$file) {
  325. warn('No file name available for video: ' . $id . "\n");
  326. next;
  327. }
  328. # Download
  329. if ($DEBUG > 1) {
  330. print STDERR join(' ', @fetch) . "\n";
  331. }
  332. sleep($DELAY);
  333. my $exit = run(EXIT_ANY, @fetch);
  334. if ($exit != 0) {
  335. warn('Error executing youtube-dl for video: ' . $file . "\n");
  336. next;
  337. }
  338. # Ensure we found something useful
  339. if (-e $file . '.part') {
  340. warn('Partial download detected: ' . $file . "\n");
  341. next;
  342. }
  343. if (!-s $file) {
  344. warn('No output video file: ' . $file . "\n");
  345. next;
  346. }
  347. # Touch the file to reflect the download time rather than the upload time
  348. touch($file);
  349. }
  350. }
  351. # Build and save the XML document
  352. my $xml = buildNFO($videos->{$id});
  353. if ($DEBUG > 1) {
  354. print STDERR 'Saving NFO: ' . $xml . "\n";
  355. }
  356. if ($NO_NFO) {
  357. print STDERR "Not saving NFO\n";
  358. } else {
  359. saveString($nfo, $xml);
  360. }
  361. }
  362. }
  363. sub saveString($$) {
  364. my ($path, $str) = @_;
  365. my $fh = undef();
  366. if (!open($fh, '>', $path)) {
  367. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  368. return undef();
  369. }
  370. print $fh $str;
  371. close($fh);
  372. return 1;
  373. }
  374. sub buildSeriesNFO($) {
  375. my ($channel) = @_;
  376. # Create an XML tree
  377. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  378. $doc->setStandalone(1);
  379. my $show = $doc->createElement('tvshow');
  380. $doc->setDocumentElement($show);
  381. my $elm;
  382. # Add data
  383. $elm = $doc->createElement('title');
  384. $elm->appendText($channel->{'title'});
  385. $show->appendChild($elm);
  386. $elm = $doc->createElement('premiered');
  387. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  388. $show->appendChild($elm);
  389. $elm = $doc->createElement('plot');
  390. $elm->appendText($channel->{'description'});
  391. $show->appendChild($elm);
  392. # Return the string
  393. return $doc->toString();
  394. }
  395. sub buildNFO($) {
  396. my ($video) = @_;
  397. # Create an XML tree
  398. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  399. $doc->setStandalone(1);
  400. my $show = $doc->createElement('episodedetails');
  401. $doc->setDocumentElement($show);
  402. my $file = $doc->createElement('fileinfo');
  403. $show->appendChild($file);
  404. my $stream = $doc->createElement('streamdetails');
  405. $file->appendChild($stream);
  406. my $stream_video = $doc->createElement('video');
  407. $stream->appendChild($stream_video);
  408. my $elm;
  409. # Add data
  410. $elm = $doc->createElement('season');
  411. $elm->appendText($video->{'season'});
  412. $show->appendChild($elm);
  413. $elm = $doc->createElement('episode');
  414. $elm->appendText($video->{'number'});
  415. $show->appendChild($elm);
  416. if (!defined($video->{'title'})) {
  417. $video->{'title'} = 'Episode ' . $video->{'number'};
  418. }
  419. $elm = $doc->createElement('title');
  420. $elm->appendText($video->{'title'});
  421. $show->appendChild($elm);
  422. if (!defined($video->{'date'})) {
  423. $video->{'date'} = time();
  424. }
  425. $elm = $doc->createElement('aired');
  426. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  427. $show->appendChild($elm);
  428. if (defined($video->{'description'})) {
  429. $elm = $doc->createElement('plot');
  430. $elm->appendText($video->{'description'});
  431. $show->appendChild($elm);
  432. }
  433. if (defined($video->{'duration'})) {
  434. $elm = $doc->createElement('runtime');
  435. $elm->appendText($video->{'duration'});
  436. $show->appendChild($elm);
  437. $elm = $doc->createElement('durationinseconds');
  438. $elm->appendText($video->{'duration'});
  439. $stream_video->appendChild($elm);
  440. }
  441. if (defined($video->{'rating'})) {
  442. $elm = $doc->createElement('rating');
  443. $elm->appendText($video->{'rating'});
  444. $show->appendChild($elm);
  445. }
  446. if (defined($video->{'creator'})) {
  447. $elm = $doc->createElement('director');
  448. $elm->appendText($video->{'creator'});
  449. $show->appendChild($elm);
  450. }
  451. # Return the string
  452. return $doc->toString();
  453. }
  454. sub findFiles() {
  455. my %files = ();
  456. # Allow complete bypass
  457. if ($NO_FILES) {
  458. return \%files;
  459. }
  460. # Read the output directory
  461. my $fh = undef();
  462. opendir($fh, '.')
  463. or die('Unable to open files directory: ' . $! . "\n");
  464. while (my $file = readdir($fh)) {
  465. my ($season, $num, $id, $suffix) = parseFilename($file);
  466. if (defined($id) && length($id) > 0) {
  467. # Create the record as needed
  468. if (!exists($files{$id})) {
  469. my %tmp = (
  470. 'season' => $season,
  471. 'number' => $num,
  472. );
  473. $files{$id} = \%tmp;
  474. }
  475. # Video or NFO?
  476. my $type = 'path';
  477. if ($suffix eq 'nfo') {
  478. $type = 'nfo';
  479. }
  480. # Deal with duplicates
  481. if (exists($files{$id}->{$type})) {
  482. if ($type eq 'nfo') {
  483. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  484. if (!$NO_RENAME) {
  485. warn("\tDeleting: " . $file . "\n");
  486. if ($SUDO_CHATTR) {
  487. system('sudo', 'chattr', '-i', $file);
  488. }
  489. unlink($file);
  490. next;
  491. }
  492. } else {
  493. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  494. if (!$NO_RENAME) {
  495. my $del = $file;
  496. if ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  497. $del = $files{$id}->{'path'};
  498. }
  499. warn("\tDeleting: " . $del . "\n");
  500. if ($SUDO_CHATTR) {
  501. system('sudo', 'chattr', '-i', $del);
  502. }
  503. unlink($del);
  504. next;
  505. }
  506. }
  507. }
  508. # Assign the component paths
  509. $files{$id}->{$type} = $file;
  510. if ($type ne 'nfo') {
  511. $files{$id}->{'suffix'} = $suffix;
  512. }
  513. }
  514. }
  515. close($fh);
  516. if ($DEBUG) {
  517. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  518. if ($DEBUG > 1) {
  519. print STDERR prettyPrint(\%files, "\t") . "\n";
  520. }
  521. }
  522. return \%files;
  523. }
  524. sub fetchParse($$) {
  525. my ($name, $id) = @_;
  526. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  527. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  528. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  529. }
  530. # Fetch
  531. if ($DEBUG) {
  532. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  533. }
  534. sleep($DELAY);
  535. my $content = get($url);
  536. if (!defined($content) || length($content) < 10) {
  537. die('Invalid content from URL: ' . $url . "\n");
  538. }
  539. # Parse
  540. my $data = decode_json($content);
  541. if (!defined($data) || ref($data) ne 'HASH') {
  542. die('Invalid JSON: ' . $content . "\n");
  543. }
  544. if ($DEBUG > 2) {
  545. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  546. }
  547. return $data;
  548. }
  549. sub getSubscriptions($$) {
  550. my ($user, $id) = @_;
  551. my $index = 1;
  552. my $itemCount = undef();
  553. my %subs = ();
  554. SUBS_LOOP:
  555. {
  556. # Build, fetch, parse
  557. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  558. my $data = fetchParse('subscriptions', $id);
  559. # It's all in the feed
  560. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  561. die("Invalid subscription data\n");
  562. }
  563. $data = $data->{'feed'};
  564. # Grab the total count, so we know when to stop
  565. if (!defined($itemCount)) {
  566. if ( !exists($data->{'openSearch$totalResults'})
  567. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  568. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  569. {
  570. die("Invalid subscription feed metadata\n");
  571. }
  572. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  573. }
  574. # Process each item
  575. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  576. die("Invalid subscription entries\n");
  577. }
  578. my $items = $data->{'entry'};
  579. foreach my $item (@{$items}) {
  580. if ( ref($item) ne 'HASH'
  581. || !exists($item->{'yt$username'})
  582. || ref($item->{'yt$username'}) ne 'HASH'
  583. || !exists($item->{'yt$username'}->{'$t'}))
  584. {
  585. next;
  586. }
  587. if ($DEBUG) {
  588. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  589. }
  590. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  591. }
  592. # Loop if there are results left to fetch
  593. $index += $BATCH_SIZE;
  594. if (defined($itemCount) && $itemCount >= $index) {
  595. # But don't go past the max supported index
  596. if ($index <= $MAX_INDEX) {
  597. redo SUBS_LOOP;
  598. }
  599. }
  600. }
  601. # Return the list of subscribed usernames
  602. return \%subs;
  603. }
  604. sub saveSubscriptions($$) {
  605. my ($folder, $subs) = @_;
  606. # Check for local subscriptions missing from YT
  607. my %locals = ();
  608. my $fh = undef();
  609. opendir($fh, '.')
  610. or die('Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  611. while (my $file = readdir($fh)) {
  612. # Skip dotfiles
  613. if ($file =~ /^\./) {
  614. next;
  615. }
  616. # Skip non-directories
  617. if (!-d $file) {
  618. next;
  619. }
  620. # YT has some trouble with case
  621. my $lcFile = lc($file);
  622. # Anything else should be in the list
  623. if (!$subs->{$file} && !$subs->{$lcFile}) {
  624. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  625. }
  626. # Note local subscriptions
  627. $locals{$lcFile} = 1;
  628. }
  629. closedir($fh);
  630. # Check for YT subscriptions missing locally
  631. foreach my $sub (keys(%{$subs})) {
  632. if (!exists($locals{ lc($sub) })) {
  633. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  634. mkdir($folder . '/' . $sub);
  635. }
  636. }
  637. }
  638. sub saveChannel($) {
  639. my ($channel) = @_;
  640. my $nfo = 'tvshow.nfo';
  641. if (!-e $nfo) {
  642. if ($DEBUG) {
  643. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  644. }
  645. # Save the poster
  646. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  647. my $jpg = get($channel->{'thumbnail'});
  648. saveString('poster.jpg', $jpg);
  649. }
  650. # Save the series NFO
  651. my $xml = buildSeriesNFO($channel);
  652. if ($DEBUG > 1) {
  653. print STDERR 'Saving NFO: ' . $xml . "\n";
  654. }
  655. saveString($nfo, $xml);
  656. }
  657. }
  658. sub getChannel($) {
  659. my ($user) = @_;
  660. # Build, fetch, parse
  661. my $data = fetchParse('channel', $user);
  662. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  663. die("Invalid channel data\n");
  664. }
  665. $data = $data->{'entry'};
  666. # Extract the data we want
  667. my %channel = (
  668. 'id' => $data->{'yt$channelId'}->{'$t'},
  669. 'title' => $data->{'title'}->{'$t'},
  670. 'date' => str2time($data->{'published'}->{'$t'}),
  671. 'description' => $data->{'summary'}->{'$t'},
  672. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  673. );
  674. return \%channel;
  675. }
  676. sub readExcludes() {
  677. my %excludes = ();
  678. # Read and parse the excludes videos file, if it exists
  679. if (-e $EXCLUDES_FILE) {
  680. my $fh;
  681. open($fh, $EXCLUDES_FILE)
  682. or die('Unable to open excludes videos file: ' . $! . "\n");
  683. while (<$fh>) {
  684. # Skip blank lines and comments
  685. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  686. next;
  687. }
  688. # Match our specific format or whine
  689. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  690. if ($DEBUG > 1) {
  691. print STDERR 'Adding exclude video: ' . $1 . "\n";
  692. }
  693. $excludes{$1} = 1;
  694. } else {
  695. print STDERR 'Skipped exclude video line: ' . $_;
  696. }
  697. }
  698. close($fh);
  699. }
  700. return \%excludes;
  701. }
  702. sub readExtras() {
  703. my %extras = ();
  704. # Read and parse the extra videos file, if it exists
  705. if (-e $EXTRAS_FILE) {
  706. my $fh;
  707. open($fh, $EXTRAS_FILE)
  708. or die('Unable to open extra videos file: ' . $! . "\n");
  709. while (<$fh>) {
  710. # Skip blank lines and comments
  711. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  712. next;
  713. }
  714. # Match our specific format or whine
  715. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  716. if ($DEBUG > 1) {
  717. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  718. }
  719. $extras{$2} = $1;
  720. } else {
  721. print STDERR 'Skipped extra video line: ' . $_;
  722. }
  723. }
  724. close($fh);
  725. }
  726. return \%extras;
  727. }
  728. sub parseVideoData($) {
  729. my ($data) = @_;
  730. my %video = (
  731. 'title' => $data->{'title'},
  732. 'date' => str2time($data->{'uploaded'}),
  733. 'description' => $data->{'description'},
  734. 'duration' => $data->{'duration'},
  735. 'rating' => $data->{'rating'},
  736. 'creator' => $data->{'uploader'},
  737. );
  738. $video{'season'} = time2str('%Y', $video{'date'});
  739. return \%video;
  740. }
  741. sub findVideo($) {
  742. my ($id) = @_;
  743. # Build, fetch, parse
  744. my $data = fetchParse('video', $id);
  745. # Validate
  746. if (!exists($data->{'data'})) {
  747. die("Invalid video list\n");
  748. }
  749. $data = $data->{'data'};
  750. # Parse the video data
  751. return parseVideoData($data);
  752. }
  753. sub dropExcludes($) {
  754. my ($videos) = @_;
  755. my $excludes = readExcludes();
  756. foreach my $id (keys(%{$excludes})) {
  757. if (!exists($videos->{$id})) {
  758. if ($DEBUG > 1) {
  759. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  760. }
  761. next;
  762. }
  763. delete($videos->{$id});
  764. }
  765. return $excludes;
  766. }
  767. sub addExtras($) {
  768. my ($videos) = @_;
  769. my $extras = readExtras();
  770. foreach my $id (keys(%{$extras})) {
  771. if (exists($videos->{$id})) {
  772. if ($DEBUG > 1) {
  773. print STDERR 'Skipping known extra video: ' . $id . "\n";
  774. }
  775. next;
  776. }
  777. my $video = findVideo($id);
  778. $videos->{$id} = $video;
  779. }
  780. return $extras;
  781. }
  782. sub findVideos($) {
  783. my ($user) = @_;
  784. my %videos = ();
  785. # Allow complete bypass
  786. if ($NO_SEARCH) {
  787. return \%videos;
  788. }
  789. # Loop through until we have all the entries
  790. my $index = 1;
  791. my $itemCount = undef();
  792. LOOP:
  793. {
  794. # Build, fetch, parse
  795. $API{'search'}{'params'}{'start-index'} = $index;
  796. my $data = fetchParse('search', $user);
  797. # Grab the total count, so we know when to stop
  798. if (!exists($data->{'data'})) {
  799. die("Invalid video list\n");
  800. }
  801. $data = $data->{'data'};
  802. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  803. $itemCount = $data->{'totalItems'};
  804. }
  805. # Process each item
  806. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  807. die("Invalid video list\n");
  808. }
  809. my $items = $data->{'items'};
  810. my $offset = 0;
  811. foreach my $item (@{$items}) {
  812. my $video = parseVideoData($item);
  813. $videos{ $item->{'id'} } = $video;
  814. $offset++;
  815. }
  816. # Loop if there are results left to fetch
  817. $index += $BATCH_SIZE;
  818. if (defined($itemCount) && $itemCount >= $index) {
  819. # But don't go past the max supported index
  820. if ($index <= $MAX_INDEX) {
  821. redo LOOP;
  822. }
  823. }
  824. }
  825. if ($DEBUG) {
  826. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  827. if ($DEBUG > 1) {
  828. print STDERR prettyPrint(\%videos, "\t") . "\n";
  829. }
  830. }
  831. return \%videos;
  832. }
  833. sub updateNFOData($$$) {
  834. my ($file, $season, $episode) = @_;
  835. my $nfoData = undef();
  836. {
  837. open(my $fh, '<', $file)
  838. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  839. $/ = undef;
  840. $nfoData = <$fh>;
  841. close($fh);
  842. }
  843. if ($nfoData) {
  844. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  845. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  846. }
  847. return $nfoData;
  848. }
  849. sub videoSE($$) {
  850. my ($season, $episode) = @_;
  851. return sprintf('S%02dE%02d - ', $season, $episode);
  852. }
  853. sub videoPath($$$$) {
  854. my ($season, $episode, $id, $ext) = @_;
  855. return videoSE($season, $episode) . $id . '.' . $ext;
  856. }
  857. sub renameVideo($$$$$$) {
  858. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  859. # Bail if disabled
  860. if ($NO_RENAME) {
  861. print STDERR 'Not renaming: ' . $video . "\n";
  862. return;
  863. }
  864. # General sanity checks
  865. if (!defined($id) || !defined($season) || !defined($episode)) {
  866. die("Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  867. }
  868. # Warning about useless calls
  869. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  870. warn("No NFO or video path provided in renameVideo()\n");
  871. return;
  872. }
  873. # Video
  874. if (defined($video) && defined($suffix)) {
  875. my $videoNew = videoPath($season, $episode, $id, $suffix);
  876. # Sanity checks, as we do dangerous work here
  877. if (!-r $video) {
  878. die('Invalid video source in rename: ' . $video . "\n");
  879. }
  880. if ($video ne $videoNew) {
  881. print STDERR 'Renaming ' . $video . ' => ' . $videoNew . "\n";
  882. if (-e $videoNew) {
  883. die('Rename: Target exists: ' . $videoNew . "\n");
  884. }
  885. if ($SUDO_CHATTR) {
  886. system('sudo', 'chattr', '-i', $video);
  887. }
  888. rename($video, $videoNew)
  889. or die('Unable to rename: ' . $video . ': ' . $! . "\n");
  890. }
  891. }
  892. # NFO
  893. if (defined($nfo)) {
  894. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  895. # Sanity checks, as we do dangerous work here
  896. if (!-r $nfo) {
  897. die('Invalid NFO source in rename: ' . $nfo . "\n");
  898. }
  899. # Parse old NFO data
  900. my $nfoData = updateNFOData($nfo, $season, $episode)
  901. or die('No NFO data found, refusing to rename: ' . $id . "\n");
  902. # Write a new NFO and unlink the old one
  903. if ($SUDO_CHATTR) {
  904. system('sudo', 'chattr', '-i', $nfo);
  905. }
  906. saveString($nfoNew, $nfoData);
  907. if ($nfo ne $nfoNew) {
  908. print STDERR 'Renaming ' . $nfo . ' => ' . $nfoNew . "\n";
  909. unlink($nfo)
  910. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  911. }
  912. }
  913. }
  914. sub parseFilename($) {
  915. my ($file) = @_;
  916. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  917. }