subscribe.pl 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Basename;
  7. use Date::Parse;
  8. use Date::Format;
  9. use LWP::Simple;
  10. use URI::Escape;
  11. use JSON;
  12. use XML::LibXML;
  13. use IPC::System::Simple qw( system );
  14. use WWW::YouTube::Download;
  15. use PrettyPrint;
  16. # Paramters
  17. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  18. my $EXTRAS_FILE = 'extra_videos.ini';
  19. my $EXCLUDES_FILE = 'exclude_videos.ini';
  20. my $CURL_BIN = 'curl';
  21. my @CURL_ARGS = ('-4', '--insecure', '-H', 'Accept-Encoding: gzip', '-C', '-', '--connect-timeout', '10', '--max-time', '1800');
  22. my $BATCH_SIZE = 50;
  23. my $MAX_INDEX = 25000;
  24. my $DRIFT_TOLERANCE = 2;
  25. my $DRIFT_FACTOR = 100.0;
  26. my $DELAY = 5;
  27. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  28. my %API = (
  29. 'search' => {
  30. 'prefix' => $API_URL . 'users/',
  31. 'suffix' => '/uploads',
  32. 'params' => {
  33. 'start-index' => 1,
  34. 'max-results' => 1,
  35. 'strict' => 1,
  36. 'v' => 2,
  37. 'alt' => 'jsonc',
  38. },
  39. },
  40. 'subscriptions' => {
  41. 'prefix' => $API_URL . 'users/',
  42. 'suffix' => '/subscriptions',
  43. 'params' => {
  44. 'start-index' => 1,
  45. 'max-results' => 1,
  46. 'strict' => 1,
  47. 'v' => 2,
  48. 'alt' => 'json',
  49. },
  50. },
  51. 'video' => {
  52. 'prefix' => $API_URL . 'videos/',
  53. 'suffix' => '',
  54. 'params' => {
  55. 'strict' => 1,
  56. 'v' => 2,
  57. 'alt' => 'jsonc'
  58. },
  59. },
  60. 'channel' => {
  61. 'prefix' => $API_URL . 'users/',
  62. 'suffix' => '',
  63. 'params' => {
  64. 'strict' => 1,
  65. 'v' => 2,
  66. 'alt' => 'json'
  67. },
  68. },
  69. );
  70. # Prototypes
  71. sub findVideos($);
  72. sub findFiles($);
  73. sub findVideo($);
  74. sub ytURL($);
  75. sub buildNFO($);
  76. sub buildSeriesNFO($);
  77. sub getSubscriptions($$);
  78. sub saveSubscriptions($$);
  79. sub saveChannel($$);
  80. sub getChannel($);
  81. sub fetchParse($$);
  82. sub saveString($$);
  83. sub readExcludes($);
  84. sub readExtras($);
  85. sub parseVideoData($);
  86. sub dropExcludes($$);
  87. sub addExtras($$);
  88. # Sanity check
  89. if (scalar(@ARGV) < 1) {
  90. die('Usage: ' . basename($0) . " output_directory\n");
  91. }
  92. # Command-line parameters
  93. my ($dir) = @ARGV;
  94. $dir =~ s/\/+$//;
  95. if (!-d $dir) {
  96. die('Invalid output directory: ' . $dir . "\n");
  97. }
  98. my $user = basename($dir);
  99. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  100. die('Invalid user: ' . $user . "\n");
  101. }
  102. # Environmental parameters (debug)
  103. my $DEBUG = 0;
  104. if ($ENV{'DEBUG'}) {
  105. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  106. $DEBUG = $1;
  107. } else {
  108. $DEBUG = 1;
  109. }
  110. }
  111. my $NO_FETCH = 0;
  112. if ($ENV{'NO_FETCH'}) {
  113. $NO_FETCH = 1;
  114. }
  115. my $NO_NFO = 0;
  116. if ($ENV{'NO_NFO'}) {
  117. $NO_NFO = 1;
  118. }
  119. my $NO_SEARCH = 0;
  120. if ($ENV{'NO_SEARCH'}) {
  121. $NO_SEARCH = 1;
  122. }
  123. my $NO_FILES = 0;
  124. if ($ENV{'NO_FILES'}) {
  125. $NO_FILES = 1;
  126. }
  127. my $NO_CHANNEL = 0;
  128. if ($ENV{'NO_CHANNEL'}) {
  129. $NO_CHANNEL = 1;
  130. }
  131. my $NO_EXTRAS = 0;
  132. if ($ENV{'NO_EXTRAS'}) {
  133. $NO_EXTRAS = 1;
  134. }
  135. my $NO_EXCLUDES = 0;
  136. if ($ENV{'NO_EXCLUDES'}) {
  137. $NO_EXCLUDES = 1;
  138. }
  139. # Environmental parameters (functional)
  140. my $RENAME = 0;
  141. if ($ENV{'RENAME'}) {
  142. $RENAME = 1;
  143. }
  144. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  145. $MAX_INDEX = $1;
  146. }
  147. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  148. $BATCH_SIZE = $1;
  149. }
  150. # Construct globals
  151. foreach my $key (keys(%API)) {
  152. if (exists($API{$key}{'params'}{'max-results'})) {
  153. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  154. }
  155. }
  156. if (!$DEBUG) {
  157. push(@CURL_ARGS, '--silent');
  158. }
  159. # Allow use as a subscription manager
  160. if ($0 =~ /subscription/i) {
  161. my %subs = ();
  162. foreach my $user (keys(%USERS)) {
  163. my $tmp = getSubscriptions($user, $USERS{$user});
  164. foreach my $sub (keys(%{$tmp})) {
  165. if (exists($subs{$sub})) {
  166. $subs{$sub} .= ', ' . $tmp->{$sub};
  167. } else {
  168. $subs{$sub} = $tmp->{$sub};
  169. }
  170. }
  171. }
  172. saveSubscriptions($dir, \%subs);
  173. exit(0);
  174. }
  175. # Grab the channel data
  176. my $channel = {};
  177. if (!$NO_CHANNEL) {
  178. $channel = getChannel($user);
  179. saveChannel($dir, $channel);
  180. }
  181. # Find all the user's videos on YT
  182. my $videos = {};
  183. if (!$NO_SEARCH) {
  184. $videos = findVideos($user);
  185. }
  186. # Find any requested "extra" videos
  187. my $extras = {};
  188. if (!$NO_EXTRAS) {
  189. $extras = addExtras($dir, $videos);
  190. }
  191. # Drop any "excludes" videos
  192. my $excludes = {};
  193. if (!$NO_EXCLUDES) {
  194. $excludes = dropExcludes($dir, $videos);
  195. }
  196. # Calculate the episode number using the publish dates
  197. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  198. my $num = 1;
  199. foreach my $id (@byDate) {
  200. $videos->{$id}->{'number'} = $num++;
  201. }
  202. # Find all existing YT files on disk
  203. my $files = {};
  204. if (!$NO_FILES) {
  205. $files = findFiles($dir);
  206. }
  207. # Whine about unknown videos
  208. foreach my $id (keys(%{$files})) {
  209. if (!exists($videos->{$id})) {
  210. print STDERR 'Local video not known to YT: ' . $id . "\n";
  211. }
  212. }
  213. # Fill in missing videos and NFOs
  214. foreach my $id (keys(%{$videos})) {
  215. my $basePath = $dir . '/S01E' . sprintf('%02d', $videos->{$id}->{'number'}) . ' - ' . $id . '.';
  216. my $nfo = $basePath . 'nfo';
  217. # Warn (and optionally rename) if the video numbers drift
  218. if (exists($files->{$id}) && $files->{$id}->{'number'} != $videos->{$id}->{'number'}) {
  219. if ($RENAME) {
  220. print STDERR 'Renaming ' . $files->{$id}->{'path'} . ' => ' . $basePath . $files->{$id}->{'suffix'} . "\n";
  221. rename($files->{$id}->{'path'}, $basePath . $files->{$id}->{'suffix'});
  222. rename($files->{$id}->{'nfo'}, $nfo);
  223. # This is a hack, but it fits nicely in one line
  224. system('sed', '-i', 's%<episode>[0-9]*</episode>%<episode>' . $videos->{$id}->{'number'} . '</episode>%', $nfo);
  225. } else {
  226. # Find the old NFO to avoid re-fetching
  227. $nfo = $files->{$id}->{'nfo'};
  228. # Ignore small changes
  229. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  230. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  231. if ($tolerance < $DRIFT_TOLERANCE) {
  232. $tolerance = $DRIFT_TOLERANCE;
  233. }
  234. if ($delta > $tolerance || $DEBUG) {
  235. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  236. }
  237. }
  238. }
  239. # If we haven't heard of the file, or don't have an NFO for it
  240. # Checking for the NFO allows use to resume failed downloads
  241. if (!exists($files->{$id}) || !-e $nfo) {
  242. if ($DEBUG) {
  243. print STDERR 'Fetching video: ' . $id . "\n";
  244. }
  245. # Find the download URL and file suffix
  246. my ($url, $suffix) = ytURL($id);
  247. if (!defined($url) || length($url) < 5) {
  248. warn('Could not determine URL for video: ' . $id . "\n");
  249. $url = undef();
  250. }
  251. # Fetch with cURL
  252. # I know LWP exists (and is even loaded) but cURL makes my life easier
  253. if ($url) {
  254. my @cmd = ($CURL_BIN);
  255. push(@cmd, @CURL_ARGS);
  256. push(@cmd, '-o', $basePath . $suffix, $url);
  257. if ($NO_FETCH) {
  258. print STDERR 'Not running: ' . join(' ', @cmd) . "\n";
  259. } else {
  260. if ($DEBUG > 1) {
  261. print STDERR join(' ', @cmd) . "\n";
  262. }
  263. sleep($DELAY);
  264. system(@cmd);
  265. }
  266. }
  267. # Build and save the XML document
  268. my $xml = buildNFO($videos->{$id});
  269. if ($DEBUG > 1) {
  270. print STDERR 'Saving NFO: ' . $xml . "\n";
  271. }
  272. if ($NO_NFO) {
  273. print STDERR "Not saving NFO\n";
  274. } else {
  275. saveString($nfo, $xml);
  276. }
  277. }
  278. }
  279. sub saveString($$) {
  280. my ($path, $str) = @_;
  281. my $fh = undef();
  282. if (!open($fh, '>', $path)) {
  283. warn('Cannot open file for writing: ' . $! . "\n");
  284. return undef();
  285. }
  286. print $fh $str;
  287. close($fh);
  288. return 1;
  289. }
  290. sub buildSeriesNFO($) {
  291. my ($channel) = @_;
  292. # Create an XML tree
  293. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  294. $doc->setStandalone(1);
  295. my $show = $doc->createElement('tvshow');
  296. $doc->setDocumentElement($show);
  297. my $elm;
  298. # Add data
  299. $elm = $doc->createElement('title');
  300. $elm->appendText($channel->{'title'});
  301. $show->appendChild($elm);
  302. $elm = $doc->createElement('premiered');
  303. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  304. $show->appendChild($elm);
  305. $elm = $doc->createElement('plot');
  306. $elm->appendText($channel->{'description'});
  307. $show->appendChild($elm);
  308. # Return the string
  309. return $doc->toString();
  310. }
  311. sub buildNFO($) {
  312. my ($video) = @_;
  313. # Create an XML tree
  314. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  315. $doc->setStandalone(1);
  316. my $show = $doc->createElement('episodedetails');
  317. $doc->setDocumentElement($show);
  318. my $elm;
  319. # Add data
  320. $elm = $doc->createElement('season');
  321. $elm->appendText('1');
  322. $show->appendChild($elm);
  323. $elm = $doc->createElement('episode');
  324. $elm->appendText($video->{'number'});
  325. $show->appendChild($elm);
  326. if (!defined($video->{'title'})) {
  327. $video->{'title'} = 'Episode ' . $video->{'number'};
  328. }
  329. $elm = $doc->createElement('title');
  330. $elm->appendText($video->{'title'});
  331. $show->appendChild($elm);
  332. if (!defined($video->{'date'})) {
  333. $video->{'date'} = time();
  334. }
  335. $elm = $doc->createElement('aired');
  336. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  337. $show->appendChild($elm);
  338. if (defined($video->{'description'})) {
  339. $elm = $doc->createElement('plot');
  340. $elm->appendText($video->{'description'});
  341. $show->appendChild($elm);
  342. }
  343. if (defined($video->{'duration'})) {
  344. $elm = $doc->createElement('runtime');
  345. $elm->appendText($video->{'duration'});
  346. $show->appendChild($elm);
  347. }
  348. if (defined($video->{'rating'})) {
  349. $elm = $doc->createElement('rating');
  350. $elm->appendText($video->{'rating'});
  351. $show->appendChild($elm);
  352. }
  353. if (defined($video->{'creator'})) {
  354. $elm = $doc->createElement('director');
  355. $elm->appendText($video->{'creator'});
  356. $show->appendChild($elm);
  357. }
  358. # Return the string
  359. return $doc->toString();
  360. }
  361. sub ytURL($) {
  362. my ($id) = @_;
  363. # Init the YT object
  364. my $tube = WWW::YouTube::Download->new;
  365. # Fetch metadata
  366. my $meta = eval { $tube->prepare_download($id); };
  367. if (!defined($meta) || ref($meta) ne 'HASH' || !exists($meta->{'video_url_map'}) || ref($meta->{'video_url_map'}) ne 'HASH') {
  368. if ($@ =~ /age gate/i) {
  369. print STDERR "Video not available due to age gate restrictions\n";
  370. }
  371. return undef();
  372. }
  373. # Find the best stream (i.e. highest resolution, prefer mp4)
  374. my $bestStream = undef();
  375. my $bestResolution = 0;
  376. foreach my $streamID (keys(%{ $meta->{'video_url_map'} })) {
  377. my $stream = $meta->{'video_url_map'}->{$streamID};
  378. my ($res) = $stream->{'resolution'} =~ /^\s*(\d+)/;
  379. if ($DEBUG > 1) {
  380. print STDERR $streamID . ' (' . $stream->{'suffix'} . ')' . ' => ' . $stream->{'resolution'} . ' : ' . $stream->{'url'} . "\n";
  381. }
  382. if ( ($res > $bestResolution)
  383. || ($res == $bestResolution && defined($bestStream) && $bestStream->{'suffix'} ne 'mp4'))
  384. {
  385. $bestStream = $stream;
  386. $bestResolution = $res;
  387. }
  388. }
  389. if (!exists($bestStream->{'suffix'}) || length($bestStream->{'suffix'}) < 2) {
  390. $bestStream->{'suffix'} = 'mp4';
  391. }
  392. return ($bestStream->{'url'}, $bestStream->{'suffix'});
  393. }
  394. sub findFiles($) {
  395. my ($dir) = @_;
  396. my %files = ();
  397. # Allow complete bypass
  398. if ($NO_FILES) {
  399. return \%files;
  400. }
  401. # Read the output directory
  402. my $fh = undef();
  403. opendir($fh, $dir)
  404. or die('Unable to open output directory: ' . $! . "\n");
  405. while (my $file = readdir($fh)) {
  406. my ($num, $id, $suffix) = $file =~ /^S01+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  407. if (defined($id) && length($id) > 0) {
  408. if ($suffix eq 'nfo') {
  409. next;
  410. }
  411. my %tmp = (
  412. 'number' => $num,
  413. 'suffix' => $suffix,
  414. 'path' => $dir . '/' . $file,
  415. );
  416. $tmp{'nfo'} = $tmp{'path'};
  417. $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/;
  418. if (exists($files{$id})) {
  419. warn('Duplicate ID: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $tmp{'path'} . "\n");
  420. if ($RENAME) {
  421. # Prefer to delete MP4 files, if the suffixes differ
  422. my $del = $files{$id};
  423. if ($tmp{'suffix'} eq 'mp4' && $del->{'suffix'} ne 'mp4') {
  424. $del = \%tmp;
  425. %tmp = %{ $files{$id} };
  426. }
  427. print STDERR 'Deleting duplicate: ' . $del->{'path'} . "\n";
  428. unlink($del->{'path'});
  429. unlink($del->{'nfo'});
  430. }
  431. }
  432. $files{$id} = \%tmp;
  433. }
  434. }
  435. close($fh);
  436. if ($DEBUG) {
  437. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  438. if ($DEBUG > 1) {
  439. print STDERR prettyPrint(\%files, "\t") . "\n";
  440. }
  441. }
  442. return \%files;
  443. }
  444. sub fetchParse($$) {
  445. my ($name, $id) = @_;
  446. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  447. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  448. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  449. }
  450. # Fetch
  451. if ($DEBUG) {
  452. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  453. }
  454. sleep($DELAY);
  455. my $content = get($url);
  456. if (!defined($content) || length($content) < 10) {
  457. die('Invalid content from URL: ' . $url . "\n");
  458. }
  459. # Parse
  460. my $data = decode_json($content);
  461. if (!defined($data) || ref($data) ne 'HASH') {
  462. die('Invalid JSON: ' . $content . "\n");
  463. }
  464. if ($DEBUG > 2) {
  465. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  466. }
  467. return $data;
  468. }
  469. sub getSubscriptions($$) {
  470. my ($user, $id) = @_;
  471. my $index = 1;
  472. my $itemCount = undef();
  473. my %subs = ();
  474. SUBS_LOOP:
  475. {
  476. # Build, fetch, parse
  477. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  478. my $data = fetchParse('subscriptions', $id);
  479. # It's all in the feed
  480. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  481. die("Invalid subscription data\n");
  482. }
  483. $data = $data->{'feed'};
  484. # Grab the total count, so we know when to stop
  485. if (!defined($itemCount)) {
  486. if ( !exists($data->{'openSearch$totalResults'})
  487. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  488. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  489. {
  490. die("Invalid subscription feed metadata\n");
  491. }
  492. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  493. }
  494. # Process each item
  495. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  496. die("Invalid subscription entries\n");
  497. }
  498. my $items = $data->{'entry'};
  499. foreach my $item (@{$items}) {
  500. if ( ref($item) ne 'HASH'
  501. || !exists($item->{'yt$username'})
  502. || ref($item->{'yt$username'}) ne 'HASH'
  503. || !exists($item->{'yt$username'}->{'$t'}))
  504. {
  505. next;
  506. }
  507. if ($DEBUG) {
  508. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  509. }
  510. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  511. }
  512. # Loop if there are results left to fetch
  513. $index += $BATCH_SIZE;
  514. if (defined($itemCount) && $itemCount >= $index) {
  515. # But don't go past the max supported index
  516. if ($index <= $MAX_INDEX) {
  517. redo SUBS_LOOP;
  518. }
  519. }
  520. }
  521. # Return the list of subscribed usernames
  522. return \%subs;
  523. }
  524. sub saveSubscriptions($$) {
  525. my ($folder, $subs) = @_;
  526. # Check for local subscriptions missing from YT
  527. my %locals = ();
  528. my $fh = undef();
  529. opendir($fh, $folder)
  530. or die('Unable to open subscriptions directory: ' . $! . "\n");
  531. while (my $file = readdir($fh)) {
  532. # Skip dotfiles
  533. if ($file =~ /^\./) {
  534. next;
  535. }
  536. # Skip non-directories
  537. if (!-d $folder . '/' . $file) {
  538. next;
  539. }
  540. # YT has case issues
  541. $file = lc($file);
  542. # Anything else should be in the list
  543. if (!$subs->{$file}) {
  544. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  545. }
  546. # Note local subscriptions
  547. $locals{$file} = 1;
  548. }
  549. closedir($fh);
  550. # Check for YT subscriptions missing locally
  551. foreach my $sub (keys(%{$subs})) {
  552. if (!exists($locals{$sub})) {
  553. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  554. mkdir($folder . '/' . $sub);
  555. }
  556. }
  557. }
  558. sub saveChannel($$) {
  559. my ($dir, $channel) = @_;
  560. my $nfo = $dir . '/tvshow.nfo';
  561. if (!-e $nfo) {
  562. if ($DEBUG) {
  563. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  564. }
  565. # Save the poster
  566. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  567. my $poster = $dir . '/poster.jpg';
  568. my $jpg = get($channel->{'thumbnail'});
  569. saveString($poster, $jpg);
  570. }
  571. # Save the series NFO
  572. my $xml = buildSeriesNFO($channel);
  573. if ($DEBUG > 1) {
  574. print STDERR 'Saving NFO: ' . $xml . "\n";
  575. }
  576. saveString($nfo, $xml);
  577. }
  578. }
  579. sub getChannel($) {
  580. my ($user) = @_;
  581. # Build, fetch, parse
  582. my $data = fetchParse('channel', $user);
  583. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  584. die("Invalid channel data\n");
  585. }
  586. $data = $data->{'entry'};
  587. # Extract the data we want
  588. my %channel = (
  589. 'id' => $data->{'yt$channelId'}->{'$t'},
  590. 'title' => $data->{'title'}->{'$t'},
  591. 'date' => str2time($data->{'published'}->{'$t'}),
  592. 'description' => $data->{'summary'}->{'$t'},
  593. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  594. );
  595. return \%channel;
  596. }
  597. sub readExcludes($) {
  598. my ($dir) = @_;
  599. my %excludes = ();
  600. # Read and parse the excludes videos file, if it exists
  601. my $file = $dir . '/' . $EXCLUDES_FILE;
  602. if (-e $file) {
  603. my $fh;
  604. open($fh, $file)
  605. or die('Unable to open excludes videos file: ' . $! . "\n");
  606. while (<$fh>) {
  607. # Skip blank lines and comments
  608. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  609. next;
  610. }
  611. # Match our specific format or whine
  612. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  613. if ($DEBUG > 1) {
  614. print STDERR 'Adding exclude video: ' . $1 . "\n";
  615. }
  616. $excludes{$1} = 1;
  617. } else {
  618. print STDERR 'Skipped exclude video line: ' . $_;
  619. }
  620. }
  621. close($fh);
  622. }
  623. return \%excludes;
  624. }
  625. sub readExtras($) {
  626. my ($dir) = @_;
  627. my %extras = ();
  628. # Read and parse the extra videos file, if it exists
  629. my $file = $dir . '/' . $EXTRAS_FILE;
  630. if (-e $file) {
  631. my $fh;
  632. open($fh, $file)
  633. or die('Unable to open extra videos file: ' . $! . "\n");
  634. while (<$fh>) {
  635. # Skip blank lines and comments
  636. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  637. next;
  638. }
  639. # Match our specific format or whine
  640. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  641. if ($DEBUG > 1) {
  642. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  643. }
  644. $extras{$2} = $1;
  645. } else {
  646. print STDERR 'Skipped extra video line: ' . $_;
  647. }
  648. }
  649. close($fh);
  650. }
  651. return \%extras;
  652. }
  653. sub parseVideoData($) {
  654. my ($data) = @_;
  655. my %video = (
  656. 'title' => $data->{'title'},
  657. 'date' => str2time($data->{'uploaded'}),
  658. 'description' => $data->{'description'},
  659. 'duration' => $data->{'duration'},
  660. 'rating' => $data->{'rating'},
  661. 'creator' => $data->{'uploader'},
  662. );
  663. return \%video;
  664. }
  665. sub findVideo($) {
  666. my ($id) = @_;
  667. # Build, fetch, parse
  668. my $data = fetchParse('video', $id);
  669. # Validate
  670. if (!exists($data->{'data'})) {
  671. die("Invalid video list\n");
  672. }
  673. $data = $data->{'data'};
  674. # Parse the video data
  675. return parseVideoData($data);
  676. }
  677. sub dropExcludes($$) {
  678. my ($dir, $videos) = @_;
  679. my $excludes = readExcludes($dir);
  680. foreach my $id (keys(%{$excludes})) {
  681. if (!exists($videos->{$id})) {
  682. if ($DEBUG > 1) {
  683. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  684. }
  685. next;
  686. }
  687. delete($videos->{$id});
  688. }
  689. return $excludes;
  690. }
  691. sub addExtras($$) {
  692. my ($dir, $videos) = @_;
  693. my $extras = readExtras($dir);
  694. foreach my $id (keys(%{$extras})) {
  695. if (exists($videos->{$id})) {
  696. if ($DEBUG > 1) {
  697. print STDERR 'Skipping known extra video: ' . $id . "\n";
  698. }
  699. next;
  700. }
  701. my $video = findVideo($id);
  702. $video->{'api_number'} = $extras->{$id};
  703. $videos->{$id} = $video;
  704. }
  705. return $extras;
  706. }
  707. sub findVideos($) {
  708. my ($user) = @_;
  709. my %videos = ();
  710. # Allow complete bypass
  711. if ($NO_SEARCH) {
  712. return \%videos;
  713. }
  714. # Loop through until we have all the entries
  715. my $index = 1;
  716. my $itemCount = undef();
  717. LOOP:
  718. {
  719. # Build, fetch, parse
  720. $API{'search'}{'params'}{'start-index'} = $index;
  721. my $data = fetchParse('search', $user);
  722. # Grab the total count, so we know when to stop
  723. if (!exists($data->{'data'})) {
  724. die("Invalid video list\n");
  725. }
  726. $data = $data->{'data'};
  727. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  728. $itemCount = $data->{'totalItems'};
  729. }
  730. # Process each item
  731. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  732. die("Invalid video list\n");
  733. }
  734. my $items = $data->{'items'};
  735. my $offset = 0;
  736. foreach my $item (@{$items}) {
  737. my $video = parseVideoData($item);
  738. $video->{'api_number'} = $itemCount - $index - $offset + 1;
  739. $videos{ $item->{'id'} } = $video;
  740. $offset++;
  741. }
  742. # Loop if there are results left to fetch
  743. $index += $BATCH_SIZE;
  744. if (defined($itemCount) && $itemCount >= $index) {
  745. # But don't go past the max supported index
  746. if ($index <= $MAX_INDEX) {
  747. redo LOOP;
  748. }
  749. }
  750. }
  751. if ($DEBUG) {
  752. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  753. if ($DEBUG > 1) {
  754. print STDERR prettyPrint(\%videos, "\t") . "\n";
  755. }
  756. }
  757. return \%videos;
  758. }