subscribe.pl 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Basename;
  7. use Date::Parse;
  8. use Date::Format;
  9. use LWP::Simple;
  10. use URI::Escape;
  11. use JSON;
  12. use XML::LibXML;
  13. use IPC::System::Simple qw( system );
  14. use WWW::YouTube::Download;
  15. use PrettyPrint;
  16. # Paramters
  17. my $EXTRAS_FILE = 'extra_videos.ini';
  18. my $EXCLUDES_FILE = 'exclude_videos.ini';
  19. my $CURL_BIN = 'curl';
  20. my @CURL_ARGS = ('-4', '--insecure', '-C', '-', '--connect-timeout', '10', '--max-time', '1800');
  21. my $BATCH_SIZE = 50;
  22. my $MAX_INDEX = 5000;
  23. my $DRIFT_TOLERANCE = 2;
  24. my $DRIFT_FACTOR = 100.0;
  25. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  26. my %API = (
  27. 'search' => {
  28. 'prefix' => $API_URL . 'users/',
  29. 'suffix' => '/uploads',
  30. 'params' => {
  31. 'start-index' => 1,
  32. 'max-results' => 1,
  33. 'strict' => 1,
  34. 'v' => 2,
  35. 'alt' => 'jsonc',
  36. },
  37. },
  38. 'video' => {
  39. 'prefix' => $API_URL . 'videos/',
  40. 'suffix' => '',
  41. 'params' => {
  42. 'strict' => 1,
  43. 'v' => 2,
  44. 'alt' => 'jsonc'
  45. },
  46. },
  47. 'channel' => {
  48. 'prefix' => $API_URL . 'users/',
  49. 'suffix' => '',
  50. 'params' => {
  51. 'strict' => 1,
  52. 'v' => 2,
  53. 'alt' => 'json'
  54. },
  55. },
  56. );
  57. # Prototypes
  58. sub findVideos($);
  59. sub findFiles($);
  60. sub findVideo($);
  61. sub ytURL($);
  62. sub buildNFO($);
  63. sub buildSeriesNFO($);
  64. sub getChannel($);
  65. sub fetchParse($$);
  66. sub saveString($$);
  67. sub readExcludes($);
  68. sub readExtras($);
  69. sub parseVideoData($);
  70. sub dropExcludes($$);
  71. sub addExtras($$);
  72. sub saveChannel($$);
  73. # Sanity check
  74. if (scalar(@ARGV) < 1) {
  75. die('Usage: ' . basename($0) . " output_directory\n");
  76. }
  77. # Command-line parameters
  78. my ($dir) = @ARGV;
  79. $dir =~ s/\/+$//;
  80. if (!-d $dir) {
  81. die('Invalid output directory: ' . $dir . "\n");
  82. }
  83. my $user = basename($dir);
  84. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  85. die('Invalid user: ' . $user . "\n");
  86. }
  87. # Environmental parameters (debug)
  88. my $DEBUG = 0;
  89. if ($ENV{'DEBUG'}) {
  90. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  91. $DEBUG = $1;
  92. } else {
  93. $DEBUG = 1;
  94. }
  95. }
  96. my $NO_FETCH = 0;
  97. if ($ENV{'NO_FETCH'}) {
  98. $NO_FETCH = 1;
  99. }
  100. my $NO_NFO = 0;
  101. if ($ENV{'NO_NFO'}) {
  102. $NO_NFO = 1;
  103. }
  104. my $NO_SEARCH = 0;
  105. if ($ENV{'NO_SEARCH'}) {
  106. $NO_SEARCH = 1;
  107. }
  108. my $NO_FILES = 0;
  109. if ($ENV{'NO_FILES'}) {
  110. $NO_FILES = 1;
  111. }
  112. my $NO_CHANNEL = 0;
  113. if ($ENV{'NO_CHANNEL'}) {
  114. $NO_CHANNEL = 1;
  115. }
  116. my $NO_EXTRAS = 0;
  117. if ($ENV{'NO_EXTRAS'}) {
  118. $NO_EXTRAS = 1;
  119. }
  120. my $NO_EXCLUDES = 0;
  121. if ($ENV{'NO_EXCLUDES'}) {
  122. $NO_EXCLUDES = 1;
  123. }
  124. # Environmental parameters (functional)
  125. my $RENAME = 0;
  126. if ($ENV{'RENAME'}) {
  127. $RENAME = 1;
  128. }
  129. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  130. $MAX_INDEX = $1;
  131. }
  132. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  133. $BATCH_SIZE = $1;
  134. }
  135. # Construct globals
  136. $API{'search'}{'params'}{'max-results'} = $BATCH_SIZE;
  137. if (!$DEBUG) {
  138. push(@CURL_ARGS, '--silent');
  139. }
  140. # Grab the channel data
  141. my $channel = {};
  142. if (!$NO_CHANNEL) {
  143. $channel = getChannel($user);
  144. saveChannel($dir, $channel);
  145. }
  146. # Find all the user's videos on YT
  147. my $videos = {};
  148. if (!$NO_SEARCH) {
  149. $videos = findVideos($user);
  150. }
  151. # Find any requested "extra" videos
  152. my $extras = {};
  153. if (!$NO_EXTRAS) {
  154. $extras = addExtras($dir, $videos);
  155. }
  156. # Drop any "excludes" videos
  157. my $excludes = {};
  158. if (!$NO_EXCLUDES) {
  159. $excludes = dropExcludes($dir, $videos);
  160. }
  161. # Calculate the episode number using the publish dates
  162. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  163. my $num = 1;
  164. foreach my $id (@byDate) {
  165. $videos->{$id}->{'number'} = $num++;
  166. }
  167. # Find all existing YT files on disk
  168. my $files = {};
  169. if (!$NO_FILES) {
  170. $files = findFiles($dir);
  171. }
  172. # Fill in missing videos and NFOs
  173. foreach my $id (keys(%{$videos})) {
  174. my $basePath = $dir . '/S01E' . sprintf('%02d', $videos->{$id}->{'number'}) . ' - ' . $id . '.';
  175. my $nfo = $basePath . 'nfo';
  176. # Warn (and optionally rename) if the video numbers drift
  177. if (exists($files->{$id}) && $files->{$id}->{'number'} != $videos->{$id}->{'number'}) {
  178. if ($RENAME) {
  179. print STDERR 'Renaming ' . $files->{$id}->{'path'} . ' => ' . $basePath . $files->{$id}->{'suffix'} . "\n";
  180. rename($files->{$id}->{'path'}, $basePath . $files->{$id}->{'suffix'});
  181. rename($files->{$id}->{'nfo'}, $nfo);
  182. # This is a hack, but it fits nicely in one line
  183. system('sed', '-i', 's%<episode>[0-9]*</episode>%<episode>' . $videos->{$id}->{'number'} . '</episode>%', $nfo);
  184. } else {
  185. # Find the old NFO to avoid re-fetching
  186. $nfo = $files->{$id}->{'nfo'};
  187. # Ignore small changes
  188. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  189. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  190. if ($tolerance < $DRIFT_TOLERANCE) {
  191. $tolerance = $DRIFT_TOLERANCE;
  192. }
  193. if ($delta > $tolerance || $DEBUG) {
  194. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  195. }
  196. }
  197. }
  198. # If we haven't heard of the file, or don't have an NFO for it
  199. # Checking for the NFO allows use to resume failed downloads
  200. if (!exists($files->{$id}) || !-e $nfo) {
  201. if ($DEBUG) {
  202. print STDERR 'Fetching video: ' . $id . "\n";
  203. }
  204. # Find the download URL and file suffix
  205. my ($url, $suffix) = ytURL($id);
  206. if (!defined($url) || length($url) < 5) {
  207. warn('Could not determine URL for video: ' . $id . "\n");
  208. $url = undef();
  209. }
  210. # Fetch with cURL
  211. # I know LWP exists (and is even loaded) but cURL makes my life easier
  212. if ($url) {
  213. my @cmd = ($CURL_BIN);
  214. push(@cmd, @CURL_ARGS);
  215. push(@cmd, '-o', $basePath . $suffix, $url);
  216. if ($NO_FETCH) {
  217. print STDERR 'Not running: ' . join(' ', @cmd) . "\n";
  218. } else {
  219. if ($DEBUG > 1) {
  220. print STDERR join(' ', @cmd) . "\n";
  221. }
  222. system(@cmd);
  223. }
  224. }
  225. # Build and save the XML document
  226. my $xml = buildNFO($videos->{$id});
  227. if ($DEBUG > 1) {
  228. print STDERR 'Saving NFO: ' . $xml . "\n";
  229. }
  230. if ($NO_NFO) {
  231. print STDERR "Not saving NFO\n";
  232. } else {
  233. saveString($nfo, $xml);
  234. }
  235. }
  236. }
  237. sub saveString($$) {
  238. my ($path, $str) = @_;
  239. my $fh = undef();
  240. if (!open($fh, '>', $path)) {
  241. warn('Cannot open file for writing: ' . $! . "\n");
  242. return undef();
  243. }
  244. print $fh $str;
  245. close($fh);
  246. return 1;
  247. }
  248. sub buildSeriesNFO($) {
  249. my ($channel) = @_;
  250. # Create an XML tree
  251. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  252. $doc->setStandalone(1);
  253. my $show = $doc->createElement('tvshow');
  254. $doc->setDocumentElement($show);
  255. my $elm;
  256. # Add data
  257. $elm = $doc->createElement('title');
  258. $elm->appendText($channel->{'title'});
  259. $show->appendChild($elm);
  260. $elm = $doc->createElement('premiered');
  261. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  262. $show->appendChild($elm);
  263. $elm = $doc->createElement('plot');
  264. $elm->appendText($channel->{'description'});
  265. $show->appendChild($elm);
  266. # Return the string
  267. return $doc->toString();
  268. }
  269. sub buildNFO($) {
  270. my ($video) = @_;
  271. # Create an XML tree
  272. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  273. $doc->setStandalone(1);
  274. my $show = $doc->createElement('episodedetails');
  275. $doc->setDocumentElement($show);
  276. my $elm;
  277. # Add data
  278. $elm = $doc->createElement('season');
  279. $elm->appendText('1');
  280. $show->appendChild($elm);
  281. $elm = $doc->createElement('episode');
  282. $elm->appendText($video->{'number'});
  283. $show->appendChild($elm);
  284. if (!defined($video->{'title'})) {
  285. $video->{'title'} = 'Episode ' . $video->{'number'};
  286. }
  287. $elm = $doc->createElement('title');
  288. $elm->appendText($video->{'title'});
  289. $show->appendChild($elm);
  290. if (!defined($video->{'date'})) {
  291. $video->{'date'} = time();
  292. }
  293. $elm = $doc->createElement('aired');
  294. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  295. $show->appendChild($elm);
  296. if (defined($video->{'description'})) {
  297. $elm = $doc->createElement('plot');
  298. $elm->appendText($video->{'description'});
  299. $show->appendChild($elm);
  300. }
  301. if (defined($video->{'duration'})) {
  302. $elm = $doc->createElement('runtime');
  303. $elm->appendText($video->{'duration'});
  304. $show->appendChild($elm);
  305. }
  306. if (defined($video->{'rating'})) {
  307. $elm = $doc->createElement('rating');
  308. $elm->appendText($video->{'rating'});
  309. $show->appendChild($elm);
  310. }
  311. if (defined($video->{'creator'})) {
  312. $elm = $doc->createElement('director');
  313. $elm->appendText($video->{'creator'});
  314. $show->appendChild($elm);
  315. }
  316. # Return the string
  317. return $doc->toString();
  318. }
  319. sub ytURL($) {
  320. my ($id) = @_;
  321. # Init the YT object
  322. my $tube = WWW::YouTube::Download->new;
  323. # Fetch metadata
  324. my $meta = eval { $tube->prepare_download($id); };
  325. if (!defined($meta) || ref($meta) ne 'HASH' || !exists($meta->{'video_url_map'}) || ref($meta->{'video_url_map'}) ne 'HASH') {
  326. if ($@ =~ /age gate/i) {
  327. print STDERR "Video not available due to age gate restrictions\n";
  328. }
  329. return undef();
  330. }
  331. # Find the best stream (i.e. highest resolution, prefer mp4)
  332. my $bestStream = undef();
  333. my $bestResolution = 0;
  334. foreach my $streamID (keys(%{ $meta->{'video_url_map'} })) {
  335. my $stream = $meta->{'video_url_map'}->{$streamID};
  336. my ($res) = $stream->{'resolution'} =~ /^\s*(\d+)/;
  337. if ($DEBUG > 1) {
  338. print STDERR $streamID . ' (' . $stream->{'suffix'} . ')' . ' => ' . $stream->{'resolution'} . ' : ' . $stream->{'url'} . "\n";
  339. }
  340. if ( ($res > $bestResolution)
  341. || ($res == $bestResolution && defined($bestStream) && $bestStream->{'suffix'} ne 'mp4'))
  342. {
  343. $bestStream = $stream;
  344. $bestResolution = $res;
  345. }
  346. }
  347. if (!exists($bestStream->{'suffix'}) || length($bestStream->{'suffix'}) < 2) {
  348. $bestStream->{'suffix'} = 'mp4';
  349. }
  350. return ($bestStream->{'url'}, $bestStream->{'suffix'});
  351. }
  352. sub findFiles($) {
  353. my ($dir) = @_;
  354. my %files = ();
  355. # Allow complete bypass
  356. if ($NO_FILES) {
  357. return \%files;
  358. }
  359. # Read the output directory
  360. my $fh = undef();
  361. opendir($fh, $dir)
  362. or die('Unable to open output directory: ' . $! . "\n");
  363. while (my $file = readdir($fh)) {
  364. my ($num, $id, $suffix) = $file =~ /^S01+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  365. if (defined($id) && length($id) > 0) {
  366. if ($suffix eq 'nfo') {
  367. next;
  368. }
  369. my %tmp = (
  370. 'number' => $num,
  371. 'suffix' => $suffix,
  372. 'path' => $dir . '/' . $file,
  373. );
  374. $tmp{'nfo'} = $tmp{'path'};
  375. $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/;
  376. if (exists($files{$id})) {
  377. warn('Duplicate ID: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $tmp{'path'} . "\n");
  378. if ($RENAME) {
  379. # Prefer to delete MP4 files, if the suffixes differ
  380. my $del = $files{$id};
  381. if ($tmp{'suffix'} eq 'mp4' && $del->{'suffix'} ne 'mp4') {
  382. $del = \%tmp;
  383. %tmp = %{ $files{$id} };
  384. }
  385. print STDERR 'Deleting duplicate: ' . $del->{'path'} . "\n";
  386. unlink($del->{'path'});
  387. unlink($del->{'nfo'});
  388. }
  389. }
  390. $files{$id} = \%tmp;
  391. }
  392. }
  393. close($fh);
  394. if ($DEBUG) {
  395. print STDERR "\nFound " . scalar(keys(%files)) . " files:\n\t" . join(', ', keys(%files)) . "\n";
  396. if ($DEBUG > 1) {
  397. print STDERR prettyPrint(\%files, "\t") . "\n";
  398. }
  399. }
  400. return \%files;
  401. }
  402. sub fetchParse($$) {
  403. my ($name, $id) = @_;
  404. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  405. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  406. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  407. }
  408. # Fetch
  409. if ($DEBUG) {
  410. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  411. }
  412. my $content = get($url);
  413. if (!defined($content) || length($content) < 10) {
  414. die('Invalid content from URL: ' . $url . "\n");
  415. }
  416. # Parse
  417. my $data = decode_json($content);
  418. if (!defined($data) || ref($data) ne 'HASH') {
  419. die('Invalid JSON: ' . $content . "\n");
  420. }
  421. if ($DEBUG > 2) {
  422. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  423. }
  424. return $data;
  425. }
  426. sub saveChannel($$) {
  427. my ($dir, $channel) = @_;
  428. my $nfo = $dir . '/tvshow.nfo';
  429. if (!-e $nfo) {
  430. if ($DEBUG) {
  431. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  432. }
  433. # Save the poster
  434. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  435. my $poster = $dir . '/poster.jpg';
  436. my $jpg = get($channel->{'thumbnail'});
  437. saveString($poster, $jpg);
  438. }
  439. # Save the series NFO
  440. my $xml = buildSeriesNFO($channel);
  441. if ($DEBUG > 1) {
  442. print STDERR 'Saving NFO: ' . $xml . "\n";
  443. }
  444. saveString($nfo, $xml);
  445. }
  446. }
  447. sub getChannel($) {
  448. my ($user) = @_;
  449. # Build, fetch, parse
  450. my $data = fetchParse('channel', $user);
  451. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  452. die("Invalid channel data\n");
  453. }
  454. $data = $data->{'entry'};
  455. # Extract the data we want
  456. my %channel = (
  457. 'id' => $data->{'yt$channelId'}->{'$t'},
  458. 'title' => $data->{'title'}->{'$t'},
  459. 'date' => str2time($data->{'published'}->{'$t'}),
  460. 'description' => $data->{'summary'}->{'$t'},
  461. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  462. );
  463. return \%channel;
  464. }
  465. sub readExcludes($) {
  466. my ($dir) = @_;
  467. my %excludes = ();
  468. # Read and parse the excludes videos file, if it exists
  469. my $file = $dir . '/' . $EXCLUDES_FILE;
  470. if (-e $file) {
  471. my $fh;
  472. open($fh, $file)
  473. or die('Unable to open excludes videos file: ' . $! . "\n");
  474. while (<$fh>) {
  475. # Skip blank lines and comments
  476. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  477. next;
  478. }
  479. # Match our specific format or whine
  480. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  481. if ($DEBUG > 1) {
  482. print STDERR 'Adding exclude video: ' . $1 . "\n";
  483. }
  484. $excludes{$1} = 1;
  485. } else {
  486. print STDERR 'Skipped exclude video line: ' . $_;
  487. }
  488. }
  489. close($fh);
  490. }
  491. return \%excludes;
  492. }
  493. sub readExtras($) {
  494. my ($dir) = @_;
  495. my %extras = ();
  496. # Read and parse the extra videos file, if it exists
  497. my $file = $dir . '/' . $EXTRAS_FILE;
  498. if (-e $file) {
  499. my $fh;
  500. open($fh, $file)
  501. or die('Unable to open extra videos file: ' . $! . "\n");
  502. while (<$fh>) {
  503. # Skip blank lines and comments
  504. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  505. next;
  506. }
  507. # Match our specific format or whine
  508. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  509. if ($DEBUG > 1) {
  510. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  511. }
  512. $extras{$2} = $1;
  513. } else {
  514. print STDERR 'Skipped extra video line: ' . $_;
  515. }
  516. }
  517. close($fh);
  518. }
  519. return \%extras;
  520. }
  521. sub parseVideoData($) {
  522. my ($data) = @_;
  523. my %video = (
  524. 'title' => $data->{'title'},
  525. 'date' => str2time($data->{'uploaded'}),
  526. 'description' => $data->{'description'},
  527. 'duration' => $data->{'duration'},
  528. 'rating' => $data->{'rating'},
  529. 'creator' => $data->{'uploader'},
  530. );
  531. return \%video;
  532. }
  533. sub findVideo($) {
  534. my ($id) = @_;
  535. # Build, fetch, parse
  536. my $data = fetchParse('video', $id);
  537. # Validate
  538. if (!exists($data->{'data'})) {
  539. die("Invalid video list\n");
  540. }
  541. $data = $data->{'data'};
  542. # Parse the video data
  543. return parseVideoData($data);
  544. }
  545. sub dropExcludes($$) {
  546. my ($dir, $videos) = @_;
  547. my $excludes = readExcludes($dir);
  548. foreach my $id (keys(%{$excludes})) {
  549. if (!exists($videos->{$id})) {
  550. if ($DEBUG > 1) {
  551. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  552. }
  553. next;
  554. }
  555. delete($videos->{$id});
  556. }
  557. return $excludes;
  558. }
  559. sub addExtras($$) {
  560. my ($dir, $videos) = @_;
  561. my $extras = readExtras($dir);
  562. foreach my $id (keys(%{$extras})) {
  563. if (exists($videos->{$id})) {
  564. if ($DEBUG > 1) {
  565. print STDERR 'Skipping known extra video: ' . $id . "\n";
  566. }
  567. next;
  568. }
  569. my $video = findVideo($id);
  570. $video->{'api_number'} = $extras->{$id};
  571. $videos->{$id} = $video;
  572. }
  573. return $extras;
  574. }
  575. sub findVideos($) {
  576. my ($user) = @_;
  577. my %videos = ();
  578. # Allow complete bypass
  579. if ($NO_SEARCH) {
  580. return \%videos;
  581. }
  582. # Loop through until we have all the entries
  583. my $index = 1;
  584. my $itemCount = undef();
  585. LOOP:
  586. {
  587. # Build, fetch, parse
  588. $API{'search'}{'params'}{'start-index'} = $index;
  589. my $data = fetchParse('search', $user);
  590. # Grab the total count, so we know when to stop
  591. if (!exists($data->{'data'})) {
  592. die("Invalid video list\n");
  593. }
  594. $data = $data->{'data'};
  595. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  596. $itemCount = $data->{'totalItems'};
  597. }
  598. # Process each item
  599. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  600. die("Invalid video list\n");
  601. }
  602. my $items = $data->{'items'};
  603. my $offset = 0;
  604. foreach my $item (@{$items}) {
  605. my $video = parseVideoData($item);
  606. $video->{'api_number'} = $itemCount - $index - $offset + 1;
  607. $videos{ $item->{'id'} } = $video;
  608. $offset++;
  609. }
  610. # Loop if there are results left to fetch
  611. $index += $BATCH_SIZE;
  612. if (defined($itemCount) && $itemCount > $index) {
  613. # But don't go past the max supported index
  614. if ($index <= $MAX_INDEX) {
  615. redo LOOP;
  616. }
  617. }
  618. }
  619. if ($DEBUG) {
  620. print STDERR "\nFound " . scalar(keys(%videos)) . ' videos for YouTube user ' . $user . ":\n\t" . join(', ', keys(%videos)) . "\n";
  621. if ($DEBUG > 1) {
  622. print STDERR prettyPrint(\%videos, "\t") . "\n";
  623. }
  624. }
  625. return \%videos;
  626. }