subscribe.pl 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Basename;
  7. use Date::Parse;
  8. use Date::Format;
  9. use LWP::Simple;
  10. use URI::Escape;
  11. use JSON;
  12. use XML::LibXML;
  13. use IPC::System::Simple qw( system );
  14. use WWW::YouTube::Download;
  15. use PrettyPrint;
  16. # Paramters
  17. my $EXTRAS_FILE = 'extra_videos.ini';
  18. my $CURL_BIN = 'curl';
  19. my @CURL_ARGS = ('-4', '--insecure', '-C', '-', '--connect-timeout', '10', '--max-time', '600');
  20. my $BATCH_SIZE = 50;
  21. my $MAX_INDEX = 2500;
  22. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  23. my %API = (
  24. 'search' => {
  25. 'prefix' => $API_URL . 'users/',
  26. 'suffix' => '/uploads',
  27. 'params' => {
  28. 'start-index' => 1,
  29. 'max-results' => 1,
  30. 'strict' => 1,
  31. 'v' => 2,
  32. 'alt' => 'jsonc',
  33. },
  34. },
  35. 'video' => {
  36. 'prefix' => $API_URL . 'videos/',
  37. 'suffix' => '',
  38. 'params' => {
  39. 'strict' => 1,
  40. 'v' => 2,
  41. 'alt' => 'jsonc'
  42. },
  43. },
  44. 'channel' => {
  45. 'prefix' => $API_URL . 'users/',
  46. 'suffix' => '',
  47. 'params' => {
  48. 'strict' => 1,
  49. 'v' => 2,
  50. 'alt' => 'json'
  51. },
  52. },
  53. );
  54. # Prototypes
  55. sub findVideos($);
  56. sub findFiles($);
  57. sub findVideo($);
  58. sub ytURL($);
  59. sub buildNFO($);
  60. sub buildSeriesNFO($);
  61. sub getChannel($);
  62. sub fetchParse($$);
  63. sub saveString($$);
  64. sub readExtras($);
  65. sub parseVideoData($);
  66. sub addExtras($$);
  67. sub saveChannel($$);
  68. # Sanity check
  69. if (scalar(@ARGV) < 1) {
  70. die('Usage: ' . basename($0) . " output_directory\n");
  71. }
  72. # Command-line parameters
  73. my ($dir) = @ARGV;
  74. $dir =~ s/\/+$//;
  75. if (!-d $dir) {
  76. die('Invalid output directory: ' . $dir . "\n");
  77. }
  78. my $user = basename($dir);
  79. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  80. die('Invalid user: ' . $user . "\n");
  81. }
  82. # Environmental parameters (debug)
  83. my $DEBUG = 0;
  84. if ($ENV{'DEBUG'}) {
  85. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  86. $DEBUG = $1;
  87. } else {
  88. $DEBUG = 1;
  89. }
  90. }
  91. my $NO_FETCH = 0;
  92. if ($ENV{'NO_FETCH'}) {
  93. $NO_FETCH = 1;
  94. }
  95. my $NO_SEARCH = 0;
  96. if ($ENV{'NO_SEARCH'}) {
  97. $NO_SEARCH = 1;
  98. }
  99. my $NO_FILES = 0;
  100. if ($ENV{'NO_FILES'}) {
  101. $NO_FILES = 1;
  102. }
  103. my $NO_CHANNEL = 0;
  104. if ($ENV{'NO_CHANNEL'}) {
  105. $NO_CHANNEL = 1;
  106. }
  107. my $NO_EXTRAS = 0;
  108. if ($ENV{'NO_EXTRAS'}) {
  109. $NO_EXTRAS = 1;
  110. }
  111. # Environmental parameters (functional)
  112. my $RENAME = 0;
  113. if ($ENV{'RENAME'}) {
  114. $RENAME = 1;
  115. }
  116. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  117. $MAX_INDEX = $1;
  118. }
  119. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  120. $BATCH_SIZE = $1;
  121. }
  122. # Construct globals
  123. $API{'search'}{'params'}{'max-results'} = $BATCH_SIZE;
  124. if (!$DEBUG) {
  125. push(@CURL_ARGS, '--silent');
  126. }
  127. # Grab the channel data
  128. my $channel = {};
  129. if (!$NO_CHANNEL) {
  130. $channel = getChannel($user);
  131. saveChannel($dir, $channel);
  132. }
  133. # Find all the user's videos on YT
  134. my $videos = {};
  135. if (!$NO_SEARCH) {
  136. $videos = findVideos($user);
  137. }
  138. # Find any requested "extra" videos
  139. my $extras = {};
  140. if (!$NO_EXTRAS) {
  141. $extras = addExtras($dir, $videos);
  142. }
  143. # Find all existing YT files on disk
  144. my $files = {};
  145. if (!$NO_FILES) {
  146. $files = findFiles($dir);
  147. }
  148. # Fill in missing videos and NFOs
  149. foreach my $id (keys(%{$videos})) {
  150. my $basePath = $dir . '/S01E' . sprintf('%02d', $videos->{$id}->{'number'}) . ' - ' . $id . '.';
  151. my $nfo = $basePath . 'nfo';
  152. # Warn (and optionally rename) if the video numbers drift
  153. if (exists($files->{$id}) && $files->{$id}->{'number'} != $videos->{$id}->{'number'}) {
  154. warn('Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n");
  155. if ($RENAME) {
  156. print STDERR 'Renaming ' . $files->{$id}->{'path'} . ' => ' . $basePath . $files->{$id}->{'suffix'} . "\n";
  157. rename($files->{$id}->{'path'}, $basePath . $files->{$id}->{'suffix'});
  158. unlink($files->{$id}->{'nfo'});
  159. delete($files->{$id});
  160. }
  161. }
  162. # If we haven't heard of the file, or don't have an NFO for it
  163. # Checking for the NFO allows use to resume failed downloads
  164. if (!exists($files->{$id}) || !-e $nfo) {
  165. if ($DEBUG) {
  166. print STDERR 'Fetching video: ' . $id . "\n";
  167. }
  168. # Find the download URL and file suffix
  169. my ($url, $suffix) = ytURL($id);
  170. if (!defined($url) || length($url) < 5) {
  171. warn('Could not determine URL for video: ' . $id . "\n");
  172. $url = undef();
  173. }
  174. # Fetch with cURL
  175. # I know LWP exists (and is even loaded) but cURL makes my life easier
  176. if ($url) {
  177. my @cmd = ($CURL_BIN);
  178. push(@cmd, @CURL_ARGS);
  179. push(@cmd, '-o', $basePath . $suffix, $url);
  180. if ($NO_FETCH) {
  181. print STDERR 'Not running: ' . join(' ', @cmd) . "\n";
  182. } else {
  183. if ($DEBUG > 1) {
  184. print STDERR join(' ', @cmd) . "\n";
  185. }
  186. system(@cmd);
  187. }
  188. }
  189. # Build and save the XML document
  190. my $xml = buildNFO($videos->{$id});
  191. if ($DEBUG > 1) {
  192. print STDERR 'Saving NFO: ' . $xml . "\n";
  193. }
  194. if ($NO_FETCH) {
  195. print STDERR "Not saving NFO\n";
  196. } else {
  197. saveString($nfo, $xml);
  198. }
  199. }
  200. }
  201. sub saveString($$) {
  202. my ($path, $str) = @_;
  203. my $fh = undef();
  204. if (!open($fh, '>', $path)) {
  205. warn('Cannot open file for writing: ' . $! . "\n");
  206. return undef();
  207. }
  208. print $fh $str;
  209. close($fh);
  210. return 1;
  211. }
  212. sub buildSeriesNFO($) {
  213. my ($channel) = @_;
  214. # Create an XML tree
  215. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  216. $doc->setStandalone(1);
  217. my $show = $doc->createElement('tvshow');
  218. $doc->setDocumentElement($show);
  219. my $elm;
  220. # Add data
  221. $elm = $doc->createElement('title');
  222. $elm->appendText($channel->{'title'});
  223. $show->appendChild($elm);
  224. $elm = $doc->createElement('premiered');
  225. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  226. $show->appendChild($elm);
  227. $elm = $doc->createElement('plot');
  228. $elm->appendText($channel->{'description'});
  229. $show->appendChild($elm);
  230. # Return the string
  231. return $doc->toString();
  232. }
  233. sub buildNFO($) {
  234. my ($video) = @_;
  235. # Create an XML tree
  236. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  237. $doc->setStandalone(1);
  238. my $show = $doc->createElement('episodedetails');
  239. $doc->setDocumentElement($show);
  240. my $elm;
  241. # Add data
  242. $elm = $doc->createElement('season');
  243. $elm->appendText('1');
  244. $show->appendChild($elm);
  245. $elm = $doc->createElement('episode');
  246. $elm->appendText($video->{'number'});
  247. $show->appendChild($elm);
  248. $elm = $doc->createElement('title');
  249. $elm->appendText($video->{'title'});
  250. $show->appendChild($elm);
  251. $elm = $doc->createElement('aired');
  252. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  253. $show->appendChild($elm);
  254. $elm = $doc->createElement('plot');
  255. $elm->appendText($video->{'description'});
  256. $show->appendChild($elm);
  257. $elm = $doc->createElement('runtime');
  258. $elm->appendText($video->{'duration'});
  259. $show->appendChild($elm);
  260. $elm = $doc->createElement('rating');
  261. $elm->appendText($video->{'rating'});
  262. $show->appendChild($elm);
  263. $elm = $doc->createElement('director');
  264. $elm->appendText($video->{'creator'});
  265. $show->appendChild($elm);
  266. # Return the string
  267. return $doc->toString();
  268. }
  269. sub ytURL($) {
  270. my ($id) = @_;
  271. # Init the YT object
  272. my $tube = WWW::YouTube::Download->new;
  273. # Fetch metadata
  274. my $meta = eval { $tube->prepare_download($id); };
  275. if (!defined($meta) || ref($meta) ne 'HASH' || !exists($meta->{'video_url_map'}) || ref($meta->{'video_url_map'}) ne 'HASH') {
  276. if ($@ =~ /age gate/i) {
  277. print STDERR "Video not available due to age gate restrictions\n";
  278. }
  279. return undef();
  280. }
  281. # Find the best stream (i.e. highest resolution, prefer mp4)
  282. my $bestStream = undef();
  283. my $bestResolution = 0;
  284. foreach my $streamID (keys(%{ $meta->{'video_url_map'} })) {
  285. my $stream = $meta->{'video_url_map'}->{$streamID};
  286. my ($res) = $stream->{'resolution'} =~ /^\s*(\d+)/;
  287. if ($DEBUG > 1) {
  288. print STDERR $streamID . ' (' . $stream->{'suffix'} . ')' . ' => ' . $stream->{'resolution'} . ' : ' . $stream->{'url'} . "\n";
  289. }
  290. if ( ($res > $bestResolution)
  291. || ($res == $bestResolution && defined($bestStream) && $bestStream->{'suffix'} ne 'mp4'))
  292. {
  293. $bestStream = $stream;
  294. $bestResolution = $res;
  295. }
  296. }
  297. if (!exists($bestStream->{'suffix'}) || length($bestStream->{'suffix'}) < 2) {
  298. $bestStream->{'suffix'} = 'mp4';
  299. }
  300. return ($bestStream->{'url'}, $bestStream->{'suffix'});
  301. }
  302. sub findFiles($) {
  303. my ($dir) = @_;
  304. my %files = ();
  305. # Allow complete bypass
  306. if ($NO_FILES) {
  307. return \%files;
  308. }
  309. # Read the output directory
  310. my $fh = undef();
  311. opendir($fh, $dir)
  312. or die('Unable to open output directory: ' . $! . "\n");
  313. while (my $file = readdir($fh)) {
  314. my ($num, $id, $suffix) = $file =~ /^S\d+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  315. if (defined($id) && length($id) > 0) {
  316. if ($suffix eq 'nfo') {
  317. next;
  318. }
  319. my %tmp = (
  320. 'number' => $num,
  321. 'suffix' => $suffix,
  322. 'path' => $dir . '/' . $file,
  323. );
  324. $tmp{'nfo'} = $tmp{'path'};
  325. $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/;
  326. if (exists($files{$id})) {
  327. warn('Duplicate ID: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $tmp{'path'} . "\n");
  328. if ($RENAME) {
  329. print STDERR 'Deleting duplicate: ' . $files{$id}->{'path'} . "\n";
  330. unlink($files{$id}->{'path'});
  331. unlink($files{$id}->{'nfo'});
  332. delete($files{$id});
  333. }
  334. }
  335. $files{$id} = \%tmp;
  336. }
  337. }
  338. close($fh);
  339. if ($DEBUG) {
  340. print STDERR "\nFound " . scalar(keys(%files)) . " files:\n\t" . join(', ', keys(%files)) . "\n";
  341. if ($DEBUG > 1) {
  342. print STDERR prettyPrint(\%files, "\t") . "\n";
  343. }
  344. }
  345. return \%files;
  346. }
  347. sub fetchParse($$) {
  348. my ($name, $id) = @_;
  349. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  350. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  351. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  352. }
  353. # Fetch
  354. if ($DEBUG) {
  355. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  356. }
  357. my $content = get($url);
  358. if (!defined($content) || length($content) < 10) {
  359. die('Invalid content from URL: ' . $url . "\n");
  360. }
  361. # Parse
  362. my $data = decode_json($content);
  363. if (!defined($data) || ref($data) ne 'HASH') {
  364. die('Invalid JSON: ' . $content . "\n");
  365. }
  366. if ($DEBUG > 2) {
  367. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  368. }
  369. return $data;
  370. }
  371. sub saveChannel($$) {
  372. my ($dir, $channel) = @_;
  373. my $nfo = $dir . '/tvshow.nfo';
  374. if (!-e $nfo) {
  375. if ($DEBUG) {
  376. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  377. }
  378. # Save the poster
  379. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  380. my $poster = $dir . '/poster.jpg';
  381. my $jpg = get($channel->{'thumbnail'});
  382. saveString($poster, $jpg);
  383. }
  384. # Save the series NFO
  385. my $xml = buildSeriesNFO($channel);
  386. if ($DEBUG > 1) {
  387. print STDERR 'Saving NFO: ' . $xml . "\n";
  388. }
  389. saveString($nfo, $xml);
  390. }
  391. }
  392. sub getChannel($) {
  393. my ($user) = @_;
  394. # Build, fetch, parse
  395. my $data = fetchParse('channel', $user);
  396. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  397. die("Invalid channel data\n");
  398. }
  399. $data = $data->{'entry'};
  400. # Extract the data we want
  401. my %channel = (
  402. 'id' => $data->{'yt$channelId'}->{'$t'},
  403. 'title' => $data->{'title'}->{'$t'},
  404. 'date' => str2time($data->{'published'}->{'$t'}),
  405. 'description' => $data->{'summary'}->{'$t'},
  406. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  407. );
  408. return \%channel;
  409. }
  410. sub readExtras($) {
  411. my ($dir) = @_;
  412. my %extras = ();
  413. # Allow complete bypass
  414. if ($NO_EXTRAS) {
  415. return \%extras;
  416. }
  417. # Read and parse the extra videos file, if it exists
  418. my $file = $dir . '/' . $EXTRAS_FILE;
  419. if (-e $file) {
  420. my $fh;
  421. open($fh, $file)
  422. or die('Unable to open extra videos file: ' . $! . "\n");
  423. while (<$fh>) {
  424. # Skip blank lines and comments
  425. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  426. next;
  427. }
  428. # Match our specific format or whine
  429. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  430. if ($DEBUG > 1) {
  431. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  432. }
  433. $extras{$2} = $1;
  434. } else {
  435. print STDERR 'Skipped extra video line: ' . $_;
  436. }
  437. }
  438. close($fh);
  439. }
  440. return \%extras;
  441. }
  442. sub parseVideoData($) {
  443. my ($data) = @_;
  444. my %video = (
  445. 'title' => $data->{'title'},
  446. 'date' => str2time($data->{'uploaded'}),
  447. 'description' => $data->{'description'},
  448. 'duration' => $data->{'duration'},
  449. 'rating' => $data->{'rating'},
  450. 'creator' => $data->{'uploader'},
  451. );
  452. return \%video;
  453. }
  454. sub findVideo($) {
  455. my ($id) = @_;
  456. # Build, fetch, parse
  457. my $data = fetchParse('video', $id);
  458. # Validate
  459. if (!exists($data->{'data'})) {
  460. die("Invalid video list\n");
  461. }
  462. $data = $data->{'data'};
  463. # Parse the video data
  464. return parseVideoData($data);
  465. }
  466. sub addExtras($$) {
  467. my ($dir, $videos) = @_;
  468. my $extras = readExtras($dir);
  469. foreach my $id (keys(%{$extras})) {
  470. if (exists($videos->{$id})) {
  471. if ($DEBUG > 1) {
  472. print STDERR 'Skipping known video: ' . $id . "\n";
  473. }
  474. next;
  475. }
  476. my $video = findVideo($id);
  477. $video->{'number'} = $extras->{$id};
  478. $videos->{$id} = $video;
  479. }
  480. }
  481. sub findVideos($) {
  482. my ($user) = @_;
  483. my %videos = ();
  484. # Allow complete bypass
  485. if ($NO_SEARCH) {
  486. return \%videos;
  487. }
  488. # Loop through until we have all the entries
  489. my $index = 1;
  490. my $itemCount = undef();
  491. LOOP:
  492. {
  493. # Build, fetch, parse
  494. $API{'search'}{'params'}{'start-index'} = $index;
  495. my $data = fetchParse('search', $user);
  496. # Grab the total count, so we know when to stop
  497. if (!exists($data->{'data'})) {
  498. die("Invalid video list\n");
  499. }
  500. $data = $data->{'data'};
  501. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  502. $itemCount = $data->{'totalItems'};
  503. }
  504. # Process each item
  505. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  506. die("Invalid video list\n");
  507. }
  508. my $items = $data->{'items'};
  509. my $offset = 0;
  510. foreach my $item (@{$items}) {
  511. my $video = parseVideoData($item);
  512. $video->{'number'} = $itemCount - $index - $offset + 1;
  513. $videos{ $item->{'id'} } = $video;
  514. $offset++;
  515. }
  516. # Loop if there are results left to fetch
  517. $index += $BATCH_SIZE;
  518. if (defined($itemCount) && $itemCount > $index) {
  519. # But don't go past the max supported index
  520. if ($index <= $MAX_INDEX) {
  521. redo LOOP;
  522. }
  523. }
  524. }
  525. if ($DEBUG) {
  526. print STDERR "\nFound " . scalar(keys(%videos)) . ' videos for YouTube user ' . $user . ":\n\t" . join(', ', keys(%videos)) . "\n";
  527. if ($DEBUG > 1) {
  528. print STDERR prettyPrint(\%videos, "\t") . "\n";
  529. }
  530. }
  531. return \%videos;
  532. }