ytSubscribe.pl 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use File::Basename;
  5. use Date::Parse;
  6. use Date::Format;
  7. use LWP::Simple;
  8. use URI::Escape;
  9. use JSON;
  10. use XML::LibXML;
  11. use IPC::System::Simple qw( system );
  12. use WWW::YouTube::Download;
  13. # Paramters
  14. my $CURL_BIN = 'curl';
  15. my @CURL_ARGS = ('-4', '--insecure', '-C', '-');
  16. my $BATCH_SIZE = 50;
  17. my $MAX_INDEX = 500;
  18. my $URL_PREFIX = 'http://gdata.youtube.com/feeds/api/users/';
  19. my $URL_SUFFIX = '/uploads';
  20. my %API_PARAMS = (
  21. 'strict' => 1,
  22. 'v' => 2,
  23. 'alt' => 'jsonc',
  24. 'orderby' => 'published',
  25. );
  26. # Prototypes
  27. sub findVideos($);
  28. sub findFiles($);
  29. sub ytURL($);
  30. sub buildNFO($);
  31. # Sanity check
  32. if (scalar(@ARGV) < 1) {
  33. die('Usage: ' . basename($0) . " output_directory\n");
  34. }
  35. # Command-line parameters
  36. my ($dir) = @ARGV;
  37. $dir =~ s/\/+$//;
  38. if (!-d $dir) {
  39. die('Invalid output directory: ' . $dir . "\n");
  40. }
  41. my $user = basename($dir);
  42. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  43. die('Invalid user: ' . $user . "\n");
  44. }
  45. # Environmental parameters
  46. my $DEBUG = 0;
  47. if ($ENV{'DEBUG'}) {
  48. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  49. $DEBUG = $1;
  50. } else {
  51. $DEBUG = 1;
  52. }
  53. }
  54. my $NO_FETCH = 0;
  55. if ($ENV{'NO_FETCH'}) {
  56. $NO_FETCH = 1;
  57. }
  58. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  59. $MAX_INDEX = $1;
  60. }
  61. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  62. $BATCH_SIZE = $1;
  63. }
  64. # Construct globals
  65. $API_PARAMS{'max-results'} = $BATCH_SIZE;
  66. if (!$DEBUG) {
  67. push(@CURL_ARGS, '--silent');
  68. }
  69. # Find all the user's videos on YT
  70. my $videos = findVideos($user);
  71. # Find all existing YT files on disk
  72. my $files = findFiles($dir);
  73. # Fill in missing videos and NFOs
  74. foreach my $id (keys(%{$videos})) {
  75. my $basePath = $dir . '/' . sprintf('%02d', $videos->{$id}->{'number'}) . ' - ' . $id . '.';
  76. my $nfo = $basePath . 'nfo';
  77. # If we haven't heard of the file, or don't have an NFO for it
  78. # Checking for the NFO allows use to resume failed downloads
  79. if (!exists($files->{$id}) || !-e $nfo) {
  80. if ($DEBUG) {
  81. print STDERR 'Fetching video: ' . $id . "\n";
  82. }
  83. # Find the download URL and file suffix
  84. my ($url, $suffix) = ytURL($id);
  85. if (!defined($url) || length($url) < 5) {
  86. warn('Could not determine URL for video: ' . $id . "\n");
  87. next;
  88. }
  89. # Fetch with cURL
  90. # I know LWP exists (and is even loaded) but cURL makes my life easier
  91. {
  92. my @cmd = ($CURL_BIN);
  93. push(@cmd, @CURL_ARGS);
  94. push(@cmd, '-o', $basePath . $suffix, $url);
  95. if ($NO_FETCH) {
  96. print STDERR 'Not fetching: ' . join(' ', @cmd) . "\n";
  97. } else {
  98. if ($DEBUG > 1) {
  99. print STDERR join(' ', @cmd) . "\n";
  100. }
  101. system(@cmd);
  102. }
  103. }
  104. # Build the XML document
  105. my $xml = buildNFO($videos->{$id});
  106. if ($DEBUG > 1) {
  107. print STDERR 'Saving NFO: ' . $xml . "\n";
  108. }
  109. # Save to disk
  110. my $fh = undef();
  111. if (!open($fh, '>', $nfo)) {
  112. warn('Cannot open NFO: ' . $! . "\n");
  113. next;
  114. }
  115. print $fh $xml;
  116. close($fh);
  117. }
  118. }
  119. sub buildNFO($) {
  120. my ($video) = @_;
  121. # Create an XML tree
  122. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  123. $doc->setStandalone(1);
  124. my $show = $doc->createElement('episodedetails');
  125. $doc->setDocumentElement($show);
  126. my $elm;
  127. # Generic elements (in case we get excited later)
  128. my $fileinfo = $doc->createElement('fileinfo');
  129. $show->appendChild($fileinfo);
  130. my $streamdetails = $doc->createElement('streamdetails');
  131. $fileinfo->appendChild($streamdetails);
  132. $elm = $doc->createElement('video');
  133. $streamdetails->appendChild($elm);
  134. $elm = $doc->createElement('playcount');
  135. $show->appendChild($elm);
  136. $elm = $doc->createElement('credits');
  137. $show->appendChild($elm);
  138. my $actor = $doc->createElement('actor');
  139. $show->appendChild($actor);
  140. $elm = $doc->createElement('name');
  141. $actor->appendChild($elm);
  142. $elm = $doc->createElement('role');
  143. $actor->appendChild($elm);
  144. # Static elements (per the nature of this process)
  145. $elm = $doc->createElement('season');
  146. $elm->appendText('1');
  147. $show->appendChild($elm);
  148. # Dynamic data
  149. $elm = $doc->createElement('episode');
  150. $elm->appendText($video->{'number'});
  151. $show->appendChild($elm);
  152. $elm = $doc->createElement('title');
  153. $elm->appendText($video->{'title'});
  154. $show->appendChild($elm);
  155. $elm = $doc->createElement('aired');
  156. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  157. $show->appendChild($elm);
  158. $elm = $doc->createElement('plot');
  159. $elm->appendText($video->{'description'});
  160. $show->appendChild($elm);
  161. $elm = $doc->createElement('runtime');
  162. $elm->appendText($video->{'duration'});
  163. $show->appendChild($elm);
  164. $elm = $doc->createElement('rating');
  165. $elm->appendText($video->{'rating'});
  166. $show->appendChild($elm);
  167. $elm = $doc->createElement('director');
  168. $elm->appendText($video->{'creator'});
  169. $show->appendChild($elm);
  170. # Return the string
  171. return $doc->toString();
  172. }
  173. sub ytURL($) {
  174. my ($id) = @_;
  175. # Init the YT object
  176. my $tube = WWW::YouTube::Download->new;
  177. # Fetch metadata
  178. my $meta = $tube->prepare_download($id);
  179. if (!defined($meta) || ref($meta) ne 'HASH' || !exists($meta->{'video_url_map'}) || ref($meta->{'video_url_map'}) ne 'HASH') {
  180. return undef();
  181. }
  182. # Find the best stream (i.e. highest resolution, prefer mp4)
  183. my $bestStream = undef();
  184. my $bestResolution = 0;
  185. foreach my $streamID (keys(%{ $meta->{'video_url_map'} })) {
  186. my $stream = $meta->{'video_url_map'}->{$streamID};
  187. my ($res) = $stream->{'resolution'} =~ /^\s*(\d+)/;
  188. if ($DEBUG > 1) {
  189. print STDERR $streamID . ' (' . $stream->{'suffix'} . ')' . ' => ' . $stream->{'resolution'} . ' : ' . $stream->{'url'} . "\n\n";
  190. }
  191. if ( ($res > $bestResolution)
  192. || ($res == $bestResolution && defined($bestStream) && $bestStream->{'suffix'} ne 'mp4'))
  193. {
  194. $bestStream = $stream;
  195. $bestResolution = $res;
  196. }
  197. }
  198. if (!exists($bestStream->{'suffix'}) || length($bestStream->{'suffix'}) < 2) {
  199. $bestStream->{'suffix'} = 'mp4';
  200. }
  201. return ($bestStream->{'url'}, $bestStream->{'suffix'});
  202. }
  203. sub findFiles($) {
  204. my ($dir) = @_;
  205. my %files = ();
  206. # Read the output directory
  207. my $fh = undef();
  208. opendir($fh, $dir)
  209. or die('Unable to open output directory: ' . $! . "\n");
  210. while (my $file = readdir($fh)) {
  211. my ($num, $id, $suffix) = $file =~ /(\d+) - ([\w\-]+)\.(\w\w\w)$/;
  212. if (defined($id) && length($id) > 0) {
  213. if ($suffix eq 'nfo') {
  214. next;
  215. }
  216. my %tmp = (
  217. 'number' => $num,
  218. 'suffix' => $suffix,
  219. 'path' => $dir . '/' . $file,
  220. );
  221. $tmp{'nfo'} = $tmp{'path'};
  222. $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/;
  223. $files{$id} = \%tmp;
  224. }
  225. }
  226. close($fh);
  227. if ($DEBUG) {
  228. print STDERR "\n\nFound " . scalar(keys(%files)) . " files:\n\t" . join("\n\t", keys(%files)) . "\n\n";
  229. if ($DEBUG > 1) {
  230. use PrettyPrint;
  231. print STDERR prettyPrint(\%files, "\t") . "\n";
  232. }
  233. }
  234. return \%files;
  235. }
  236. sub findVideos($) {
  237. my ($user) = @_;
  238. my %videos = ();
  239. # Build the base URL
  240. my $baseURL = $URL_PREFIX . $user . $URL_SUFFIX;
  241. # And the static URL parameters
  242. my %params = %API_PARAMS;
  243. # Loop through until we have all the entries
  244. my $index = 1;
  245. my $itemCount = undef();
  246. LOOP:
  247. {
  248. # Build the actual URL
  249. my $url = $baseURL . '?start-index=' . $index;
  250. foreach my $name (keys(%params)) {
  251. $url .= '&' . uri_escape($name) . '=' . uri_escape($params{$name});
  252. }
  253. # Fetch
  254. if ($DEBUG) {
  255. print STDERR 'Fetching list URL: ' . $url . "\n";
  256. }
  257. my $content = get($url);
  258. if (!defined($content) || length($content) < 10) {
  259. die('Invalid content from URL: ' . $url . "\n");
  260. }
  261. # Parse
  262. my $data = decode_json($content);
  263. if (!defined($data) || ref($data) ne 'HASH') {
  264. die('Invalid JSON: ' . $content . "\n");
  265. }
  266. if (!exists($data->{'data'})) {
  267. die('Invalid data set: ' . $content . "\n");
  268. }
  269. $data = $data->{'data'};
  270. # Grab the total count, so we know when to stop
  271. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  272. $itemCount = $data->{'totalItems'};
  273. }
  274. # Process each item
  275. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  276. die('Invalid item list: ' . $content . "\n");
  277. }
  278. my $items = $data->{'items'};
  279. my $offset = 0;
  280. foreach my $item (@{$items}) {
  281. my %tmp = (
  282. 'number' => ($itemCount - $index - $offset + 1),
  283. 'title' => $item->{'title'},
  284. 'date' => str2time($item->{'uploaded'}),
  285. 'description' => $item->{'description'},
  286. 'duration' => $item->{'duration'},
  287. 'rating' => $item->{'rating'},
  288. 'creator' => $item->{'uploader'},
  289. );
  290. $videos{ $item->{'id'} } = \%tmp;
  291. $offset++;
  292. }
  293. # Loop if there are results left to fetch
  294. $index += $BATCH_SIZE;
  295. if (defined($itemCount) && $itemCount > $index) {
  296. # But don't go past the max supported index
  297. if ($index <= $MAX_INDEX) {
  298. redo LOOP;
  299. }
  300. }
  301. }
  302. if ($DEBUG) {
  303. print STDERR "\n\nFound " . scalar(keys(%videos)) . ' videos for YouTube user ' . $user . ":\n\t" . join("\n\t", keys(%videos)) . "\n\n";
  304. if ($DEBUG > 1) {
  305. use PrettyPrint;
  306. print STDERR prettyPrint(\%videos, "\t") . "\n";
  307. }
  308. }
  309. return \%videos;
  310. }