subscribe.pl 24 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::Simple;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my @YTDL_DEBUG = ('--verbose');
  25. my $BATCH_SIZE = 50;
  26. my $MAX_INDEX = 25000;
  27. my $FETCH_LIMIT = 50;
  28. my $DRIFT_TOLERANCE = 2;
  29. my $DRIFT_FACTOR = 100.0;
  30. my $DELAY = 5;
  31. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  32. my %API = (
  33. 'search' => {
  34. 'prefix' => $API_URL . 'users/',
  35. 'suffix' => '/uploads',
  36. 'params' => {
  37. 'start-index' => 1,
  38. 'max-results' => 1,
  39. 'strict' => 1,
  40. 'v' => 2,
  41. 'alt' => 'jsonc',
  42. },
  43. },
  44. 'subscriptions' => {
  45. 'prefix' => $API_URL . 'users/',
  46. 'suffix' => '/subscriptions',
  47. 'params' => {
  48. 'start-index' => 1,
  49. 'max-results' => 1,
  50. 'strict' => 1,
  51. 'v' => 2,
  52. 'alt' => 'json',
  53. },
  54. },
  55. 'video' => {
  56. 'prefix' => $API_URL . 'videos/',
  57. 'suffix' => '',
  58. 'params' => {
  59. 'strict' => 1,
  60. 'v' => 2,
  61. 'alt' => 'jsonc'
  62. },
  63. },
  64. 'channel' => {
  65. 'prefix' => $API_URL . 'users/',
  66. 'suffix' => '',
  67. 'params' => {
  68. 'strict' => 1,
  69. 'v' => 2,
  70. 'alt' => 'json'
  71. },
  72. },
  73. );
  74. # Prototypes
  75. sub findVideos($);
  76. sub findFiles();
  77. sub findVideo($);
  78. sub buildNFO($);
  79. sub buildSeriesNFO($);
  80. sub getSubscriptions($$);
  81. sub saveSubscriptions($$);
  82. sub saveChannel($);
  83. sub getChannel($);
  84. sub fetchParse($$);
  85. sub saveString($$);
  86. sub readExcludes();
  87. sub readExtras();
  88. sub parseVideoData($);
  89. sub dropExcludes($);
  90. sub addExtras($);
  91. sub updateNFOData($$$);
  92. sub videoSE($$);
  93. sub videoPath($$$$);
  94. sub renameVideo($$$$$$);
  95. sub parseFilename($);
  96. # Sanity check
  97. if (scalar(@ARGV) < 1) {
  98. die('Usage: ' . basename($0) . " output_directory\n");
  99. }
  100. # Command-line parameters
  101. my ($dir) = @ARGV;
  102. $dir =~ s/\/+$//;
  103. if (!-d $dir) {
  104. die('Invalid output directory: ' . $dir . "\n");
  105. }
  106. my $user = basename($dir);
  107. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  108. die('Invalid user: ' . $user . "\n");
  109. }
  110. # Move to the target directory so we can use relative paths later
  111. chdir($dir);
  112. # Environmental parameters (debug)
  113. my $DEBUG = 0;
  114. if ($ENV{'DEBUG'}) {
  115. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  116. $DEBUG = $1;
  117. } else {
  118. $DEBUG = 1;
  119. }
  120. }
  121. my $NO_FETCH = 0;
  122. if ($ENV{'NO_FETCH'}) {
  123. $NO_FETCH = 1;
  124. }
  125. my $NO_NFO = 0;
  126. if ($ENV{'NO_NFO'}) {
  127. $NO_NFO = 1;
  128. }
  129. my $NO_SEARCH = 0;
  130. if ($ENV{'NO_SEARCH'}) {
  131. $NO_SEARCH = 1;
  132. }
  133. my $NO_FILES = 0;
  134. if ($ENV{'NO_FILES'}) {
  135. $NO_FILES = 1;
  136. }
  137. my $NO_CHANNEL = 0;
  138. if ($ENV{'NO_CHANNEL'}) {
  139. $NO_CHANNEL = 1;
  140. }
  141. my $NO_EXTRAS = 0;
  142. if ($ENV{'NO_EXTRAS'}) {
  143. $NO_EXTRAS = 1;
  144. }
  145. my $NO_EXCLUDES = 0;
  146. if ($ENV{'NO_EXCLUDES'}) {
  147. $NO_EXCLUDES = 1;
  148. }
  149. my $NO_RENAME = 0;
  150. if ($ENV{'NO_RENAME'}) {
  151. $NO_RENAME = 1;
  152. }
  153. # Environmental parameters (functional)
  154. my $FORCE_RENAME = 0;
  155. if ($ENV{'FORCE_RENAME'}) {
  156. $FORCE_RENAME = 1;
  157. }
  158. my $SUDO_CHATTR = 1;
  159. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  160. $SUDO_CHATTR = 0;
  161. }
  162. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  163. $MAX_INDEX = $1;
  164. }
  165. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  166. $BATCH_SIZE = $1;
  167. }
  168. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  169. $FETCH_LIMIT = $1;
  170. }
  171. # Construct globals
  172. foreach my $key (keys(%API)) {
  173. if (exists($API{$key}{'params'}{'max-results'})) {
  174. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  175. }
  176. }
  177. if ($DEBUG > 1) {
  178. push(@YTDL_ARGS, @YTDL_DEBUG);
  179. } elsif (!$DEBUG) {
  180. push(@YTDL_ARGS, @YTDL_QUIET);
  181. }
  182. # Allow use as a subscription manager
  183. if ($0 =~ /subscription/i) {
  184. my %subs = ();
  185. foreach my $user (keys(%USERS)) {
  186. my $tmp = getSubscriptions($user, $USERS{$user});
  187. foreach my $sub (keys(%{$tmp})) {
  188. if (exists($subs{$sub})) {
  189. $subs{$sub} .= ', ' . $tmp->{$sub};
  190. } else {
  191. $subs{$sub} = $tmp->{$sub};
  192. }
  193. }
  194. }
  195. saveSubscriptions($dir, \%subs);
  196. exit(0);
  197. }
  198. # Grab the channel data
  199. my $channel = {};
  200. if (!$NO_CHANNEL) {
  201. $channel = getChannel($user);
  202. saveChannel($channel);
  203. }
  204. # Find all the user's videos on YT
  205. my $videos = {};
  206. if (!$NO_SEARCH) {
  207. $videos = findVideos($user);
  208. }
  209. # Find any requested "extra" videos
  210. my $extras = {};
  211. if (!$NO_EXTRAS) {
  212. $extras = addExtras($videos);
  213. }
  214. # Drop any "excludes" videos
  215. my $excludes = {};
  216. if (!$NO_EXCLUDES) {
  217. $excludes = dropExcludes($videos);
  218. }
  219. # Calculate the episode number using the publish dates
  220. {
  221. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  222. my $num = undef();
  223. my $lastSeason = undef();
  224. foreach my $id (@byDate) {
  225. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  226. $num = 1;
  227. $lastSeason = $videos->{$id}->{'season'};
  228. }
  229. $videos->{$id}->{'number'} = $num++;
  230. }
  231. }
  232. # Find all existing YT files on disk
  233. my $files = {};
  234. if (!$NO_FILES) {
  235. $files = findFiles();
  236. }
  237. # Whine about unknown videos
  238. foreach my $id (keys(%{$files})) {
  239. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  240. print STDERR 'Local video not known to YT channel (' . $user . '): ' . $id . "\n";
  241. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  242. }
  243. }
  244. # Fill in missing videos and NFOs
  245. my $fetched = 0;
  246. foreach my $id (keys(%{$videos})) {
  247. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  248. # Skip remote-only videos
  249. if (!exists($files->{$id})) {
  250. next;
  251. }
  252. # Determine if we may or must rename
  253. my $rename = 0;
  254. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  255. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  256. $rename = -1;
  257. } elsif ($files->{$id}->{'number'} != $videos->{$id}->{'number'}
  258. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  259. {
  260. $rename = 1;
  261. }
  262. # Warn (and optionally rename) if the video numbers drift
  263. if ($rename != 0) {
  264. if ($rename < 0 || $FORCE_RENAME) {
  265. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  266. } else {
  267. # Find the old NFO to avoid re-fetching
  268. $nfo = $files->{$id}->{'nfo'};
  269. # Calculate the drift magnitude
  270. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  271. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  272. if ($tolerance < $DRIFT_TOLERANCE) {
  273. $tolerance = $DRIFT_TOLERANCE;
  274. }
  275. # Ignore small changes
  276. my $rename = 0;
  277. if ($FORCE_RENAME || $delta > $tolerance) {
  278. $rename = 1;
  279. }
  280. # Execute
  281. if ($DEBUG || $rename) {
  282. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  283. if ($rename) {
  284. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  285. }
  286. }
  287. }
  288. }
  289. # If we haven't heard of the file, or don't have an NFO for it
  290. # Checking for the NFO allows use to resume failed downloads
  291. if (!exists($files->{$id}) || !-e $nfo) {
  292. if ($DEBUG) {
  293. print STDERR 'Fetching video: ' . $id . "\n";
  294. }
  295. # Let youtube-dl handle the URLs and downloading
  296. {
  297. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  298. my @name = ($YTDL_BIN);
  299. push(@name, @YTDL_ARGS);
  300. my @fetch = @name;
  301. push(@name, '--get-filename');
  302. push(@fetch, @args);
  303. push(@name, @args);
  304. if ($NO_FETCH) {
  305. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  306. } else {
  307. # Count fetch attempts (even if they fail later)
  308. $fetched++;
  309. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  310. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $user . "\n";
  311. if ($DEBUG) {
  312. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  313. }
  314. exit 1;
  315. }
  316. # Find the output file name
  317. if ($DEBUG > 1) {
  318. print STDERR join(' ', @name) . "\n";
  319. }
  320. sleep($DELAY);
  321. my $file = capture(EXIT_ANY, @name);
  322. if ($EXITVAL != 0) {
  323. warn('Error executing youtube-dl (name): ' . $EXITVAL . "\n");
  324. next;
  325. }
  326. $file =~ s/^\s+//;
  327. $file =~ s/\s+$//;
  328. # Sanity check
  329. if (!$file) {
  330. warn('No file name available for video: ' . $id . "\n");
  331. next;
  332. }
  333. # Download
  334. if ($DEBUG > 1) {
  335. print STDERR join(' ', @fetch) . "\n";
  336. }
  337. sleep($DELAY);
  338. my $exit = run(EXIT_ANY, @fetch);
  339. if ($exit != 0) {
  340. warn('Error executing youtube-dl for video: ' . $file . "\n");
  341. next;
  342. }
  343. # Ensure we found something useful
  344. if (-e $file . '.part') {
  345. warn('Partial download detected: ' . $file . "\n");
  346. next;
  347. }
  348. if (!-s $file) {
  349. warn('No output video file: ' . $file . "\n");
  350. next;
  351. }
  352. # Touch the file to reflect the download time rather than the upload time
  353. touch($file);
  354. }
  355. }
  356. # Build and save the XML document
  357. my $xml = buildNFO($videos->{$id});
  358. if ($DEBUG > 1) {
  359. print STDERR 'Saving NFO: ' . $xml . "\n";
  360. }
  361. if ($NO_NFO) {
  362. print STDERR "Not saving NFO\n";
  363. } else {
  364. saveString($nfo, $xml);
  365. }
  366. }
  367. }
  368. sub saveString($$) {
  369. my ($path, $str) = @_;
  370. my $fh = undef();
  371. if (!open($fh, '>', $path)) {
  372. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  373. return undef();
  374. }
  375. print $fh $str;
  376. close($fh);
  377. return 1;
  378. }
  379. sub buildSeriesNFO($) {
  380. my ($channel) = @_;
  381. # Create an XML tree
  382. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  383. $doc->setStandalone(1);
  384. my $show = $doc->createElement('tvshow');
  385. $doc->setDocumentElement($show);
  386. my $elm;
  387. # Add data
  388. $elm = $doc->createElement('title');
  389. $elm->appendText($channel->{'title'});
  390. $show->appendChild($elm);
  391. $elm = $doc->createElement('premiered');
  392. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  393. $show->appendChild($elm);
  394. $elm = $doc->createElement('plot');
  395. $elm->appendText($channel->{'description'});
  396. $show->appendChild($elm);
  397. # Return the string
  398. return $doc->toString();
  399. }
  400. sub buildNFO($) {
  401. my ($video) = @_;
  402. # Create an XML tree
  403. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  404. $doc->setStandalone(1);
  405. my $show = $doc->createElement('episodedetails');
  406. $doc->setDocumentElement($show);
  407. my $file = $doc->createElement('fileinfo');
  408. $show->appendChild($file);
  409. my $stream = $doc->createElement('streamdetails');
  410. $file->appendChild($stream);
  411. my $stream_video = $doc->createElement('video');
  412. $stream->appendChild($stream_video);
  413. my $elm;
  414. # Add data
  415. $elm = $doc->createElement('season');
  416. $elm->appendText($video->{'season'});
  417. $show->appendChild($elm);
  418. $elm = $doc->createElement('episode');
  419. $elm->appendText($video->{'number'});
  420. $show->appendChild($elm);
  421. if (!defined($video->{'title'})) {
  422. $video->{'title'} = 'Episode ' . $video->{'number'};
  423. }
  424. $elm = $doc->createElement('title');
  425. $elm->appendText($video->{'title'});
  426. $show->appendChild($elm);
  427. if (!defined($video->{'date'})) {
  428. $video->{'date'} = time();
  429. }
  430. $elm = $doc->createElement('aired');
  431. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  432. $show->appendChild($elm);
  433. if (defined($video->{'description'})) {
  434. $elm = $doc->createElement('plot');
  435. $elm->appendText($video->{'description'});
  436. $show->appendChild($elm);
  437. }
  438. if (defined($video->{'duration'})) {
  439. $elm = $doc->createElement('runtime');
  440. $elm->appendText($video->{'duration'});
  441. $show->appendChild($elm);
  442. $elm = $doc->createElement('durationinseconds');
  443. $elm->appendText($video->{'duration'});
  444. $stream_video->appendChild($elm);
  445. }
  446. if (defined($video->{'rating'})) {
  447. $elm = $doc->createElement('rating');
  448. $elm->appendText($video->{'rating'});
  449. $show->appendChild($elm);
  450. }
  451. if (defined($video->{'creator'})) {
  452. $elm = $doc->createElement('director');
  453. $elm->appendText($video->{'creator'});
  454. $show->appendChild($elm);
  455. }
  456. # Return the string
  457. return $doc->toString();
  458. }
  459. sub findFiles() {
  460. my %files = ();
  461. # Allow complete bypass
  462. if ($NO_FILES) {
  463. return \%files;
  464. }
  465. # Read the output directory
  466. my $fh = undef();
  467. opendir($fh, '.')
  468. or die('Unable to open files directory: ' . $! . "\n");
  469. while (my $file = readdir($fh)) {
  470. my ($season, $num, $id, $suffix) = parseFilename($file);
  471. if (defined($id) && length($id) > 0) {
  472. my $path = $dir . '/' . $file;
  473. # Create the record as needed
  474. if (!exists($files{$id})) {
  475. my %tmp = (
  476. 'season' => $season,
  477. 'number' => $num,
  478. );
  479. $files{$id} = \%tmp;
  480. }
  481. # Video or NFO?
  482. my $type = 'path';
  483. if ($suffix eq 'nfo') {
  484. $type = 'nfo';
  485. }
  486. # Deal with duplicates
  487. if (exists($files{$id}->{$type})) {
  488. if ($type eq 'nfo') {
  489. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  490. if (!$NO_RENAME) {
  491. if ($SUDO_CHATTR) {
  492. system('sudo', 'chattr', '-i', $path);
  493. }
  494. unlink($path);
  495. next;
  496. }
  497. } else {
  498. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $path . "\n");
  499. if (!$NO_RENAME) {
  500. my $del = $path;
  501. if ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  502. $del = $files{$id}->{'path'};
  503. }
  504. if ($SUDO_CHATTR) {
  505. system('sudo', 'chattr', '-i', $del);
  506. }
  507. unlink($del);
  508. next;
  509. }
  510. }
  511. }
  512. # Assign the component paths
  513. $files{$id}->{$type} = $file;
  514. if ($type ne 'nfo') {
  515. $files{$id}->{'suffix'} = $suffix;
  516. }
  517. }
  518. }
  519. close($fh);
  520. if ($DEBUG) {
  521. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  522. if ($DEBUG > 1) {
  523. print STDERR prettyPrint(\%files, "\t") . "\n";
  524. }
  525. }
  526. return \%files;
  527. }
  528. sub fetchParse($$) {
  529. my ($name, $id) = @_;
  530. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  531. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  532. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  533. }
  534. # Fetch
  535. if ($DEBUG) {
  536. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  537. }
  538. sleep($DELAY);
  539. my $content = get($url);
  540. if (!defined($content) || length($content) < 10) {
  541. die('Invalid content from URL: ' . $url . "\n");
  542. }
  543. # Parse
  544. my $data = decode_json($content);
  545. if (!defined($data) || ref($data) ne 'HASH') {
  546. die('Invalid JSON: ' . $content . "\n");
  547. }
  548. if ($DEBUG > 2) {
  549. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  550. }
  551. return $data;
  552. }
  553. sub getSubscriptions($$) {
  554. my ($user, $id) = @_;
  555. my $index = 1;
  556. my $itemCount = undef();
  557. my %subs = ();
  558. SUBS_LOOP:
  559. {
  560. # Build, fetch, parse
  561. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  562. my $data = fetchParse('subscriptions', $id);
  563. # It's all in the feed
  564. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  565. die("Invalid subscription data\n");
  566. }
  567. $data = $data->{'feed'};
  568. # Grab the total count, so we know when to stop
  569. if (!defined($itemCount)) {
  570. if ( !exists($data->{'openSearch$totalResults'})
  571. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  572. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  573. {
  574. die("Invalid subscription feed metadata\n");
  575. }
  576. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  577. }
  578. # Process each item
  579. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  580. die("Invalid subscription entries\n");
  581. }
  582. my $items = $data->{'entry'};
  583. foreach my $item (@{$items}) {
  584. if ( ref($item) ne 'HASH'
  585. || !exists($item->{'yt$username'})
  586. || ref($item->{'yt$username'}) ne 'HASH'
  587. || !exists($item->{'yt$username'}->{'$t'}))
  588. {
  589. next;
  590. }
  591. if ($DEBUG) {
  592. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  593. }
  594. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  595. }
  596. # Loop if there are results left to fetch
  597. $index += $BATCH_SIZE;
  598. if (defined($itemCount) && $itemCount >= $index) {
  599. # But don't go past the max supported index
  600. if ($index <= $MAX_INDEX) {
  601. redo SUBS_LOOP;
  602. }
  603. }
  604. }
  605. # Return the list of subscribed usernames
  606. return \%subs;
  607. }
  608. sub saveSubscriptions($$) {
  609. my ($folder, $subs) = @_;
  610. # Check for local subscriptions missing from YT
  611. my %locals = ();
  612. my $fh = undef();
  613. opendir($fh, '.')
  614. or die('Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  615. while (my $file = readdir($fh)) {
  616. # Skip dotfiles
  617. if ($file =~ /^\./) {
  618. next;
  619. }
  620. # Skip non-directories
  621. if (!-d $file) {
  622. next;
  623. }
  624. # YT has some trouble with case
  625. my $lcFile = lc($file);
  626. # Anything else should be in the list
  627. if (!$subs->{$file} && !$subs->{$lcFile}) {
  628. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  629. }
  630. # Note local subscriptions
  631. $locals{$lcFile} = 1;
  632. }
  633. closedir($fh);
  634. # Check for YT subscriptions missing locally
  635. foreach my $sub (keys(%{$subs})) {
  636. if (!exists($locals{ lc($sub) })) {
  637. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  638. mkdir($folder . '/' . $sub);
  639. }
  640. }
  641. }
  642. sub saveChannel($) {
  643. my ($channel) = @_;
  644. my $nfo = 'tvshow.nfo';
  645. if (!-e $nfo) {
  646. if ($DEBUG) {
  647. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  648. }
  649. # Save the poster
  650. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  651. my $jpg = get($channel->{'thumbnail'});
  652. saveString('poster.jpg', $jpg);
  653. }
  654. # Save the series NFO
  655. my $xml = buildSeriesNFO($channel);
  656. if ($DEBUG > 1) {
  657. print STDERR 'Saving NFO: ' . $xml . "\n";
  658. }
  659. saveString($nfo, $xml);
  660. }
  661. }
  662. sub getChannel($) {
  663. my ($user) = @_;
  664. # Build, fetch, parse
  665. my $data = fetchParse('channel', $user);
  666. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  667. die("Invalid channel data\n");
  668. }
  669. $data = $data->{'entry'};
  670. # Extract the data we want
  671. my %channel = (
  672. 'id' => $data->{'yt$channelId'}->{'$t'},
  673. 'title' => $data->{'title'}->{'$t'},
  674. 'date' => str2time($data->{'published'}->{'$t'}),
  675. 'description' => $data->{'summary'}->{'$t'},
  676. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  677. );
  678. return \%channel;
  679. }
  680. sub readExcludes() {
  681. my %excludes = ();
  682. # Read and parse the excludes videos file, if it exists
  683. if (-e $EXCLUDES_FILE) {
  684. my $fh;
  685. open($fh, $EXCLUDES_FILE)
  686. or die('Unable to open excludes videos file: ' . $! . "\n");
  687. while (<$fh>) {
  688. # Skip blank lines and comments
  689. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  690. next;
  691. }
  692. # Match our specific format or whine
  693. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  694. if ($DEBUG > 1) {
  695. print STDERR 'Adding exclude video: ' . $1 . "\n";
  696. }
  697. $excludes{$1} = 1;
  698. } else {
  699. print STDERR 'Skipped exclude video line: ' . $_;
  700. }
  701. }
  702. close($fh);
  703. }
  704. return \%excludes;
  705. }
  706. sub readExtras() {
  707. my %extras = ();
  708. # Read and parse the extra videos file, if it exists
  709. if (-e $EXTRAS_FILE) {
  710. my $fh;
  711. open($fh, $EXTRAS_FILE)
  712. or die('Unable to open extra videos file: ' . $! . "\n");
  713. while (<$fh>) {
  714. # Skip blank lines and comments
  715. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  716. next;
  717. }
  718. # Match our specific format or whine
  719. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  720. if ($DEBUG > 1) {
  721. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  722. }
  723. $extras{$2} = $1;
  724. } else {
  725. print STDERR 'Skipped extra video line: ' . $_;
  726. }
  727. }
  728. close($fh);
  729. }
  730. return \%extras;
  731. }
  732. sub parseVideoData($) {
  733. my ($data) = @_;
  734. my %video = (
  735. 'title' => $data->{'title'},
  736. 'date' => str2time($data->{'uploaded'}),
  737. 'description' => $data->{'description'},
  738. 'duration' => $data->{'duration'},
  739. 'rating' => $data->{'rating'},
  740. 'creator' => $data->{'uploader'},
  741. );
  742. $video{'season'} = time2str('%Y', $video{'date'});
  743. return \%video;
  744. }
  745. sub findVideo($) {
  746. my ($id) = @_;
  747. # Build, fetch, parse
  748. my $data = fetchParse('video', $id);
  749. # Validate
  750. if (!exists($data->{'data'})) {
  751. die("Invalid video list\n");
  752. }
  753. $data = $data->{'data'};
  754. # Parse the video data
  755. return parseVideoData($data);
  756. }
  757. sub dropExcludes($) {
  758. my ($videos) = @_;
  759. my $excludes = readExcludes();
  760. foreach my $id (keys(%{$excludes})) {
  761. if (!exists($videos->{$id})) {
  762. if ($DEBUG > 1) {
  763. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  764. }
  765. next;
  766. }
  767. delete($videos->{$id});
  768. }
  769. return $excludes;
  770. }
  771. sub addExtras($) {
  772. my ($videos) = @_;
  773. my $extras = readExtras();
  774. foreach my $id (keys(%{$extras})) {
  775. if (exists($videos->{$id})) {
  776. if ($DEBUG > 1) {
  777. print STDERR 'Skipping known extra video: ' . $id . "\n";
  778. }
  779. next;
  780. }
  781. my $video = findVideo($id);
  782. $videos->{$id} = $video;
  783. }
  784. return $extras;
  785. }
  786. sub findVideos($) {
  787. my ($user) = @_;
  788. my %videos = ();
  789. # Allow complete bypass
  790. if ($NO_SEARCH) {
  791. return \%videos;
  792. }
  793. # Loop through until we have all the entries
  794. my $index = 1;
  795. my $itemCount = undef();
  796. LOOP:
  797. {
  798. # Build, fetch, parse
  799. $API{'search'}{'params'}{'start-index'} = $index;
  800. my $data = fetchParse('search', $user);
  801. # Grab the total count, so we know when to stop
  802. if (!exists($data->{'data'})) {
  803. die("Invalid video list\n");
  804. }
  805. $data = $data->{'data'};
  806. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  807. $itemCount = $data->{'totalItems'};
  808. }
  809. # Process each item
  810. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  811. die("Invalid video list\n");
  812. }
  813. my $items = $data->{'items'};
  814. my $offset = 0;
  815. foreach my $item (@{$items}) {
  816. my $video = parseVideoData($item);
  817. $videos{ $item->{'id'} } = $video;
  818. $offset++;
  819. }
  820. # Loop if there are results left to fetch
  821. $index += $BATCH_SIZE;
  822. if (defined($itemCount) && $itemCount >= $index) {
  823. # But don't go past the max supported index
  824. if ($index <= $MAX_INDEX) {
  825. redo LOOP;
  826. }
  827. }
  828. }
  829. if ($DEBUG) {
  830. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  831. if ($DEBUG > 1) {
  832. print STDERR prettyPrint(\%videos, "\t") . "\n";
  833. }
  834. }
  835. return \%videos;
  836. }
  837. sub updateNFOData($$$) {
  838. my ($file, $season, $episode) = @_;
  839. my $nfoData = undef();
  840. {
  841. open(my $fh, '<', $file)
  842. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  843. $/ = undef;
  844. $nfoData = <$fh>;
  845. close($fh);
  846. }
  847. if ($nfoData) {
  848. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  849. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  850. }
  851. return $nfoData;
  852. }
  853. sub videoSE($$) {
  854. my ($season, $episode) = @_;
  855. return sprintf('S%02dE%02d - ', $season, $episode);
  856. }
  857. sub videoPath($$$$) {
  858. my ($season, $episode, $id, $ext) = @_;
  859. return videoSE($season, $episode) . $id . '.' . $ext;
  860. }
  861. sub renameVideo($$$$$$) {
  862. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  863. # Bail if disabled
  864. if ($NO_RENAME) {
  865. print STDERR 'Not renaming: ' . $video . "\n";
  866. return;
  867. }
  868. # General sanity checks
  869. if (!defined($video) || !defined($suffix) || !defined($nfo) || !defined($id) || !defined($season) || !defined($episode)) {
  870. die("Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  871. }
  872. # Build a new path
  873. my $videoNew = videoPath($season, $episode, $id, $suffix);
  874. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  875. # Parse old NFO data
  876. my $nfoData = updateNFOData($nfo, $season, $episode)
  877. or die('No NFO data found, refusing to rename: ' . $id . "\n");
  878. # Specific sanity checks, as we do dangerous work here
  879. if (!-r $video || !-r $nfo) {
  880. die('Invalid video or NFO sources in rename: ' . $video . '/' . $nfo);
  881. }
  882. # Rename video
  883. if ($video ne $videoNew) {
  884. print STDERR 'Renaming ' . $video . ' => ' . $videoNew . "\n";
  885. if (-e $videoNew) {
  886. die('Rename: Target exists: ' . $videoNew . "\n");
  887. }
  888. if ($SUDO_CHATTR) {
  889. system('sudo', 'chattr', '-i', $video);
  890. }
  891. rename($video, $videoNew)
  892. or die('Unable to rename: ' . $video . ': ' . $! . "\n");
  893. }
  894. # Write a new NFO and unlink the old one
  895. if ($SUDO_CHATTR) {
  896. system('sudo', 'chattr', '-i', $nfo);
  897. }
  898. saveString($nfoNew, $nfoData);
  899. if ($nfo ne $nfoNew) {
  900. print STDERR 'Renaming ' . $nfo . ' => ' . $nfoNew . "\n";
  901. unlink($nfo)
  902. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  903. }
  904. }
  905. sub parseFilename($) {
  906. my ($file) = @_;
  907. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  908. }