subscribe.pl 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::Simple;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'kj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'hfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my $BATCH_SIZE = 50;
  25. my $MAX_INDEX = 25000;
  26. my $DRIFT_TOLERANCE = 2;
  27. my $DRIFT_FACTOR = 100.0;
  28. my $DELAY = 5;
  29. my $API_URL = 'https://gdata.youtube.com/feeds/api/';
  30. my %API = (
  31. 'search' => {
  32. 'prefix' => $API_URL . 'users/',
  33. 'suffix' => '/uploads',
  34. 'params' => {
  35. 'start-index' => 1,
  36. 'max-results' => 1,
  37. 'strict' => 1,
  38. 'v' => 2,
  39. 'alt' => 'jsonc',
  40. },
  41. },
  42. 'subscriptions' => {
  43. 'prefix' => $API_URL . 'users/',
  44. 'suffix' => '/subscriptions',
  45. 'params' => {
  46. 'start-index' => 1,
  47. 'max-results' => 1,
  48. 'strict' => 1,
  49. 'v' => 2,
  50. 'alt' => 'json',
  51. },
  52. },
  53. 'video' => {
  54. 'prefix' => $API_URL . 'videos/',
  55. 'suffix' => '',
  56. 'params' => {
  57. 'strict' => 1,
  58. 'v' => 2,
  59. 'alt' => 'jsonc'
  60. },
  61. },
  62. 'channel' => {
  63. 'prefix' => $API_URL . 'users/',
  64. 'suffix' => '',
  65. 'params' => {
  66. 'strict' => 1,
  67. 'v' => 2,
  68. 'alt' => 'json'
  69. },
  70. },
  71. );
  72. # Prototypes
  73. sub findVideos($);
  74. sub findFiles();
  75. sub findVideo($);
  76. sub buildNFO($);
  77. sub buildSeriesNFO($);
  78. sub getSubscriptions($$);
  79. sub saveSubscriptions($$);
  80. sub saveChannel($);
  81. sub getChannel($);
  82. sub fetchParse($$);
  83. sub saveString($$);
  84. sub readExcludes();
  85. sub readExtras();
  86. sub parseVideoData($);
  87. sub dropExcludes($);
  88. sub addExtras($);
  89. sub updateNFOData($$$);
  90. sub videoSE($$);
  91. sub videoPath($$$$);
  92. sub renameVideo($$$$$);
  93. # Sanity check
  94. if (scalar(@ARGV) < 1) {
  95. die('Usage: ' . basename($0) . " output_directory\n");
  96. }
  97. # Command-line parameters
  98. my ($dir) = @ARGV;
  99. $dir =~ s/\/+$//;
  100. if (!-d $dir) {
  101. die('Invalid output directory: ' . $dir . "\n");
  102. }
  103. my $user = basename($dir);
  104. if (length($user) < 1 || !($user =~ /^\w+$/)) {
  105. die('Invalid user: ' . $user . "\n");
  106. }
  107. # Move to the target directory so we can use relative paths later
  108. chdir($dir);
  109. # Environmental parameters (debug)
  110. my $DEBUG = 0;
  111. if ($ENV{'DEBUG'}) {
  112. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  113. $DEBUG = $1;
  114. } else {
  115. $DEBUG = 1;
  116. }
  117. }
  118. my $NO_FETCH = 0;
  119. if ($ENV{'NO_FETCH'}) {
  120. $NO_FETCH = 1;
  121. }
  122. my $NO_NFO = 0;
  123. if ($ENV{'NO_NFO'}) {
  124. $NO_NFO = 1;
  125. }
  126. my $NO_SEARCH = 0;
  127. if ($ENV{'NO_SEARCH'}) {
  128. $NO_SEARCH = 1;
  129. }
  130. my $NO_FILES = 0;
  131. if ($ENV{'NO_FILES'}) {
  132. $NO_FILES = 1;
  133. }
  134. my $NO_CHANNEL = 0;
  135. if ($ENV{'NO_CHANNEL'}) {
  136. $NO_CHANNEL = 1;
  137. }
  138. my $NO_EXTRAS = 0;
  139. if ($ENV{'NO_EXTRAS'}) {
  140. $NO_EXTRAS = 1;
  141. }
  142. my $NO_EXCLUDES = 0;
  143. if ($ENV{'NO_EXCLUDES'}) {
  144. $NO_EXCLUDES = 1;
  145. }
  146. my $NO_RENAME = 0;
  147. if ($ENV{'NO_RDNAME'}) {
  148. $NO_RENAME = 1;
  149. }
  150. # Environmental parameters (functional)
  151. my $FORCE_RENAME = 0;
  152. if ($ENV{'FORCE_RENAME'}) {
  153. $FORCE_RENAME = 1;
  154. }
  155. my $SUDO_CHATTR = 1;
  156. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  157. $SUDO_CHATTR = 0;
  158. }
  159. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  160. $MAX_INDEX = $1;
  161. }
  162. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  163. $BATCH_SIZE = $1;
  164. }
  165. # Construct globals
  166. foreach my $key (keys(%API)) {
  167. if (exists($API{$key}{'params'}{'max-results'})) {
  168. $API{$key}{'params'}{'max-results'} = $BATCH_SIZE;
  169. }
  170. }
  171. if (!$DEBUG) {
  172. push(@YTDL_ARGS, @YTDL_QUIET);
  173. }
  174. # Allow use as a subscription manager
  175. if ($0 =~ /subscription/i) {
  176. my %subs = ();
  177. foreach my $user (keys(%USERS)) {
  178. my $tmp = getSubscriptions($user, $USERS{$user});
  179. foreach my $sub (keys(%{$tmp})) {
  180. if (exists($subs{$sub})) {
  181. $subs{$sub} .= ', ' . $tmp->{$sub};
  182. } else {
  183. $subs{$sub} = $tmp->{$sub};
  184. }
  185. }
  186. }
  187. saveSubscriptions($dir, \%subs);
  188. exit(0);
  189. }
  190. # Grab the channel data
  191. my $channel = {};
  192. if (!$NO_CHANNEL) {
  193. $channel = getChannel($user);
  194. saveChannel($channel);
  195. }
  196. # Find all the user's videos on YT
  197. my $videos = {};
  198. if (!$NO_SEARCH) {
  199. $videos = findVideos($user);
  200. }
  201. # Find any requested "extra" videos
  202. my $extras = {};
  203. if (!$NO_EXTRAS) {
  204. $extras = addExtras($videos);
  205. }
  206. # Drop any "excludes" videos
  207. my $excludes = {};
  208. if (!$NO_EXCLUDES) {
  209. $excludes = dropExcludes($videos);
  210. }
  211. # Calculate the episode number using the publish dates
  212. {
  213. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  214. my $num = undef();
  215. my $lastSeason = undef();
  216. foreach my $id (@byDate) {
  217. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  218. $num = 1;
  219. $lastSeason = $videos->{$id}->{'season'};
  220. }
  221. $videos->{$id}->{'number'} = $num++;
  222. }
  223. }
  224. # Find all existing YT files on disk
  225. my $files = {};
  226. if (!$NO_FILES) {
  227. $files = findFiles();
  228. }
  229. # Whine about unknown videos
  230. foreach my $id (keys(%{$files})) {
  231. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  232. print STDERR 'Local video not known to YT channel (' . $user . '): ' . $id . "\n";
  233. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  234. }
  235. }
  236. # Fill in missing videos and NFOs
  237. foreach my $id (keys(%{$videos})) {
  238. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  239. # Warn (and optionally rename) if the video numbers drift
  240. if (
  241. exists($files->{$id})
  242. && ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  243. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  244. )
  245. {
  246. if ($FORCE_RENAME) {
  247. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  248. } else {
  249. # Find the old NFO to avoid re-fetching
  250. $nfo = $files->{$id}->{'nfo'};
  251. # Calculate the drift magnitude
  252. my $delta = abs($files->{$id}->{'number'} - $videos->{$id}->{'number'});
  253. my $tolerance = $files->{$id}->{'number'} / $DRIFT_FACTOR;
  254. if ($tolerance < $DRIFT_TOLERANCE) {
  255. $tolerance = $DRIFT_TOLERANCE;
  256. }
  257. # Ignore small changes
  258. my $rename = 0;
  259. if ($FORCE_RENAME || $delta > $tolerance) {
  260. $rename = 1;
  261. }
  262. # Execute
  263. if ($DEBUG || $rename) {
  264. print STDERR 'Video ' . $id . ' had video number ' . $files->{$id}->{'number'} . ' but now has video number ' . $videos->{$id}->{'number'} . "\n";
  265. if ($rename) {
  266. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  267. }
  268. }
  269. }
  270. }
  271. # If we haven't heard of the file, or don't have an NFO for it
  272. # Checking for the NFO allows use to resume failed downloads
  273. if (!exists($files->{$id}) || !-e $nfo) {
  274. if ($DEBUG) {
  275. print STDERR 'Fetching video: ' . $id . "\n";
  276. }
  277. # Let youtube-dl handle the URLs and downloading
  278. {
  279. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  280. my @name = ($YTDL_BIN);
  281. push(@name, @YTDL_ARGS);
  282. my @fetch = @name;
  283. push(@name, '--get-filename');
  284. push(@fetch, @args);
  285. push(@name, @args);
  286. if ($NO_FETCH) {
  287. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  288. } else {
  289. # Find the output file name
  290. if ($DEBUG > 1) {
  291. print STDERR join(' ', @name) . "\n";
  292. }
  293. sleep($DELAY);
  294. my $file = capture(EXIT_ANY, @name);
  295. if ($EXITVAL != 0) {
  296. warn('Error executing youtube-dl (name): ' . $EXITVAL . "\n");
  297. next;
  298. }
  299. $file =~ s/^\s+//;
  300. $file =~ s/\s+$//;
  301. # Sanity check
  302. if (!$file) {
  303. warn('No file name available for video: ' . $id . "\n");
  304. next;
  305. }
  306. # Download
  307. if ($DEBUG > 1) {
  308. print STDERR join(' ', @fetch) . "\n";
  309. }
  310. sleep($DELAY);
  311. my $exit = run(EXIT_ANY, @fetch);
  312. if ($exit != 0) {
  313. warn('Error executing youtube-dl (fetch): ' . $exit . "\n");
  314. next;
  315. }
  316. # Ensure we found something useful
  317. if (-e $file . '.part') {
  318. warn('Partial download detected: ' . $file . "\n");
  319. next;
  320. }
  321. if (!-s $file) {
  322. warn('No output video file: ' . $file . "\n");
  323. next;
  324. }
  325. # Touch the file to reflect the download time rather than the upload time
  326. touch($file);
  327. }
  328. }
  329. # Build and save the XML document
  330. my $xml = buildNFO($videos->{$id});
  331. if ($DEBUG > 1) {
  332. print STDERR 'Saving NFO: ' . $xml . "\n";
  333. }
  334. if ($NO_NFO) {
  335. print STDERR "Not saving NFO\n";
  336. } else {
  337. saveString($nfo, $xml);
  338. }
  339. }
  340. }
  341. sub saveString($$) {
  342. my ($path, $str) = @_;
  343. my $fh = undef();
  344. if (!open($fh, '>', $path)) {
  345. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  346. return undef();
  347. }
  348. print $fh $str;
  349. close($fh);
  350. return 1;
  351. }
  352. sub buildSeriesNFO($) {
  353. my ($channel) = @_;
  354. # Create an XML tree
  355. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  356. $doc->setStandalone(1);
  357. my $show = $doc->createElement('tvshow');
  358. $doc->setDocumentElement($show);
  359. my $elm;
  360. # Add data
  361. $elm = $doc->createElement('title');
  362. $elm->appendText($channel->{'title'});
  363. $show->appendChild($elm);
  364. $elm = $doc->createElement('premiered');
  365. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  366. $show->appendChild($elm);
  367. $elm = $doc->createElement('plot');
  368. $elm->appendText($channel->{'description'});
  369. $show->appendChild($elm);
  370. # Return the string
  371. return $doc->toString();
  372. }
  373. sub buildNFO($) {
  374. my ($video) = @_;
  375. # Create an XML tree
  376. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  377. $doc->setStandalone(1);
  378. my $show = $doc->createElement('episodedetails');
  379. $doc->setDocumentElement($show);
  380. my $elm;
  381. # Add data
  382. $elm = $doc->createElement('season');
  383. $elm->appendText($video->{'season'});
  384. $show->appendChild($elm);
  385. $elm = $doc->createElement('episode');
  386. $elm->appendText($video->{'number'});
  387. $show->appendChild($elm);
  388. if (!defined($video->{'title'})) {
  389. $video->{'title'} = 'Episode ' . $video->{'number'};
  390. }
  391. $elm = $doc->createElement('title');
  392. $elm->appendText($video->{'title'});
  393. $show->appendChild($elm);
  394. if (!defined($video->{'date'})) {
  395. $video->{'date'} = time();
  396. }
  397. $elm = $doc->createElement('aired');
  398. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  399. $show->appendChild($elm);
  400. if (defined($video->{'description'})) {
  401. $elm = $doc->createElement('plot');
  402. $elm->appendText($video->{'description'});
  403. $show->appendChild($elm);
  404. }
  405. if (defined($video->{'duration'})) {
  406. $elm = $doc->createElement('runtime');
  407. $elm->appendText($video->{'duration'});
  408. $show->appendChild($elm);
  409. }
  410. if (defined($video->{'rating'})) {
  411. $elm = $doc->createElement('rating');
  412. $elm->appendText($video->{'rating'});
  413. $show->appendChild($elm);
  414. }
  415. if (defined($video->{'creator'})) {
  416. $elm = $doc->createElement('director');
  417. $elm->appendText($video->{'creator'});
  418. $show->appendChild($elm);
  419. }
  420. # Return the string
  421. return $doc->toString();
  422. }
  423. sub findFiles() {
  424. my %files = ();
  425. # Allow complete bypass
  426. if ($NO_FILES) {
  427. return \%files;
  428. }
  429. # Read the output directory
  430. my $fh = undef();
  431. opendir($fh, '.')
  432. or die('Unable to open files directory: ' . $! . "\n");
  433. while (my $file = readdir($fh)) {
  434. my ($season, $num, $id, $suffix) = $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  435. if (defined($id) && length($id) > 0) {
  436. if ($suffix eq 'nfo') {
  437. next;
  438. }
  439. my %tmp = (
  440. 'season' => $season,
  441. 'number' => $num,
  442. 'suffix' => $suffix,
  443. 'path' => $file,
  444. );
  445. $tmp{'nfo'} = $tmp{'path'};
  446. $tmp{'nfo'} =~ s/\.\w\w\w$/\.nfo/;
  447. if (exists($files{$id})) {
  448. warn('Duplicate ID: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $tmp{'path'} . "\n");
  449. if (!$NO_RENAME) {
  450. # Prefer to delete MP4 files, if the suffixes differ
  451. my $del = $files{$id};
  452. if ($tmp{'suffix'} eq 'mp4' && $del->{'suffix'} ne 'mp4') {
  453. $del = \%tmp;
  454. %tmp = %{ $files{$id} };
  455. }
  456. print STDERR 'Deleting duplicate: ' . $del->{'path'} . "\n";
  457. unlink($del->{'path'});
  458. if ($del->{'nfo'} ne $tmp{'nfo'}) {
  459. unlink($del->{'nfo'});
  460. }
  461. }
  462. }
  463. $files{$id} = \%tmp;
  464. }
  465. }
  466. close($fh);
  467. if ($DEBUG) {
  468. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  469. if ($DEBUG > 1) {
  470. print STDERR prettyPrint(\%files, "\t") . "\n";
  471. }
  472. }
  473. return \%files;
  474. }
  475. sub fetchParse($$) {
  476. my ($name, $id) = @_;
  477. my $url = $API{$name}{'prefix'} . $id . $API{$name}{'suffix'} . '?';
  478. foreach my $key (keys(%{ $API{$name}{'params'} })) {
  479. $url .= '&' . uri_escape($key) . '=' . uri_escape($API{$name}{'params'}{$key});
  480. }
  481. # Fetch
  482. if ($DEBUG) {
  483. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  484. }
  485. sleep($DELAY);
  486. my $content = get($url);
  487. if (!defined($content) || length($content) < 10) {
  488. die('Invalid content from URL: ' . $url . "\n");
  489. }
  490. # Parse
  491. my $data = decode_json($content);
  492. if (!defined($data) || ref($data) ne 'HASH') {
  493. die('Invalid JSON: ' . $content . "\n");
  494. }
  495. if ($DEBUG > 2) {
  496. print STDERR "Raw JSON data:\n" . prettyPrint($data, ' ') . "\n";
  497. }
  498. return $data;
  499. }
  500. sub getSubscriptions($$) {
  501. my ($user, $id) = @_;
  502. my $index = 1;
  503. my $itemCount = undef();
  504. my %subs = ();
  505. SUBS_LOOP:
  506. {
  507. # Build, fetch, parse
  508. $API{'subscriptions'}{'params'}{'start-index'} = $index;
  509. my $data = fetchParse('subscriptions', $id);
  510. # It's all in the feed
  511. if (!exists($data->{'feed'}) || ref($data->{'feed'}) ne 'HASH') {
  512. die("Invalid subscription data\n");
  513. }
  514. $data = $data->{'feed'};
  515. # Grab the total count, so we know when to stop
  516. if (!defined($itemCount)) {
  517. if ( !exists($data->{'openSearch$totalResults'})
  518. || ref($data->{'openSearch$totalResults'}) ne 'HASH'
  519. || !exists($data->{'openSearch$totalResults'}->{'$t'}))
  520. {
  521. die("Invalid subscription feed metadata\n");
  522. }
  523. $itemCount = $data->{'openSearch$totalResults'}->{'$t'};
  524. }
  525. # Process each item
  526. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'ARRAY') {
  527. die("Invalid subscription entries\n");
  528. }
  529. my $items = $data->{'entry'};
  530. foreach my $item (@{$items}) {
  531. if ( ref($item) ne 'HASH'
  532. || !exists($item->{'yt$username'})
  533. || ref($item->{'yt$username'}) ne 'HASH'
  534. || !exists($item->{'yt$username'}->{'$t'}))
  535. {
  536. next;
  537. }
  538. if ($DEBUG) {
  539. print STDERR prettyPrint($item->{'yt$username'}, "\t") . "\n";
  540. }
  541. $subs{ $item->{'yt$username'}->{'$t'} } = $user;
  542. }
  543. # Loop if there are results left to fetch
  544. $index += $BATCH_SIZE;
  545. if (defined($itemCount) && $itemCount >= $index) {
  546. # But don't go past the max supported index
  547. if ($index <= $MAX_INDEX) {
  548. redo SUBS_LOOP;
  549. }
  550. }
  551. }
  552. # Return the list of subscribed usernames
  553. return \%subs;
  554. }
  555. sub saveSubscriptions($$) {
  556. my ($folder, $subs) = @_;
  557. # Check for local subscriptions missing from YT
  558. my %locals = ();
  559. my $fh = undef();
  560. opendir($fh, '.')
  561. or die('Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  562. while (my $file = readdir($fh)) {
  563. # Skip dotfiles
  564. if ($file =~ /^\./) {
  565. next;
  566. }
  567. # Skip non-directories
  568. if (!-d $file) {
  569. next;
  570. }
  571. # YT has some trouble with case
  572. my $lcFile = lc($file);
  573. # Anything else should be in the list
  574. if (!$subs->{$file} && !$subs->{$lcFile}) {
  575. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  576. }
  577. # Note local subscriptions
  578. $locals{$lcFile} = 1;
  579. }
  580. closedir($fh);
  581. # Check for YT subscriptions missing locally
  582. foreach my $sub (keys(%{$subs})) {
  583. if (!exists($locals{ lc($sub) })) {
  584. print STDERR 'Adding local subscription for: ' . $sub . ' (' . $subs->{$sub} . ")\n";
  585. mkdir($folder . '/' . $sub);
  586. }
  587. }
  588. }
  589. sub saveChannel($) {
  590. my ($channel) = @_;
  591. my $nfo = 'tvshow.nfo';
  592. if (!-e $nfo) {
  593. if ($DEBUG) {
  594. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  595. }
  596. # Save the poster
  597. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  598. my $jpg = get($channel->{'thumbnail'});
  599. saveString('poster.jpg', $jpg);
  600. }
  601. # Save the series NFO
  602. my $xml = buildSeriesNFO($channel);
  603. if ($DEBUG > 1) {
  604. print STDERR 'Saving NFO: ' . $xml . "\n";
  605. }
  606. saveString($nfo, $xml);
  607. }
  608. }
  609. sub getChannel($) {
  610. my ($user) = @_;
  611. # Build, fetch, parse
  612. my $data = fetchParse('channel', $user);
  613. if (!exists($data->{'entry'}) || ref($data->{'entry'}) ne 'HASH') {
  614. die("Invalid channel data\n");
  615. }
  616. $data = $data->{'entry'};
  617. # Extract the data we want
  618. my %channel = (
  619. 'id' => $data->{'yt$channelId'}->{'$t'},
  620. 'title' => $data->{'title'}->{'$t'},
  621. 'date' => str2time($data->{'published'}->{'$t'}),
  622. 'description' => $data->{'summary'}->{'$t'},
  623. 'thumbnail' => $data->{'media$thumbnail'}->{'url'},
  624. );
  625. return \%channel;
  626. }
  627. sub readExcludes() {
  628. my %excludes = ();
  629. # Read and parse the excludes videos file, if it exists
  630. if (-e $EXCLUDES_FILE) {
  631. my $fh;
  632. open($fh, $EXCLUDES_FILE)
  633. or die('Unable to open excludes videos file: ' . $! . "\n");
  634. while (<$fh>) {
  635. # Skip blank lines and comments
  636. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  637. next;
  638. }
  639. # Match our specific format or whine
  640. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  641. if ($DEBUG > 1) {
  642. print STDERR 'Adding exclude video: ' . $1 . "\n";
  643. }
  644. $excludes{$1} = 1;
  645. } else {
  646. print STDERR 'Skipped exclude video line: ' . $_;
  647. }
  648. }
  649. close($fh);
  650. }
  651. return \%excludes;
  652. }
  653. sub readExtras() {
  654. my %extras = ();
  655. # Read and parse the extra videos file, if it exists
  656. if (-e $EXTRAS_FILE) {
  657. my $fh;
  658. open($fh, $EXTRAS_FILE)
  659. or die('Unable to open extra videos file: ' . $! . "\n");
  660. while (<$fh>) {
  661. # Skip blank lines and comments
  662. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  663. next;
  664. }
  665. # Match our specific format or whine
  666. if ($_ =~ /^\s*(\d+)\s*[=:>]+\s*([\w\-]+)\s*$/) {
  667. if ($DEBUG > 1) {
  668. print STDERR 'Adding extra video: ' . $1 . ' => ' . $2 . "\n";
  669. }
  670. $extras{$2} = $1;
  671. } else {
  672. print STDERR 'Skipped extra video line: ' . $_;
  673. }
  674. }
  675. close($fh);
  676. }
  677. return \%extras;
  678. }
  679. sub parseVideoData($) {
  680. my ($data) = @_;
  681. my %video = (
  682. 'title' => $data->{'title'},
  683. 'date' => str2time($data->{'uploaded'}),
  684. 'description' => $data->{'description'},
  685. 'duration' => $data->{'duration'},
  686. 'rating' => $data->{'rating'},
  687. 'creator' => $data->{'uploader'},
  688. );
  689. $video{'season'} = time2str('%Y', $video{'date'});
  690. return \%video;
  691. }
  692. sub findVideo($) {
  693. my ($id) = @_;
  694. # Build, fetch, parse
  695. my $data = fetchParse('video', $id);
  696. # Validate
  697. if (!exists($data->{'data'})) {
  698. die("Invalid video list\n");
  699. }
  700. $data = $data->{'data'};
  701. # Parse the video data
  702. return parseVideoData($data);
  703. }
  704. sub dropExcludes($) {
  705. my ($videos) = @_;
  706. my $excludes = readExcludes();
  707. foreach my $id (keys(%{$excludes})) {
  708. if (!exists($videos->{$id})) {
  709. if ($DEBUG > 1) {
  710. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  711. }
  712. next;
  713. }
  714. delete($videos->{$id});
  715. }
  716. return $excludes;
  717. }
  718. sub addExtras($) {
  719. my ($videos) = @_;
  720. my $extras = readExtras();
  721. foreach my $id (keys(%{$extras})) {
  722. if (exists($videos->{$id})) {
  723. if ($DEBUG > 1) {
  724. print STDERR 'Skipping known extra video: ' . $id . "\n";
  725. }
  726. next;
  727. }
  728. my $video = findVideo($id);
  729. $videos->{$id} = $video;
  730. }
  731. return $extras;
  732. }
  733. sub findVideos($) {
  734. my ($user) = @_;
  735. my %videos = ();
  736. # Allow complete bypass
  737. if ($NO_SEARCH) {
  738. return \%videos;
  739. }
  740. # Loop through until we have all the entries
  741. my $index = 1;
  742. my $itemCount = undef();
  743. LOOP:
  744. {
  745. # Build, fetch, parse
  746. $API{'search'}{'params'}{'start-index'} = $index;
  747. my $data = fetchParse('search', $user);
  748. # Grab the total count, so we know when to stop
  749. if (!exists($data->{'data'})) {
  750. die("Invalid video list\n");
  751. }
  752. $data = $data->{'data'};
  753. if (!defined($itemCount) && exists($data->{'totalItems'})) {
  754. $itemCount = $data->{'totalItems'};
  755. }
  756. # Process each item
  757. if (!exists($data->{'items'}) || ref($data->{'items'}) ne 'ARRAY') {
  758. die("Invalid video list\n");
  759. }
  760. my $items = $data->{'items'};
  761. my $offset = 0;
  762. foreach my $item (@{$items}) {
  763. my $video = parseVideoData($item);
  764. $videos{ $item->{'id'} } = $video;
  765. $offset++;
  766. }
  767. # Loop if there are results left to fetch
  768. $index += $BATCH_SIZE;
  769. if (defined($itemCount) && $itemCount >= $index) {
  770. # But don't go past the max supported index
  771. if ($index <= $MAX_INDEX) {
  772. redo LOOP;
  773. }
  774. }
  775. }
  776. if ($DEBUG) {
  777. print STDERR 'Found ' . scalar(keys(%videos)) . " remote videos\n";
  778. if ($DEBUG > 1) {
  779. print STDERR prettyPrint(\%videos, "\t") . "\n";
  780. }
  781. }
  782. return \%videos;
  783. }
  784. sub updateNFOData($$$) {
  785. my ($file, $season, $episode) = @_;
  786. my $nfoData = undef();
  787. {
  788. open(my $fh, '<', $file)
  789. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  790. $/ = undef;
  791. $nfoData = <$fh>;
  792. close($fh);
  793. }
  794. if ($nfoData) {
  795. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  796. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  797. }
  798. return $nfoData;
  799. }
  800. sub videoSE($$) {
  801. my ($season, $episode) = @_;
  802. return sprintf('S%02dE%02d - ', $season, $episode);
  803. }
  804. sub videoPath($$$$) {
  805. my ($season, $episode, $id, $ext) = @_;
  806. return videoSE($season, $episode) . $id . '.' . $ext;
  807. }
  808. sub renameVideo($$$$$) {
  809. my ($video, $suffix, $id, $season, $episode) = @_;
  810. # Bail if disabled
  811. if ($NO_RENAME) {
  812. print STDERR 'Not renaming: ' . $video . "\n";
  813. return;
  814. }
  815. # General sanity checks
  816. if (!defined($video) || !defined($suffix) || !defined($id) || !defined($season) || !defined($episode)) {
  817. die("Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  818. }
  819. # We could ask for this but there are already a lot of parameters
  820. my $nfo = $video;
  821. $nfo =~ s/\.${suffix}$/\.nfo/;
  822. # Build a new path
  823. my $videoNew = videoPath($season, $episode, $id, $suffix);
  824. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  825. print STDERR 'Renaming ' . $video . ' => ' . $videoNew . "\n";
  826. # Parse old NFO data
  827. my $nfoData = updateNFOData($nfo, $season, $episode)
  828. or die('No NFO data found, refusing to rename: ' . $id . "\n");
  829. # Specific sanity checks, as we do dangerous work here
  830. if (!-r $video || !-r $nfo) {
  831. die('Invalid video or NFO sources in rename: ' . $video . '/' . $nfo);
  832. }
  833. if (-e $videoNew) {
  834. die('Rename: Target exists: ' . $videoNew . "\n");
  835. }
  836. # Rename video
  837. if ($SUDO_CHATTR) {
  838. system('sudo', 'chattr', '-i', $video);
  839. }
  840. rename($video, $videoNew)
  841. or die('Unable to rename: ' . $video . ': ' . $! . "\n");
  842. # Write a new NFO and unlink the old one
  843. saveString($nfoNew, $nfoData);
  844. if ($SUDO_CHATTR) {
  845. system('sudo', 'chattr', '-i', $nfo);
  846. }
  847. unlink($nfo)
  848. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  849. }