subscribe.pl 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use FindBin;
  5. use lib "$FindBin::Bin/";
  6. use File::Touch;
  7. use File::Basename;
  8. use Date::Parse;
  9. use Date::Format;
  10. use LWP::UserAgent;
  11. use URI::Escape;
  12. use JSON;
  13. use XML::LibXML;
  14. use IPC::System::Simple qw( system run capture EXIT_ANY $EXITVAL );
  15. use IPC::Cmd qw( can_run );
  16. use PrettyPrint;
  17. # Paramters
  18. my %USERS = ('profplump' => 'UCkj-Ob6eYHvzo-P0UWfnQzA', 'shanda' => 'UChfwMHzkPXOOFDce5hyQkTA');
  19. my $EXTRAS_FILE = 'extra_videos.ini';
  20. my $EXCLUDES_FILE = 'exclude_videos.ini';
  21. my $YTDL_BIN = $ENV{'HOME'} . '/bin/video/yt/youtube-dl';
  22. my @YTDL_ARGS = ('--force-ipv4', '--socket-timeout', '10', '--no-playlist', '--max-downloads', '1', '--age-limit', '99');
  23. my @YTDL_QUIET = ('--quiet', '--no-warnings');
  24. my @YTDL_DEBUG = ('--verbose');
  25. my $BATCH_SIZE = 50;
  26. my $MAX_INDEX = 25000;
  27. my $FETCH_LIMIT = 50;
  28. my $DELAY = 0.5;
  29. my $HTTP_TIMEOUT = 10;
  30. my $HTTP_UA = 'ZachBot/1.0 (Plex)';
  31. my $HTTP_VERIFY = 0;
  32. my $API_URL = 'https://www.googleapis.com/youtube/v3/';
  33. my $API_KEY = $ENV{'YT_API_KEY'};
  34. my $API_ST_REL = 0.50;
  35. my $API_ST_ABS = 5;
  36. my %API = (
  37. #https://www.googleapis.com/youtube/v3/channels?part=id&forUsername={channelName}&key={YOUR_API_KEY}
  38. 'channelID' => {
  39. 'url' => $API_URL . 'channels',
  40. 'params' => {
  41. 'part' => 'id',
  42. 'key' => $API_KEY,
  43. },
  44. },
  45. #https://www.googleapis.com/youtube/v3/channels?part=snippet&id={channelID}&key={YOUR_API_KEY}
  46. 'channel' => {
  47. 'url' => $API_URL . 'channels',
  48. 'params' => {
  49. 'part' => 'snippet',
  50. 'key' => $API_KEY,
  51. },
  52. },
  53. #https://www.googleapis.com/youtube/v3/search?part=snippet&channelId={channelID}&maxResults={maxResults}&pageToken={foo}&safeSearch=none&type=video&key={YOUR_API_KEY}
  54. 'search' => {
  55. 'url' => $API_URL . 'search',
  56. 'params' => {
  57. 'part' => 'id',
  58. 'key' => $API_KEY,
  59. 'maxResults' => 5,
  60. 'safeSearch' => 'none',
  61. 'type' => 'video',
  62. 'order' => 'date',
  63. },
  64. },
  65. #https://www.googleapis.com/youtube/v3/videos?part=snippet&id={videoID}&key={YOUR_API_KEY}
  66. 'video' => {
  67. 'url' => $API_URL . 'videos',
  68. 'params' => {
  69. 'part' => 'snippet,contentDetails',
  70. 'key' => $API_KEY,
  71. },
  72. },
  73. #https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&channelId={channelID}&key={YOUR_API_KEY}
  74. 'subscriptions' => {
  75. 'url' => $API_URL . 'subscriptions',
  76. 'params' => {
  77. 'part' => 'snippet',
  78. 'key' => $API_KEY,
  79. 'maxResults' => 5,
  80. },
  81. },
  82. );
  83. # Prototypes
  84. sub getVideoData($);
  85. sub findVideos($);
  86. sub findFiles($);
  87. sub buildNFO($);
  88. sub buildSeriesNFO($);
  89. sub getSubscriptions($$);
  90. sub saveSubscriptions($$);
  91. sub saveChannel($);
  92. sub getChannel($);
  93. sub fetchParse($$);
  94. sub saveString($$);
  95. sub readExcludes();
  96. sub readExtras();
  97. sub parseVideoData($);
  98. sub dropExcludes($);
  99. sub addExtras($);
  100. sub updateNFOData($$$);
  101. sub videoNumberStr($$);
  102. sub videoSE($$);
  103. sub videoPath($$$$);
  104. sub renameVideo($$$$$$);
  105. sub parseFilename($);
  106. sub fetch($$);
  107. # Sanity check
  108. if (scalar(@ARGV) < 1) {
  109. die('Usage: ' . basename($0) . " output_directory\n");
  110. }
  111. my $SUBSCRIPTIONS = 0;
  112. if ($0 =~ /subscription/i) {
  113. $SUBSCRIPTIONS = 1;
  114. }
  115. # Command-line parameters
  116. our ($DIR) = @ARGV;
  117. $DIR =~ s/\/+$//;
  118. if (!-d $DIR) {
  119. die('Invalid output directory: ' . $DIR . "\n");
  120. }
  121. our $NAME = basename($DIR);
  122. our $ID = $NAME;
  123. if ($NAME =~ /^(.*)\s\(([\w\-]+)\)$/) {
  124. $NAME = $1;
  125. $ID = $2;
  126. if (length($ID) < 1 || !($ID =~ /^[\w\-]+$/)) {
  127. die('Invalid channel ID: ' . $ID . "\n");
  128. }
  129. } elsif (!$SUBSCRIPTIONS) {
  130. die('Invalid folder: ' . $DIR . "\n");
  131. }
  132. # Move to the target directory so we can use relative paths later
  133. chdir($DIR) or die('Unable to chdir to: ' . $DIR . "\n");
  134. # Environmental parameters (debug)
  135. my $DEBUG = 0;
  136. if ($ENV{'DEBUG'}) {
  137. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  138. $DEBUG = $1;
  139. } else {
  140. $DEBUG = 1;
  141. }
  142. $DELAY /= 2;
  143. }
  144. my $NO_FETCH = 0;
  145. if ($ENV{'NO_FETCH'}) {
  146. $NO_FETCH = 1;
  147. }
  148. my $NO_NFO = 0;
  149. if ($ENV{'NO_NFO'}) {
  150. $NO_NFO = 1;
  151. }
  152. my $NO_SEARCH = 0;
  153. if ($ENV{'NO_SEARCH'}) {
  154. $NO_SEARCH = 1;
  155. }
  156. my $NO_FILES = 0;
  157. if ($ENV{'NO_FILES'}) {
  158. $NO_FILES = 1;
  159. }
  160. my $NO_CHANNEL = 0;
  161. if ($ENV{'NO_CHANNEL'}) {
  162. $NO_CHANNEL = 1;
  163. }
  164. my $NO_EXTRAS = 0;
  165. if ($ENV{'NO_EXTRAS'}) {
  166. $NO_EXTRAS = 1;
  167. }
  168. my $NO_EXCLUDES = 0;
  169. if ($ENV{'NO_EXCLUDES'}) {
  170. $NO_EXCLUDES = 1;
  171. }
  172. my $NO_RENAME = 0;
  173. if ($ENV{'NO_RENAME'}) {
  174. $NO_RENAME = 1;
  175. }
  176. # Environmental parameters (functional)
  177. my $SUDO_CHATTR = 1;
  178. if ($ENV{'NO_CHATTR'} || !can_run('sudo') || !can_run('chattr')) {
  179. $SUDO_CHATTR = 0;
  180. }
  181. if ($ENV{'MAX_INDEX'} && $ENV{'MAX_INDEX'} =~ /(\d+)/) {
  182. $MAX_INDEX = $1;
  183. }
  184. if ($ENV{'BATCH_SIZE'} && $ENV{'BATCH_SIZE'} =~ /(\d+)/) {
  185. $BATCH_SIZE = $1;
  186. }
  187. if (exists($ENV{'FETCH_LIMIT'}) && $ENV{'FETCH_LIMIT'} =~ /(\d+)/) {
  188. $FETCH_LIMIT = $1;
  189. }
  190. # Construct globals
  191. my $LWP = undef();
  192. foreach my $key (keys(%API)) {
  193. if (exists($API{$key}{'params'}{'maxResults'})) {
  194. $API{$key}{'params'}{'maxResults'} = $BATCH_SIZE;
  195. }
  196. }
  197. if ($DEBUG > 1) {
  198. push(@YTDL_ARGS, @YTDL_DEBUG);
  199. } elsif (!$DEBUG) {
  200. push(@YTDL_ARGS, @YTDL_QUIET);
  201. }
  202. # Allow use as a subscription manager
  203. if ($SUBSCRIPTIONS) {
  204. my %subs = ();
  205. foreach my $user (keys(%USERS)) {
  206. my $tmp = getSubscriptions($user, $USERS{$user});
  207. foreach my $sub (keys(%{$tmp})) {
  208. if (exists($subs{$sub})) {
  209. $subs{$sub} .= ', ' . $tmp->{$sub};
  210. } else {
  211. $subs{$sub} = $tmp->{$sub};
  212. }
  213. }
  214. }
  215. saveSubscriptions($DIR, \%subs);
  216. exit(0);
  217. }
  218. # Grab the channel data
  219. my $channel = {};
  220. if (!$NO_CHANNEL) {
  221. $channel = getChannel($ID);
  222. saveChannel($channel);
  223. }
  224. # Find all the user's videos on YT
  225. my $videos = {};
  226. if (!$NO_SEARCH) {
  227. $videos = findVideos($ID);
  228. }
  229. # Find any requested "extra" videos
  230. my $extras = {};
  231. if (!$NO_EXTRAS) {
  232. $extras = addExtras($videos);
  233. }
  234. # Drop any "excludes" videos
  235. my $excludes = {};
  236. if (!$NO_EXCLUDES) {
  237. $excludes = dropExcludes($videos);
  238. }
  239. # Find all existing YT files on disk
  240. my $files = {};
  241. if (!$NO_FILES) {
  242. $files = findFiles($videos);
  243. }
  244. # Deal with unknown videos
  245. foreach my $id (keys(%{$files})) {
  246. if (!exists($videos->{$id}) && $files->{$id}->{'season'} > 0) {
  247. # Check the ID specifically to find things the APIv3 fuzzy search misses
  248. if ($DEBUG) {
  249. print STDERR 'Validating local video by ID: ' . $id . "\n";
  250. }
  251. my $video = getVideoData([$id]);
  252. if ($video->[0] && $video->[0]->{'season'} > 0 && $video->[0]->{'channelId'} eq $ID) {
  253. $videos->{$id} = $video->[0];
  254. next;
  255. }
  256. # Whine when videos do go away
  257. warn('Local video not known to YT channel (' . $NAME . '): ' . $id . "\n");
  258. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, 0, $files->{$id}->{'season'} . $files->{$id}->{'number'});
  259. }
  260. }
  261. # Calculate the episode number using the publish dates
  262. {
  263. my @byDate = sort { $videos->{$a}->{'date'} <=> $videos->{$b}->{'date'} || $a cmp $b } keys %{$videos};
  264. my $num = undef();
  265. my $lastSeason = undef();
  266. foreach my $id (@byDate) {
  267. if (!$lastSeason || $lastSeason != $videos->{$id}->{'season'}) {
  268. $num = 1;
  269. $lastSeason = $videos->{$id}->{'season'};
  270. }
  271. $videos->{$id}->{'number'} = $num++;
  272. }
  273. }
  274. # Fill in missing videos and NFOs
  275. my $fetched = 0;
  276. FETCH_LOOP: foreach my $id (keys(%{$videos})) {
  277. if ($DEBUG > 1) {
  278. print STDERR 'Checking remote video: ' . $id . "\n";
  279. if (!exists($files->{$id})) {
  280. print STDERR "\tLocal file: <missing>\n";
  281. } else {
  282. print STDERR "\tLocal media: " . $files->{$id}->{'path'} . "\n";
  283. print STDERR "\tLocal NFO: " . $files->{$id}->{'nfo'} . "\n";
  284. }
  285. }
  286. my $nfo = videoPath($videos->{$id}->{'season'}, $videos->{$id}->{'number'}, $id, 'nfo');
  287. # Determine if we may or must rename
  288. my $rename = 0;
  289. if (exists($files->{$id})) {
  290. # Rename if we drift
  291. if (!$rename && $files->{$id}->{'path'}) {
  292. if ( $files->{$id}->{'number'} != $videos->{$id}->{'number'}
  293. || $files->{$id}->{'season'} != $videos->{$id}->{'season'})
  294. {
  295. if ($DEBUG) {
  296. print STDERR "Rename due to drift\n";
  297. }
  298. $rename = 1;
  299. }
  300. }
  301. # Always rename if the NFO does not match the media
  302. if (!$rename && $files->{$id}->{'path'} && $files->{$id}->{'nfo'}) {
  303. my ($season, $number) = parseFilename($files->{$id}->{'nfo'});
  304. if ($files->{$id}->{'season'} != $season || $files->{$id}->{'number'} != $number) {
  305. if ($DEBUG) {
  306. print STDERR "Rename due to media-metadata mismatch\n";
  307. }
  308. $rename = 1;
  309. }
  310. }
  311. }
  312. # Warn (and optionally rename) as selected
  313. if ($rename) {
  314. print STDERR 'Video ' . $id . ' had number ' . $files->{$id}->{'number'} . ' but now has number ' . $videos->{$id}->{'number'} . "\n";
  315. renameVideo($files->{$id}->{'path'}, $files->{$id}->{'suffix'}, $files->{$id}->{'nfo'}, $id, $videos->{$id}->{'season'}, $videos->{$id}->{'number'});
  316. }
  317. # If we haven't heard of the file, or don't have an NFO for it
  318. # Checking for the NFO allows use to resume failed downloads
  319. if (!exists($files->{$id}) || !-e $nfo) {
  320. if ($DEBUG) {
  321. print STDERR 'Fetching video: ' . $id . "\n";
  322. }
  323. # Let youtube-dl handle the URLs and downloading
  324. {
  325. my @args = ('--output', videoSE($videos->{$id}->{'season'}, $videos->{$id}->{'number'}) . '%(id)s.%(ext)s', '--', $id);
  326. my @name = ($YTDL_BIN);
  327. push(@name, @YTDL_ARGS);
  328. my @fetch = @name;
  329. push(@name, '--get-filename');
  330. push(@fetch, @args);
  331. push(@name, @args);
  332. if ($NO_FETCH) {
  333. print STDERR 'Not running: ' . join(' ', @fetch) . "\n";
  334. } else {
  335. # Count fetch attempts (even if they fail later)
  336. $fetched++;
  337. if ($FETCH_LIMIT && $fetched > $FETCH_LIMIT) {
  338. print STDERR 'Reached fetch limit (' . $FETCH_LIMIT . ') for: ' . $NAME . "\n";
  339. if ($DEBUG) {
  340. print STDERR "\tLocal/Remote videos at start:" . scalar(keys(%{$files})) . '/' . scalar(keys(%{$videos})) . "\n";
  341. }
  342. exit 1;
  343. }
  344. # Find the output file name
  345. if ($DEBUG > 1) {
  346. print STDERR join(' ', @name) . "\n";
  347. }
  348. sleep($DELAY);
  349. my $file = capture(EXIT_ANY, @name);
  350. if ($EXITVAL != 0) {
  351. warn('Error executing youtube-dl for name: ' . $id . ' (' . $EXITVAL . ")\n");
  352. next FETCH_LOOP;
  353. }
  354. $file =~ s/^\s+//;
  355. $file =~ s/\s+$//;
  356. # Sanity check
  357. if (!$file) {
  358. warn('No file name available for video: ' . $id . "\n");
  359. next FETCH_LOOP;
  360. }
  361. # Download
  362. if ($DEBUG > 1) {
  363. print STDERR join(' ', @fetch) . "\n";
  364. }
  365. sleep($DELAY);
  366. my $exit = run(EXIT_ANY, @fetch);
  367. if ($exit != 0) {
  368. warn('Error executing youtube-dl for video: ' . $file . "\n");
  369. next FETCH_LOOP;
  370. }
  371. # Ensure we found something useful
  372. if (-e $file . '.part') {
  373. warn('Partial download detected: ' . $file . "\n");
  374. next FETCH_LOOP;
  375. }
  376. if (!-s $file) {
  377. warn('No output video file: ' . $file . "\n");
  378. next FETCH_LOOP;
  379. }
  380. # Touch the file to reflect the download time rather than the upload time
  381. touch($file);
  382. }
  383. }
  384. # Build and save the XML document
  385. my $xml = buildNFO($videos->{$id});
  386. if ($DEBUG > 1) {
  387. print STDERR 'Saving NFO: ' . $xml . "\n";
  388. }
  389. if ($NO_NFO) {
  390. print STDERR "Not saving NFO\n";
  391. } else {
  392. saveString($nfo, $xml);
  393. }
  394. }
  395. }
  396. sub saveString($$) {
  397. my ($path, $str) = @_;
  398. my $fh = undef();
  399. if (!open($fh, '>', $path)) {
  400. warn('Cannot open file for writing: ' . $path . ': ' . $! . "\n");
  401. return undef();
  402. }
  403. print $fh $str;
  404. close($fh);
  405. return 1;
  406. }
  407. sub buildSeriesNFO($) {
  408. my ($channel) = @_;
  409. # Create an XML tree
  410. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  411. $doc->setStandalone(1);
  412. my $show = $doc->createElement('tvshow');
  413. $doc->setDocumentElement($show);
  414. my $elm;
  415. # Add data
  416. $elm = $doc->createElement('title');
  417. $elm->appendText($channel->{'title'});
  418. $show->appendChild($elm);
  419. $elm = $doc->createElement('premiered');
  420. $elm->appendText(time2str('%Y-%m-%d', $channel->{'date'}));
  421. $show->appendChild($elm);
  422. $elm = $doc->createElement('plot');
  423. $elm->appendText($channel->{'description'});
  424. $show->appendChild($elm);
  425. # Return the string
  426. return $doc->toString();
  427. }
  428. sub buildNFO($) {
  429. my ($video) = @_;
  430. # Create an XML tree
  431. my $doc = XML::LibXML::Document->new('1.0', 'UTF-8');
  432. $doc->setStandalone(1);
  433. my $show = $doc->createElement('episodedetails');
  434. $doc->setDocumentElement($show);
  435. my $file = $doc->createElement('fileinfo');
  436. $show->appendChild($file);
  437. my $stream = $doc->createElement('streamdetails');
  438. $file->appendChild($stream);
  439. my $stream_video = $doc->createElement('video');
  440. $stream->appendChild($stream_video);
  441. my $elm;
  442. # Add data
  443. $elm = $doc->createElement('season');
  444. $elm->appendText($video->{'season'});
  445. $show->appendChild($elm);
  446. $elm = $doc->createElement('episode');
  447. $elm->appendText(videoNumberStr($video->{'season'}, $video->{'number'}));
  448. $show->appendChild($elm);
  449. if (!defined($video->{'title'})) {
  450. $video->{'title'} = 'Episode ' . $video->{'number'};
  451. }
  452. $elm = $doc->createElement('title');
  453. $elm->appendText($video->{'title'});
  454. $show->appendChild($elm);
  455. if (!defined($video->{'date'})) {
  456. $video->{'date'} = time();
  457. }
  458. $elm = $doc->createElement('aired');
  459. $elm->appendText(time2str('%Y-%m-%d', $video->{'date'}));
  460. $show->appendChild($elm);
  461. if (defined($video->{'description'})) {
  462. $elm = $doc->createElement('plot');
  463. $elm->appendText($video->{'description'});
  464. $show->appendChild($elm);
  465. }
  466. if (defined($video->{'duration'})) {
  467. $elm = $doc->createElement('runtime');
  468. $elm->appendText($video->{'duration'});
  469. $show->appendChild($elm);
  470. $elm = $doc->createElement('durationinseconds');
  471. $elm->appendText($video->{'duration'});
  472. $stream_video->appendChild($elm);
  473. }
  474. if (defined($video->{'creator'})) {
  475. $elm = $doc->createElement('director');
  476. $elm->appendText($video->{'creator'});
  477. $show->appendChild($elm);
  478. }
  479. # Return the string
  480. return $doc->toString();
  481. }
  482. sub findFiles($) {
  483. my ($videos) = @_;
  484. my %files = ();
  485. our $NAME;
  486. # Allow complete bypass
  487. if ($NO_FILES) {
  488. return \%files;
  489. }
  490. # Read the output directory
  491. my $fh = undef();
  492. opendir($fh, '.')
  493. or die($NAME . ': Unable to open files directory: ' . $! . "\n");
  494. while (my $file = readdir($fh)) {
  495. my ($season, $number, $id, $suffix) = parseFilename($file);
  496. if (defined($id) && length($id) > 0) {
  497. # Determine if there is another video with the same season/number
  498. my $exists = 0;
  499. if ( exists($videos->{$id})
  500. && exists($videos->{$id}->{'season'})
  501. && exists($videos->{$id}->{'number'})
  502. && $season == $videos->{$id}->{'season'}
  503. && $number == $videos->{$id}->{'number'})
  504. {
  505. $exists = 1;
  506. }
  507. # Create the record as needed
  508. if (!exists($files{$id})) {
  509. my %tmp = (
  510. 'season' => $season,
  511. 'number' => $number,
  512. );
  513. $files{$id} = \%tmp;
  514. }
  515. # Video or NFO?
  516. my $type = 'path';
  517. if ($suffix eq 'nfo') {
  518. $type = 'nfo';
  519. }
  520. # Deal with duplicates
  521. if (exists($files{$id}->{$type})) {
  522. if ($type eq 'nfo') {
  523. warn('Duplicate NFO: ' . $id . "\n\t" . $files{$id}->{'nfo'} . "\n\t" . $file . "\n");
  524. if (!$NO_RENAME) {
  525. my $del = $file;
  526. if ($exists) {
  527. $del = $files{$id}->{'nfo'};
  528. }
  529. warn("\tDeleting: " . $file . "\n");
  530. if ($SUDO_CHATTR) {
  531. system('sudo', 'chattr', '-i', $file);
  532. }
  533. unlink($file);
  534. next;
  535. }
  536. } else {
  537. warn('Duplicate video: ' . $id . "\n\t" . $files{$id}->{'path'} . "\n\t" . $file . "\n");
  538. if (!$NO_RENAME) {
  539. my $del = $file;
  540. if ($exists) {
  541. $del = $files{$id}->{'path'};
  542. } elsif ($suffix ne 'mp4' && $files{$id}->{'suffix'} eq 'mp4') {
  543. $del = $files{$id}->{'path'};
  544. }
  545. warn("\tDeleting: " . $del . "\n");
  546. if ($SUDO_CHATTR) {
  547. system('sudo', 'chattr', '-i', $del);
  548. }
  549. unlink($del);
  550. next;
  551. }
  552. }
  553. }
  554. # Assign the component paths
  555. $files{$id}->{$type} = $file;
  556. if ($type ne 'nfo') {
  557. $files{$id}->{'suffix'} = $suffix;
  558. }
  559. }
  560. }
  561. close($fh);
  562. if ($DEBUG) {
  563. print STDERR 'Found ' . scalar(keys(%files)) . " local videos\n";
  564. if ($DEBUG > 1) {
  565. print STDERR prettyPrint(\%files, "\t") . "\n";
  566. }
  567. }
  568. return \%files;
  569. }
  570. sub fetchParse($$) {
  571. my ($name, $params) = @_;
  572. our $NAME;
  573. my $url = $API{$name}{'url'} . '?';
  574. foreach my $key (keys(%{ $API{$name}{'params'} }), keys(%{$params})) {
  575. $url .= '&' . uri_escape($key) . '=';
  576. if (exists($params->{$key})) {
  577. $url .= uri_escape($params->{$key});
  578. } else {
  579. $url .= uri_escape($API{$name}{'params'}{$key});
  580. }
  581. }
  582. # Fetch
  583. if ($DEBUG > 1) {
  584. print STDERR 'Fetching ' . $name . ' API URL: ' . $url . "\n";
  585. }
  586. sleep($DELAY);
  587. my $content;
  588. my $code = fetch($url, \$content);
  589. if ($code != 200 || !defined($content) || length($content) < 10) {
  590. my $msg = $NAME . ': Invalid content from URL (' . $code . '): ' . $url . "\n" . $content . "\n";
  591. if ($code >= 500 && $code < 600) {
  592. if ($DEBUG) {
  593. print STDERR $msg;
  594. }
  595. exit(0);
  596. }
  597. if ($code >= 400 && $code < 500) {
  598. if ($DEBUG) {
  599. print STDERR "Wainting 30 seconds after 400 error\n";
  600. }
  601. sleep(30);
  602. }
  603. die($msg);
  604. }
  605. # Parse
  606. if ($DEBUG > 2) {
  607. print STDERR "Raw JSON data:\n" . $content . "\n";
  608. }
  609. my $data = decode_json($content);
  610. if (!defined($data) || ref($data) ne 'HASH') {
  611. die($NAME . ': Invalid JSON: ' . $content . "\n");
  612. }
  613. return $data;
  614. }
  615. sub getSubscriptions($$) {
  616. my ($user, $id) = @_;
  617. our $NAME;
  618. # Loop through until we have all the entries
  619. my $pageToken = '';
  620. my %subs = ();
  621. SUBS_LOOP:
  622. {
  623. # Build, fetch, parse, check
  624. my $data = fetchParse('subscriptions', { 'channelId' => $id, 'pageToken' => $pageToken });
  625. $pageToken = '';
  626. if ( exists($data->{'pageInfo'})
  627. && ref($data->{'pageInfo'}) eq 'HASH'
  628. && exists($data->{'pageInfo'}->{'totalResults'})
  629. && $data->{'pageInfo'}->{'totalResults'} > 0
  630. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  631. && exists($data->{'items'})
  632. && ref($data->{'items'}) eq 'ARRAY')
  633. {
  634. if (exists($data->{'nextPageToken'})) {
  635. $pageToken = $data->{'nextPageToken'};
  636. }
  637. $data = $data->{'items'};
  638. } else {
  639. die($NAME . ": Invalid subscription data\n");
  640. }
  641. foreach my $item (@{$data}) {
  642. if ( !exists($item->{'snippet'})
  643. || ref($item->{'snippet'}) ne 'HASH'
  644. || !exists($item->{'snippet'}->{'title'})
  645. || !exists($item->{'snippet'}->{'resourceId'})
  646. || ref($item->{'snippet'}->{'resourceId'}) ne 'HASH'
  647. || !exists($item->{'snippet'}->{'resourceId'}->{'channelId'}))
  648. {
  649. warn($NAME . ": Skipping invalid subscription\n");
  650. if ($DEBUG > 2) {
  651. print STDERR prettyPrint($item, "\t") . "\n";
  652. }
  653. next;
  654. }
  655. if ($DEBUG) {
  656. print STDERR $item->{'snippet'}->{'title'} . ' => ' . $item->{'snippet'}->{'resourceId'}->{'channelId'} . "\n";
  657. }
  658. $subs{ $item->{'snippet'}->{'title'} . ' (' . $item->{'snippet'}->{'resourceId'}->{'channelId'} . ')' } = $user;
  659. }
  660. # Loop if there are results left to fetch
  661. if ($pageToken) {
  662. redo SUBS_LOOP;
  663. }
  664. }
  665. # Return the list of subscribed usernames
  666. return \%subs;
  667. }
  668. sub saveSubscriptions($$) {
  669. my ($folder, $subs) = @_;
  670. our $NAME;
  671. # Check for local subscriptions missing from YT
  672. my %locals = ();
  673. my $fh = undef();
  674. opendir($fh, '.')
  675. or die($NAME . ': Unable to open subscriptions directory: ' . $folder . ': ' . $! . "\n");
  676. while (my $file = readdir($fh)) {
  677. # Skip dotfiles
  678. if ($file =~ /^\./) {
  679. next;
  680. }
  681. # Skip non-directories
  682. if (!-d $file) {
  683. next;
  684. }
  685. # YT has some trouble with case
  686. my $lcFile = lc($file);
  687. # Anything else should be in the list
  688. if (!$subs->{$file} && !$subs->{$lcFile}) {
  689. my (undef(), undef(), $hour) = localtime(time);
  690. if ($hour == 0 || $DEBUG) {
  691. print STDERR 'Missing YT subscription for: ' . $file . "\n";
  692. }
  693. }
  694. # Note local subscriptions
  695. $locals{$lcFile} = 1;
  696. }
  697. closedir($fh);
  698. # Check for YT subscriptions missing locally
  699. foreach my $sub (keys(%{$subs})) {
  700. if (!exists($locals{ lc($sub) })) {
  701. print STDERR 'Adding local subscription for: ' . $sub . ' => ' . $subs->{$sub} . "\n";
  702. mkdir($folder . '/' . $sub);
  703. }
  704. }
  705. }
  706. sub saveChannel($) {
  707. my ($channel) = @_;
  708. my $nfo = 'tvshow.nfo';
  709. if (!-e $nfo) {
  710. if ($DEBUG) {
  711. print STDERR 'Saving series data for: ' . $channel->{'title'} . "\n";
  712. }
  713. # Save the poster
  714. if (exists($channel->{'thumbnail'}) && length($channel->{'thumbnail'}) > 5) {
  715. my $jpg;
  716. if (fetch($channel->{'thumbnail'}, \$jpg) == 200) {
  717. saveString('poster.jpg', $jpg);
  718. }
  719. }
  720. # Save the series NFO
  721. my $xml = buildSeriesNFO($channel);
  722. if ($DEBUG > 1) {
  723. print STDERR 'Saving NFO: ' . $xml . "\n";
  724. }
  725. saveString($nfo, $xml);
  726. }
  727. }
  728. sub getChannel($) {
  729. my ($id) = @_;
  730. our $NAME;
  731. # Build, fetch, parse, check
  732. my $data = fetchParse('channel', { 'id' => $id });
  733. if ( exists($data->{'pageInfo'})
  734. && ref($data->{'pageInfo'}) eq 'HASH'
  735. && exists($data->{'pageInfo'}->{'totalResults'})
  736. && $data->{'pageInfo'}->{'totalResults'} == 1
  737. && exists($data->{'items'})
  738. && ref($data->{'items'}) eq 'ARRAY'
  739. && scalar(@{ $data->{'items'} }) > 0
  740. && ref($data->{'items'}[0]) eq 'HASH'
  741. && exists($data->{'items'}[0]->{'snippet'})
  742. && ref($data->{'items'}[0]->{'snippet'}) eq 'HASH')
  743. {
  744. $data = $data->{'items'}[0]->{'snippet'};
  745. } else {
  746. die($NAME . ": Invalid channel data\n");
  747. }
  748. # Extract the data we want
  749. my %channel = (
  750. 'id' => $id,
  751. 'title' => $data->{'title'},
  752. 'date' => str2time($data->{'publishedAt'}),
  753. 'description' => $data->{'description'},
  754. 'thumbnail' => $data->{'thumbnails'}->{'high'}->{'url'},
  755. );
  756. return \%channel;
  757. }
  758. sub readExcludes() {
  759. my %excludes = ();
  760. our $NAME;
  761. # Read and parse the excludes videos file, if it exists
  762. if (-e $EXCLUDES_FILE) {
  763. my $fh;
  764. open($fh, $EXCLUDES_FILE)
  765. or die($NAME . ': Unable to open excludes videos file: ' . $! . "\n");
  766. while (<$fh>) {
  767. # Skip blank lines and comments
  768. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  769. next;
  770. }
  771. # Match our specific format or whine
  772. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  773. if ($DEBUG > 1) {
  774. print STDERR 'Adding exclude video: ' . $1 . "\n";
  775. }
  776. $excludes{$1} = 1;
  777. } else {
  778. print STDERR 'Skipped exclude video line: ' . $_;
  779. }
  780. }
  781. close($fh);
  782. }
  783. return \%excludes;
  784. }
  785. sub readExtras() {
  786. my %extras = ();
  787. our $NAME;
  788. # Read and parse the extra videos file, if it exists
  789. if (-e $EXTRAS_FILE) {
  790. my $fh;
  791. open($fh, $EXTRAS_FILE)
  792. or die($NAME . ': Unable to open extra videos file: ' . $! . "\n");
  793. while (<$fh>) {
  794. # Skip blank lines and comments
  795. if ($_ =~ /^\s*#/ || $_ =~ /^\s*$/) {
  796. next;
  797. }
  798. # Match our specific format or whine
  799. if ($_ =~ /^\s*([\w\-]+)\s*$/) {
  800. if ($DEBUG > 1) {
  801. print STDERR 'Adding extra video: ' . $1 . "\n";
  802. }
  803. $extras{$1} = 1;
  804. } else {
  805. print STDERR 'Skipped extra video line: ' . $_;
  806. }
  807. }
  808. close($fh);
  809. }
  810. return \%extras;
  811. }
  812. sub parseVideoData($) {
  813. my ($data) = @_;
  814. if ( !exists($data->{'id'})
  815. || !exists($data->{'snippet'})
  816. || ref($data->{'snippet'}) ne 'HASH'
  817. || !exists($data->{'contentDetails'})
  818. || ref($data->{'contentDetails'}) ne 'HASH'
  819. || !exists($data->{'contentDetails'}->{'duration'}))
  820. {
  821. if ($DEBUG > 2) {
  822. print STDERR "Unable to parse video data:\n" . prettyPrint($data, "\t") . "\n";
  823. }
  824. return undef();
  825. }
  826. # Direct extraction
  827. my %video = (
  828. 'id' => $data->{'id'},
  829. 'title' => $data->{'snippet'}->{'title'},
  830. 'description' => $data->{'snippet'}->{'description'},
  831. 'creator' => $data->{'snippet'}->{'channelTitle'},
  832. 'publishedAt' => $data->{'snippet'}->{'publishedAt'},
  833. 'channelId' => $data->{'snippet'}->{'channelId'},
  834. 'rawDuration' => $data->{'contentDetails'}->{'duration'},
  835. );
  836. # Convert datetime formats
  837. $video{'date'} = str2time($video{'publishedAt'});
  838. $video{'season'} = time2str('%Y', $video{'date'});
  839. if ($video{'rawDuration'} =~ /PT(\d+)M(\d+)S/) {
  840. $video{'duration'} = ($1 * 60) + $2;
  841. } elsif ($video{'rawDuration'} =~ /PT(\d+)S/) {
  842. $video{'duration'} = $1;
  843. }
  844. return \%video;
  845. }
  846. sub dropExcludes($) {
  847. my ($videos) = @_;
  848. my $excludes = readExcludes();
  849. foreach my $id (keys(%{$excludes})) {
  850. if (!exists($videos->{$id})) {
  851. if ($DEBUG > 1) {
  852. print STDERR 'Skipping unknown excludes video: ' . $id . "\n";
  853. }
  854. next;
  855. }
  856. delete($videos->{$id});
  857. }
  858. return $excludes;
  859. }
  860. sub addExtras($) {
  861. my ($videos) = @_;
  862. my $extras = readExtras();
  863. foreach my $id (keys(%{$extras})) {
  864. if (exists($videos->{$id})) {
  865. if ($DEBUG > 1) {
  866. print STDERR 'Skipping known extra video: ' . $id . "\n";
  867. }
  868. next;
  869. }
  870. my $video = getVideoData([$id]);
  871. $videos->{$id} = $video->[0];
  872. }
  873. return $extras;
  874. }
  875. sub getVideoData($) {
  876. my ($ids) = @_;
  877. my @videos = ();
  878. our $NAME;
  879. # Do a batch request for the entire batch of video data
  880. my $data = fetchParse('video', { 'id' => join(',', @{$ids}) });
  881. if ( exists($data->{'pageInfo'})
  882. && ref($data->{'pageInfo'}) eq 'HASH'
  883. && exists($data->{'pageInfo'}->{'totalResults'})
  884. && exists($data->{'items'})
  885. && ref($data->{'items'}) eq 'ARRAY')
  886. {
  887. if (scalar(@{$ids}) > 1 && $data->{'pageInfo'}->{'totalResults'} != scalar(@{$ids})) {
  888. die($NAME . ': Video/metadata search count mismatch: ' . $data->{'pageInfo'}->{'totalResults'} . '/' . scalar(@{$ids}) . "\n");
  889. }
  890. $data = $data->{'items'};
  891. } else {
  892. die($NAME . ": Invalid search video data\n" . prettyPrint($data, "\t") . "\n");
  893. }
  894. # Build each metadata record
  895. foreach my $item (@{$data}) {
  896. if (!exists($item->{'snippet'}) || ref($item->{'snippet'}) ne 'HASH') {
  897. warn($NAME . ": Skipping invalid metadata item\n");
  898. if ($DEBUG > 2) {
  899. print STDERR prettyPrint($item, "\t") . "\n";
  900. }
  901. next;
  902. }
  903. my $video = parseVideoData($item);
  904. if (!$video) {
  905. die($NAME . ": Unable to parse video data\n");
  906. }
  907. push(@videos, $video);
  908. }
  909. # Parse the video data
  910. return \@videos;
  911. }
  912. sub findVideos($) {
  913. my ($id) = @_;
  914. my %videos = ();
  915. our $NAME;
  916. # Allow complete bypass
  917. if ($NO_SEARCH) {
  918. return \%videos;
  919. }
  920. # Loop through until we have all the entries
  921. my $totalCount = 0;
  922. my %params = ('channelId' => $id);
  923. LOOP:
  924. {
  925. # Loop status
  926. my $count = scalar(keys(%videos));
  927. if ($DEBUG && $totalCount) {
  928. print STDERR 'Fetching ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%)\n";
  929. }
  930. # Build, fetch, parse, check
  931. my $data = fetchParse('search', \%params);
  932. delete($params{'pageToken'});
  933. if ( exists($data->{'pageInfo'})
  934. && ref($data->{'pageInfo'}) eq 'HASH'
  935. && exists($data->{'pageInfo'}->{'totalResults'})
  936. && $data->{'pageInfo'}->{'totalResults'} > 0
  937. && exists($data->{'pageInfo'}->{'resultsPerPage'})
  938. && exists($data->{'items'})
  939. && ref($data->{'items'}) eq 'ARRAY')
  940. {
  941. if (!$totalCount) {
  942. $totalCount = $data->{'pageInfo'}->{'totalResults'};
  943. }
  944. $data = $data->{'items'};
  945. } else {
  946. die($NAME . ": Invalid search data\n");
  947. }
  948. # Sanity check
  949. if (scalar(@{$data}) < 1) {
  950. last LOOP;
  951. }
  952. # Grab each video ID
  953. my @ids = ();
  954. foreach my $item (@{$data}) {
  955. if (!exists($item->{'id'}) || ref($item->{'id'}) ne 'HASH' || !exists($item->{'id'}->{'videoId'})) {
  956. warn($NAME . ": Skipping invalid video item\n");
  957. if ($DEBUG > 2) {
  958. print STDERR prettyPrint($data, "\t") . "\n";
  959. }
  960. next;
  961. }
  962. push(@ids, $item->{'id'}->{'videoId'});
  963. }
  964. # Do a batch request for the entire batch of video data
  965. my $more = 0;
  966. my $records = getVideoData(\@ids);
  967. foreach my $rec (@{$records}) {
  968. if (exists($rec->{'duration'}) && $rec->{'duration'} == 0) {
  969. if ($DEBUG > 1) {
  970. print STDERR 'Skipping 0-length video: ' . $rec->{'id'} . "\n";
  971. }
  972. next;
  973. }
  974. $videos{ $rec->{'id'} } = $rec;
  975. if (!defined($params{'publishedBefore'}) || (defined($rec->{'publishedAt'}) && ($rec->{'publishedAt'} cmp $params{'publishedBefore'}) < 0)) {
  976. $params{'publishedBefore'} = $rec->{'publishedAt'};
  977. $more = 1;
  978. }
  979. }
  980. # Loop if there are results left to fetch
  981. if ($more && $params{'publishedBefore'} && $count <= $MAX_INDEX) {
  982. redo LOOP;
  983. }
  984. }
  985. # Sanity check/debug
  986. {
  987. my $count = scalar(keys(%videos));
  988. if ($DEBUG) {
  989. print STDERR 'Found ' . $count . " remote videos\n";
  990. if ($DEBUG > 1) {
  991. print STDERR prettyPrint(\%videos, "\t") . "\n";
  992. }
  993. }
  994. if (abs(1 - ($count / $totalCount)) > $API_ST_REL && abs($totalCount - $count) > $API_ST_ABS) {
  995. die($NAME . ': Found only ' . $count . ' of ' . $totalCount . ' (' . int(100 * $count / $totalCount) . "%) remote videos. Aborting...\n");
  996. }
  997. }
  998. return \%videos;
  999. }
  1000. sub updateNFOData($$$) {
  1001. my ($file, $season, $episode) = @_;
  1002. my $nfoData = undef();
  1003. {
  1004. open(my $fh, '<', $file)
  1005. or warn('Unable to open NFO for renumber: ' . $file . ': ' . $! . "\n");
  1006. $/ = undef;
  1007. $nfoData = <$fh>;
  1008. close($fh);
  1009. }
  1010. if ($nfoData) {
  1011. $nfoData =~ s/<season>\d+<\/season>/<season>${season}<\/season>/;
  1012. $nfoData =~ s/<episode>\d+<\/episode>/<episode>${episode}<\/episode>/;
  1013. }
  1014. return $nfoData;
  1015. }
  1016. sub videoNumberStr($$) {
  1017. my ($season, $episode) = @_;
  1018. my $str = '';
  1019. if ($season == 0 && $episode =~ /^(20[012]\d)(\d{2,4})$/) {
  1020. $str = sprintf('%04d%04d', $1, $2);
  1021. } else {
  1022. $str = sprintf('%02d', $episode);
  1023. }
  1024. return $str;
  1025. }
  1026. sub videoSE($$) {
  1027. my ($season, $episode) = @_;
  1028. return sprintf('S%0dE%s - ', $season, videoNumberStr($season, $episode));
  1029. }
  1030. sub videoPath($$$$) {
  1031. my ($season, $episode, $id, $ext) = @_;
  1032. return videoSE($season, $episode) . $id . '.' . $ext;
  1033. }
  1034. sub renameVideo($$$$$$) {
  1035. my ($video, $suffix, $nfo, $id, $season, $episode) = @_;
  1036. our $NAME;
  1037. # Bail if disabled
  1038. if ($NO_RENAME) {
  1039. print STDERR 'Not renaming: ' . $video . "\n";
  1040. return;
  1041. }
  1042. # General sanity checks
  1043. if (!defined($id) || !defined($season) || !defined($episode)) {
  1044. die($NAME . ": Invalid call to renameVideo()\n\t" . $video . "\n\t" . $suffix . "\n\t" . $nfo . "\n\t" . $id . "\n\t" . $season . "\n\t" . $episode . "\n");
  1045. }
  1046. # Warning about useless calls
  1047. if (!defined($nfo) && (!defined($video) || !defined($suffix))) {
  1048. warn("No NFO or video path provided in renameVideo()\n");
  1049. return;
  1050. }
  1051. # Video
  1052. if (defined($video) && defined($suffix)) {
  1053. my $videoNew = videoPath($season, $episode, $id, $suffix);
  1054. # Sanity checks, as we do dangerous work here
  1055. if (!-r $video) {
  1056. die($NAME . ': Invalid video source in rename: ' . $video . "\n");
  1057. }
  1058. if ($video ne $videoNew) {
  1059. print STDERR 'Renaming ' . $NAME . '/' . $video . ' => ' . $videoNew . "\n";
  1060. if (-e $videoNew) {
  1061. die($NAME . ': Rename: Target exists: ' . $videoNew . "\n");
  1062. }
  1063. if ($SUDO_CHATTR) {
  1064. system('sudo', 'chattr', '-i', $video);
  1065. }
  1066. rename($video, $videoNew)
  1067. or die($NAME . ': Unable to rename: ' . $video . ': ' . $! . "\n");
  1068. }
  1069. }
  1070. # NFO
  1071. if (defined($nfo)) {
  1072. my $nfoNew = videoPath($season, $episode, $id, 'nfo');
  1073. # Sanity checks, as we do dangerous work here
  1074. if (!-r $nfo) {
  1075. die($NAME . ': Invalid NFO source in rename: ' . $nfo . "\n");
  1076. }
  1077. # Parse old NFO data
  1078. my $nfoData = updateNFOData($nfo, $season, $episode)
  1079. or die($NAME . ': No NFO data found, refusing to rename: ' . $id . "\n");
  1080. # Write a new NFO and unlink the old one
  1081. if ($SUDO_CHATTR) {
  1082. system('sudo', 'chattr', '-i', $nfo);
  1083. }
  1084. saveString($nfoNew, $nfoData);
  1085. if ($nfo ne $nfoNew) {
  1086. print STDERR 'Renaming ' . $NAME . '/' . $nfo . ' => ' . $nfoNew . "\n";
  1087. unlink($nfo)
  1088. or warn('Unable to delete NFO during rename: ' . $! . "\n");
  1089. }
  1090. }
  1091. }
  1092. sub parseFilename($) {
  1093. my ($file) = @_;
  1094. return $file =~ /^S(\d+)+E(\d+) - ([\w\-]+)\.(\w\w\w)$/i;
  1095. }
  1096. # GET the requested URL, accepting encoded data and extracting the HTTP status code
  1097. sub fetch($$) {
  1098. my ($url, $data) = @_;
  1099. # Init the global LWP as needed
  1100. if (!defined($LWP)) {
  1101. $LWP = new LWP::UserAgent;
  1102. $LWP->agent($HTTP_UA);
  1103. $LWP->timeout($HTTP_TIMEOUT);
  1104. $LWP->ssl_opts({ 'verify_hostname' => $HTTP_VERIFY });
  1105. }
  1106. # Make a request
  1107. my $request = HTTP::Request->new('GET' => $url);
  1108. $request->header('Accept-Encoding' => HTTP::Message::decodable);
  1109. my $response = $LWP->request($request);
  1110. # Do something useful with the response
  1111. my $code = 400;
  1112. if ($response->status_line =~ /^(\d+)\s/) {
  1113. $code = $1;
  1114. }
  1115. ${$data} = $response->decoded_content();
  1116. return $code;
  1117. }