findTorrent.pl 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. # Defaults
  5. my $DEBUG = 0;
  6. my $SYSLOG = 1;
  7. my $NO_QUALITY_CHECKS = 0;
  8. my $MORE_NUMBER_FORMATS = 0;
  9. my $MIN_DAYS_BACK = 0;
  10. my $MAX_DAYS_BACK = 3;
  11. my $NEXT_EPISODES = 3;
  12. # Local config
  13. my $TV_DIR = `~/bin/video/mediaPath` . '/TV';
  14. my $CONF_FILE = $ENV{'HOME'} . +'/.findTorrent.config';
  15. my $EXCLUDES_FILE = $ENV{'HOME'} . '/.findTorrent.exclude';
  16. # Selection parameters
  17. my $MIN_COUNT = 10;
  18. my $MIN_SIZE = 100;
  19. my $SIZE_BONUS = 5;
  20. my $SIZE_PENALTY = $SIZE_BONUS;
  21. my $TITLE_PENALTY = $SIZE_BONUS / 2;
  22. my $MAX_SEEDS = 500;
  23. my $AGE_PENALTY = ($MAX_SEEDS / 2) / (86400 * 2);
  24. my $MAX_SEED_RATIO = .25;
  25. my $SEED_RATIO_COUNT = 10;
  26. my $TRACKER_LOOKUP = 1;
  27. # App config
  28. my $SLEEP = 1;
  29. my $DELAY = 10;
  30. my $TIMEOUT = 15;
  31. my $ERR_DELAY = $TIMEOUT * 2;
  32. my $ERR_RETRIES = 3;
  33. my $COOKIES = undef();
  34. my $FETCH = undef();
  35. my $PHANTOM = '/usr/local/bin/phantomjs';
  36. my $UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/538.39.41 (KHTML, like Gecko) Version/8.0 Safari/538.39.41';
  37. my $PHANTOM_CONFIG = "var system = require('system');
  38. var page = require('webpage').create();
  39. var resources = [];
  40. page.open(system.args[1], function(status)
  41. {
  42. console.log(resources[0].status);
  43. console.log(page.content);
  44. phantom.exit();
  45. });
  46. // Add responses for completed text/html resources
  47. page.onResourceReceived = function(response) {
  48. if (response.stage !== 'end') return;
  49. if (response.headers.filter(function(header) {
  50. if (header.name == 'Content-Type' && header.value.indexOf('text/html') == 0) {
  51. return true;
  52. }
  53. return false;
  54. }).length > 0)
  55. resources.push(response);
  56. };";
  57. # Includes
  58. use Date::Parse;
  59. use URI::Encode qw(uri_encode);
  60. use HTML::Entities;
  61. use HTML::Strip;
  62. use JSON;
  63. use Sys::Syslog qw(:standard :macros);
  64. use Capture::Tiny ':all';
  65. use File::Temp;
  66. use File::Spec;
  67. use File::Basename;
  68. use FindBin qw($Bin);
  69. use lib $Bin;
  70. use Fetch;
  71. # Prototypes
  72. sub getHash($);
  73. sub resolveSecondary($);
  74. sub resolveTrackers($);
  75. sub splitTags($$$$);
  76. sub findSE($);
  77. sub initSources();
  78. sub findProxy($$);
  79. sub fetch($;$$);
  80. sub phantomFetch($$);
  81. # Command line
  82. my ($dir, $search) = @ARGV;
  83. if (!defined($dir)) {
  84. die('Usage: ' . basename($0) . " input_directory [search_string]\n");
  85. }
  86. # Environment
  87. if ($ENV{'DEBUG'}) {
  88. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  89. $DEBUG = $1;
  90. } else {
  91. $DEBUG = 1;
  92. }
  93. }
  94. if (exists($ENV{'EXCLUDES_FILE'})) {
  95. $EXCLUDES_FILE = $ENV{'EXCLUDES_FILE'};
  96. }
  97. if ($ENV{'NO_QUALITY_CHECKS'}) {
  98. $NO_QUALITY_CHECKS = 1;
  99. }
  100. if ($ENV{'MORE_NUMBER_FORMATS'}) {
  101. $MORE_NUMBER_FORMATS = 1;
  102. }
  103. if (defined($ENV{'MIN_DAYS_BACK'})) {
  104. $MIN_DAYS_BACK = $ENV{'MIN_DAYS_BACK'};
  105. }
  106. if (defined($ENV{'MAX_DAYS_BACK'})) {
  107. $MAX_DAYS_BACK = $ENV{'MAX_DAYS_BACK'};
  108. }
  109. if (defined($ENV{'NEXT_EPISODES'})) {
  110. $NEXT_EPISODES = $ENV{'NEXT_EPISODES'};
  111. }
  112. if (defined($ENV{'SYSLOG'})) {
  113. $SYSLOG = $ENV{'SYSLOG'};
  114. }
  115. # Read the config file
  116. my %CONFIG = ();
  117. if ($CONF_FILE && -r $CONF_FILE) {
  118. my $fh;
  119. open($fh, '<', $CONF_FILE)
  120. or die('Unable to open config file: ' . $CONF_FILE . ': ' . $! . "\n");
  121. while (<$fh>) {
  122. # Skip blank lines and comments
  123. if (/^\s*#/ || /^\s*$/) {
  124. next;
  125. }
  126. # Assume everything else is key = val pairs
  127. if (my ($key, $val) = $_ =~ /^\s*(\S[^\=]+)\=\>?\s*(\S.*)$/) {
  128. $key =~ s/^\s+//;
  129. $key =~ s/\s+$//;
  130. $val =~ s/^\s+//;
  131. $val =~ s/\s+$//;
  132. $CONFIG{$key} = $val;
  133. if ($DEBUG) {
  134. print STDERR 'Adding config: ' . $key . ' => ' . $val . "\n";
  135. }
  136. } else {
  137. warn('Ignoring config line: ' . $_ . "\n");
  138. }
  139. }
  140. close($fh);
  141. }
  142. # Parse config items
  143. my $PROTOCOL = 'https';
  144. if (exists($CONFIG{'PROTOCOL'})) {
  145. $PROTOCOL = $CONFIG{'PROTOCOL'};
  146. }
  147. my @TRACKERS = ();
  148. if (exists($CONFIG{'TRACKERS'})) {
  149. foreach my $tracker (split(/\s*,\s*/, $CONFIG{'TRACKERS'})) {
  150. $tracker =~ s/^\s+//;
  151. $tracker =~ s/\s+$//;
  152. push(@TRACKERS, $tracker);
  153. }
  154. }
  155. my %ENABLE_SOURCE = ();
  156. if (exists($CONFIG{'SOURCES'})) {
  157. foreach my $source (split(/\s*,\s*/, $CONFIG{'SOURCES'})) {
  158. $source =~ s/^\s+//;
  159. $source =~ s/\s+$//;
  160. $source = uc($source);
  161. $ENABLE_SOURCE{$source} = 1;
  162. }
  163. }
  164. # Phantom setup
  165. my $SCRIPT = undef();
  166. # Open the log if enabled
  167. if ($SYSLOG) {
  168. openlog(basename($0), '', LOG_DAEMON);
  169. }
  170. # Read the torrent excludes list
  171. my %EXCLUDES = ();
  172. if ($EXCLUDES_FILE && -r $EXCLUDES_FILE) {
  173. my $fh;
  174. open($fh, '<', $EXCLUDES_FILE)
  175. or die('Unable to open excludes file: ' . $EXCLUDES_FILE . ': ' . $! . "\n");
  176. while (<$fh>) {
  177. # Skip blank lines and comments
  178. if (/^\s*$/ || /^\s*#/) {
  179. next;
  180. }
  181. # Assume everything else is one BT hash per line
  182. if (/^\s*(\w+)\s*$/) {
  183. my $hash = lc($1);
  184. if ($DEBUG > 1) {
  185. print STDERR 'Excluding hash: ' . $hash . "\n";
  186. }
  187. $EXCLUDES{$hash} = 1;
  188. } else {
  189. die('Invalid exclude line: ' . $_ . "\n");
  190. }
  191. }
  192. close($fh);
  193. } elsif ($DEBUG) {
  194. print STDERR 'No excludes file available';
  195. if ($EXCLUDES_FILE) {
  196. print STDERR ': ' . $EXCLUDES_FILE;
  197. }
  198. print STDERR "\n";
  199. }
  200. # Figure out what we're searching for
  201. my $show = '';
  202. my @urls = ();
  203. my $CUSTOM_SEARCH = 0;
  204. my $season = 0;
  205. my @need = ();
  206. my %need = ();
  207. # Clean up the input "directory" path
  208. {
  209. # Allow use of the raw series name
  210. if (!($dir =~ /\//)) {
  211. $show = $dir;
  212. $dir = $TV_DIR . '/' . $dir;
  213. }
  214. # Allow use of relative paths
  215. $dir = File::Spec->rel2abs($dir);
  216. # Sanity check
  217. if (!-d $dir) {
  218. die('Invalid input directory: ' . $dir . "\n");
  219. }
  220. # Isolate the season from the path, if provided
  221. $dir =~ /\/Season\s+(\d+)\/?$/i;
  222. if ($1) {
  223. $season = $1;
  224. $dir = dirname($dir);
  225. }
  226. # If no season is provided find the latest
  227. if (!$season) {
  228. opendir(SERIES, $dir)
  229. or die("Unable to open series directory: ${!}\n");
  230. while (my $file = readdir(SERIES)) {
  231. if ($file =~ /^Season\s+(\d+)$/i) {
  232. if (!$season || $season < $1) {
  233. $season = $1;
  234. }
  235. }
  236. }
  237. closedir(SERIES);
  238. }
  239. # Isolate and clean the series name
  240. if (!$show) {
  241. $show = basename($dir);
  242. }
  243. $show =~ s/[\'\"\.]//g;
  244. if ($DEBUG) {
  245. print STDERR 'Checking directory: ' . $dir . "\n";
  246. }
  247. if ($SYSLOG) {
  248. syslog(LOG_NOTICE, $show);
  249. }
  250. }
  251. # Allow the show name to be overriden
  252. {
  253. my $search_name = $dir . '/search_name';
  254. if (-e $search_name) {
  255. local ($/, *FH);
  256. open(FH, $search_name)
  257. or die('Unable to read search_name for show: ' . $show . ': ' . $! . "\n");
  258. my $text = <FH>;
  259. close(FH);
  260. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  261. $show = $1;
  262. } else {
  263. print STDERR 'Skipping invalid search_name for show: ' . $show . ': ' . $text . "\n";
  264. }
  265. }
  266. if ($DEBUG) {
  267. print STDERR 'Searching with series title: ' . $show . "\n";
  268. }
  269. }
  270. # Allow quality checks to be disabled
  271. if (-e $dir . '/no_quality_checks') {
  272. $NO_QUALITY_CHECKS = 1;
  273. if ($DEBUG) {
  274. print STDERR 'Searching with no quality checks: ' . $show . "\n";
  275. }
  276. }
  277. # Allow use of more number formats
  278. if (-e $dir . '/more_number_formats') {
  279. $MORE_NUMBER_FORMATS = 1;
  280. if ($DEBUG) {
  281. print STDERR 'Searching with more number formats: ' . $show . "\n";
  282. }
  283. }
  284. # Read the search excludes file, if any
  285. my $exclude = '';
  286. my %TITLE_EXCLUDES = ();
  287. if (-e $dir . '/excludes') {
  288. local $/ = undef;
  289. open(EX, $dir . '/excludes')
  290. or die("Unable to open series excludes file: ${!}\n");
  291. my $ex = <EX>;
  292. close(EX);
  293. $ex =~ s/^\s+//;
  294. $ex =~ s/\s+$//;
  295. my @excludes = split(/\s*,\s*/, $ex);
  296. foreach my $ex (@excludes) {
  297. $TITLE_EXCLUDES{$ex} = 1;
  298. if (length($exclude)) {
  299. $exclude .= ' ';
  300. }
  301. $exclude .= '-"' . $ex . '"';
  302. }
  303. $exclude = uri_encode(' ' . $exclude);
  304. }
  305. # Setup our sources
  306. my $SOURCES = initSources();
  307. # Adjust the inter-page delay with respect to the number of unique sources
  308. $DELAY /= scalar(keys(%{$SOURCES})) / 2;
  309. # Handle custom searches
  310. if ((scalar(@urls) < 1) && defined($search) && length($search) > 0) {
  311. # Note the custom search string
  312. $CUSTOM_SEARCH = 1;
  313. if ($DEBUG) {
  314. print STDERR "Custom search\n";
  315. }
  316. # Create the relevent search strings
  317. foreach my $key (keys(%{$SOURCES})) {
  318. my $source = $SOURCES->{$key};
  319. push(@urls, $source->{'search_url'} . $search . $exclude . $source->{'search_suffix'});
  320. }
  321. }
  322. # Handle search-by-date series
  323. if ((scalar(@urls) < 1) && -e $dir . '/search_by_date') {
  324. # Note the search-by-date status
  325. # Set the CUSTOM_SERACH flag to skip season/epsiode matching
  326. $CUSTOM_SEARCH = 1;
  327. # Read the find-by-date string
  328. my $search_by_date = '';
  329. local ($/, *FH);
  330. open(FH, $dir . '/search_by_date')
  331. or die('Unable to read search_by_date for show: ' . $show . ': ' . $! . "\n");
  332. my $text = <FH>;
  333. close(FH);
  334. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  335. $search_by_date = $1;
  336. } else {
  337. die('Skipping invalid search_by_date for show: ' . $show . ': ' . $text . "\n");
  338. }
  339. # Create search strings for each date in the range, unless the related file already exists
  340. my (%years, %months, %days) = ();
  341. for (my $days_back = $MIN_DAYS_BACK ; $days_back <= $MAX_DAYS_BACK ; $days_back++) {
  342. # Calculate the date
  343. my (undef(), undef(), undef(), $day, $month, $year) = localtime(time() - (86400 * $days_back));
  344. # Format as strings
  345. $year = sprintf('%04d', $year + 1900);
  346. $month = sprintf('%02d', $month + 1);
  347. $day = sprintf('%02d', $day);
  348. # Check for an existing file
  349. my $exists = 0;
  350. {
  351. my $season_dir = $dir . '/Season ' . $year;
  352. my $prefix = qr/${year}\-${month}\-${day}\s*\-\s*/;
  353. opendir(SEASON, $season_dir)
  354. or die('Unable to open season directory (' . $season_dir . '): ' . $! . "\n");
  355. while (my $file = readdir(SEASON)) {
  356. if ($file =~ $prefix) {
  357. $exists = 1;
  358. last;
  359. }
  360. }
  361. closedir(SEASON);
  362. }
  363. if ($exists) {
  364. next;
  365. }
  366. # Save all the date string components
  367. $years{$year} = 1;
  368. $months{$month} = 1;
  369. $days{$day} = 1;
  370. # Create the relevent search strings
  371. my $search_str = $search_by_date;
  372. $search_str =~ s/%Y/${year}/g;
  373. $search_str =~ s/%m/${month}/g;
  374. $search_str =~ s/%d/${day}/g;
  375. foreach my $key (keys(%{$SOURCES})) {
  376. my $source = $SOURCES->{$key};
  377. push(@urls, $source->{'search_url'} . $search_str . $exclude . $source->{'search_suffix'});
  378. }
  379. }
  380. # Build a date string matching regex
  381. my $str = '\b(?:' . join('|', keys(%years)) . ')\b';
  382. $str .= '.*';
  383. $str .= '\b(?:' . join('|', keys(%months)) . ')\b';
  384. $str .= '.*';
  385. $str .= '\b(?:' . join('|', keys(%days)) . ')\b';
  386. $CUSTOM_SEARCH = qr/${str}/;
  387. # Debug
  388. if ($DEBUG) {
  389. print STDERR 'Searching with date template: ' . $str . "\n";
  390. }
  391. }
  392. # Handle standard series
  393. if (scalar(@urls) < 1) {
  394. # Validate the season number
  395. if (!defined($season) || $season < 1 || $season > 2000) {
  396. die('Invalid season number: ' . $show . ' => ' . $season . "\n");
  397. }
  398. # Get the last episode number
  399. my $no_next = 0;
  400. my %episodes = ();
  401. my $highest = 0;
  402. opendir(SEASON, $dir . '/Season ' . $season)
  403. or die("Unable to open season directory: ${!}\n");
  404. while (my $file = readdir(SEASON)) {
  405. # Skip support files
  406. if ($file =~ /\.(?:png|xml|jpg|gif|tbn|txt|nfo|torrent)\s*$/i) {
  407. next;
  408. }
  409. # Check for a season_done file
  410. if ($file eq 'season_done') {
  411. $no_next = 1;
  412. next;
  413. }
  414. # Extract the episode number
  415. my ($num) = $file =~ /^\s*\.?(\d+)\s*\-\s*/i;
  416. if (defined($num)) {
  417. $num = int($num);
  418. if ($DEBUG) {
  419. print STDERR 'Found episode number: ' . $num . ' in file: ' . $file . "\n";
  420. }
  421. # Record it
  422. $episodes{$num} = 1;
  423. # Track the highest episode number
  424. if ($num > $highest) {
  425. $highest = $num;
  426. }
  427. }
  428. }
  429. close(SEASON);
  430. # Assume we need the next 2 episodes, unless no_next is set (i.e. season_done)
  431. if (!$no_next) {
  432. for (my $i = 1 ; $i <= $NEXT_EPISODES ; $i++) {
  433. push(@need, $highest + $i);
  434. }
  435. }
  436. # Find any missing episodes
  437. for (my $i = 1 ; $i <= $highest ; $i++) {
  438. if (!$episodes{$i}) {
  439. push(@need, $i);
  440. }
  441. }
  442. if ($DEBUG) {
  443. print STDERR 'Needed episodes: ' . join(', ', @need) . "\n";
  444. }
  445. # Reverse the array for later matching
  446. foreach my $episode (@need) {
  447. $need{$episode} = 1;
  448. }
  449. # Construct a URL-friendly show name
  450. my $safeShow = $show;
  451. $safeShow =~ s/\s+\&\s+/ and /i;
  452. $safeShow =~ s/^\s*The\b//i;
  453. $safeShow =~ s/\s+\-\s+/ /g;
  454. $safeShow =~ s/[\'\:]//g;
  455. $safeShow =~ s/[^\w\"\-]+/ /g;
  456. $safeShow =~ s/^\s+//;
  457. $safeShow =~ s/\s+$//;
  458. $safeShow =~ s/\s\s+/ /g;
  459. $safeShow =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
  460. $safeShow =~ s/\%20/\+/g;
  461. # Calculate possible name variations
  462. my @urlShowVarients = ();
  463. {
  464. # Default name
  465. push(@urlShowVarients, $safeShow);
  466. # Search for both "and" and "&"
  467. if ($safeShow =~ /\+and\+/i) {
  468. my $tmp = $safeShow;
  469. $tmp =~ s/\+and\+/\+%26\+/;
  470. push(@urlShowVarients, $tmp);
  471. }
  472. }
  473. # Construct the URL for each title varient of each needed episode
  474. foreach my $urlShow (@urlShowVarients) {
  475. foreach my $episode (@need) {
  476. my $episode_long = sprintf('%02d', $episode);
  477. my $season_long = sprintf('%02d', $season);
  478. foreach my $source (values(%{$SOURCES})) {
  479. # Allow custom handling
  480. if (defined($source->{'custom_search'}) && ref($source->{'custom_search'}) eq 'CODE') {
  481. $source->{'custom_search'}->(\@urls, $urlShow, $season, $episode);
  482. next;
  483. }
  484. # Use quotes around the show name if the source needs them
  485. my $quote = '%22';
  486. if (!$source->{'quote'}) {
  487. $quote = '';
  488. }
  489. # Calculate the compete search prefix and suffix to simplify later concatenations
  490. my $prefix = $source->{'search_url'} . $quote . $urlShow . $quote;
  491. my $suffix = '';
  492. if ($source->{'search_exclude'}) {
  493. $suffix .= $exclude;
  494. }
  495. if ($source->{'search_suffix'}) {
  496. $suffix .= $source->{'search_suffix'};
  497. }
  498. # SXXEYY
  499. my $url = $prefix . '+s' . $season_long . 'e' . $episode_long . $suffix;
  500. push(@urls, $url);
  501. # Extra searches for shows that have lazy/non-standard number formats
  502. if ($MORE_NUMBER_FORMATS) {
  503. # SXEY
  504. if ($season_long ne $season || $episode_long ne $episode) {
  505. $url = $prefix . '+s' . $season . 'e' . $episode . $suffix;
  506. push(@urls, $url);
  507. }
  508. # SXX EYY
  509. $url = $prefix . '+s' . $season_long . '+e' . $episode_long . $suffix;
  510. push(@urls, $url);
  511. # Season XX Episode YY
  512. $url = $prefix . '+season+' . $season_long . '+episode+' . $episode_long . $suffix;
  513. push(@urls, $url);
  514. # Series X Episode Y
  515. $url = $prefix . '+series+' . $season . '+episode+' . $episode . $suffix;
  516. push(@urls, $url);
  517. # SxEE
  518. $url = $prefix . '+' . $season . 'x' . $episode_long . $suffix;
  519. push(@urls, $url);
  520. # Season X
  521. if ($NO_QUALITY_CHECKS) {
  522. $url = $prefix . '+Season+' . $season . $suffix;
  523. push(@urls, $url);
  524. }
  525. }
  526. }
  527. }
  528. }
  529. }
  530. my @html_content = ();
  531. my @json_content = ();
  532. foreach my $url (@urls) {
  533. # Fetch the page
  534. my $errCount = 0;
  535. my $content = '';
  536. HTTP_ERR_LOOP: {
  537. my $code = 0;
  538. if ($DEBUG) {
  539. print STDERR 'Searching with URL: ' . $url . "\n";
  540. }
  541. # Fetch
  542. ($content, $code) = fetch($url, 'primary.html');
  543. # Check for useful errors
  544. if ($code == 404) {
  545. if ($DEBUG) {
  546. print STDERR "Skipping content from 404 response\n";
  547. }
  548. next;
  549. }
  550. # Check for less useful errors
  551. if ($code != 200) {
  552. if ($code >= 500 && $code <= 599) {
  553. if ($errCount > $ERR_RETRIES) {
  554. print STDERR 'Unable to fetch URL: ' . $url . "\n";
  555. $errCount = 0;
  556. next;
  557. }
  558. if ($DEBUG) {
  559. print STDERR 'Retrying URL (' . $code . '): ' . $url . "\n";
  560. }
  561. $errCount++;
  562. sleep($ERR_DELAY);
  563. redo HTTP_ERR_LOOP;
  564. } else {
  565. print STDERR 'Error fetching URL (' . $code . '): ' . $url . "\n";
  566. }
  567. next;
  568. }
  569. }
  570. # Save the content, discriminating by data type
  571. if ($DEBUG) {
  572. print STDERR 'Fetched ' . length($content) . " bytes\n";
  573. }
  574. if ($content =~ /^\s*{/) {
  575. my $json = eval { decode_json($content); };
  576. if (defined($json) && ref($json)) {
  577. push(@json_content, $json);
  578. } elsif ($DEBUG) {
  579. print STDERR 'JSON parsing failure on: ' . $content . "\n";
  580. }
  581. } else {
  582. push(@html_content, scalar($content));
  583. }
  584. }
  585. # We need someplace to store parsed torrent data from the various sources
  586. my @tors = ();
  587. # Handle HTML content
  588. foreach my $content (@html_content) {
  589. if ($content =~ /\<title\>The Pirate Bay/i) {
  590. # Find TR elements from TPB
  591. my @trs = splitTags($content, 'TR', 'TH', undef());
  592. foreach my $tr (@trs) {
  593. # Find the show title
  594. my ($title) = $tr =~ /title\=\"Details\s+for\s+([^\"]*)\"/i;
  595. if (!defined($title) || length($title) < 1) {
  596. if ($DEBUG) {
  597. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  598. }
  599. next;
  600. }
  601. $title =~ s/^\s+//;
  602. # Extract the season and episode numbers
  603. my ($fileSeason, $episode) = findSE($title);
  604. # Extract the URL
  605. my ($url) = $tr =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  606. if (!defined($url) || length($url) < 1) {
  607. if ($DEBUG) {
  608. print STDERR "Skipping TR with no magnet URL\n";
  609. }
  610. next;
  611. }
  612. $url = decode_entities($url);
  613. # Count the sum of seeders and leachers
  614. my $seeds = 0;
  615. my $leaches = 0;
  616. if ($tr =~ /\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*$/i) {
  617. $seeds = $1;
  618. $leaches = $2;
  619. }
  620. # Extract the size (from separate column or inline)
  621. my $size = 0;
  622. my $unit = 'M';
  623. if ($tr =~ m/Size (\d+(?:\.\d+)?)\&nbsp\;(G|M)iB/) {
  624. $size = $1;
  625. $unit = $2;
  626. } elsif ($tr =~ m/(\d+(?:\.\d+)?)\&nbsp\;(G|M)iB\<\/[tT][dD]\>/) {
  627. $size = $1;
  628. $unit = $2;
  629. }
  630. if ($unit eq 'G') {
  631. $size *= 1024;
  632. }
  633. $size = int($size);
  634. if ($DEBUG) {
  635. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  636. }
  637. # Save the extracted data
  638. my %tor = (
  639. 'title' => $title,
  640. 'season' => $fileSeason,
  641. 'episode' => $episode,
  642. 'seeds' => $seeds,
  643. 'leaches' => $leaches,
  644. 'size' => $size,
  645. 'url' => $url,
  646. 'source' => 'TPB'
  647. );
  648. push(@tors, \%tor);
  649. }
  650. } elsif ($content =~ /\<title\>[^\<\>]*\bisoHunt\b[^\<\>]*\<\/title\>/i) {
  651. # Find each TR element from ISOHunt
  652. my @trs = splitTags($content, 'TR', 'TH', undef());
  653. foreach my $tr (@trs) {
  654. # Split each TD element from the row
  655. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  656. if (scalar(@tds) > 10 || scalar(@tds) < 9) {
  657. if ($tds[1] =~ /No results found/) {
  658. next;
  659. }
  660. print STDERR 'Skipping invalid ISO TR: ' . $tr . "\n";
  661. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  662. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  663. }
  664. next;
  665. }
  666. # Skip sponsored results
  667. if ($tds[1] =~ /\bSponsored\b/i) {
  668. if ($DEBUG > 1) {
  669. print STDERR "Skipping sponsored result\n";
  670. }
  671. next;
  672. }
  673. # Find the torrent ID and show title
  674. my ($id, $title) = $tds[2] =~ /\<a(?:\s+[^\>]*)?href=\"\/torrent_details\/(\d+\/[^\"]+)\"\>\<span\>([^\>]+)\<\/span\>/i;
  675. if (!defined($id) || length($id) < 1 || !defined($title) || length($title) < 1) {
  676. if ($DEBUG > 1) {
  677. print STDERR "Unable to find show ID or title in TD:\n\t" . $tds[2] . "\n";
  678. }
  679. next;
  680. }
  681. $title =~ s/^\s+//;
  682. # Extract the season and episode numbers
  683. my ($fileSeason, $episode) = findSE($title);
  684. # Extract the size
  685. my $size = 0;
  686. my $unit = 'M';
  687. if ($tds[6] =~ m/(\d+(?:\.\d+)?) (G|M)B<\/td\>/i) {
  688. $size = $1;
  689. $unit = $2;
  690. }
  691. if ($unit eq 'G') {
  692. $size *= 1024;
  693. }
  694. $size = int($size);
  695. # Count the sum of seeders and leachers
  696. my $seeds = 0;
  697. if ($tds[7] =~ /(\d+)\<\/td\>/i) {
  698. $seeds = $1;
  699. }
  700. my $leaches = 0;
  701. if ($tds[8] =~ /(\d+)\<\/td\>/i) {
  702. $leaches = $1;
  703. }
  704. # Build the detail page URL
  705. my $url = $SOURCES->{'ISO'}->{'protocol'} . '://' . $SOURCES->{'ISO'}->{'host'} . '/torrent_details/' . $id;
  706. if ($DEBUG) {
  707. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  708. }
  709. # Save the extracted data
  710. my %tor = (
  711. 'title' => $title,
  712. 'season' => $fileSeason,
  713. 'episode' => $episode,
  714. 'seeds' => $seeds,
  715. 'leaches' => $leaches,
  716. 'size' => $size,
  717. 'url' => $url,
  718. 'source' => 'ISO'
  719. );
  720. push(@tors, \%tor);
  721. }
  722. } elsif ($content =~ /Kickass\s*Torrents\<\/title\>/i) {
  723. # Find each TR element from Kickass
  724. my @trs = splitTags($content, 'TR', 'TH', undef());
  725. foreach my $tr (@trs) {
  726. # Split each TD element from the row
  727. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  728. if (scalar(@tds) != 6) {
  729. print STDERR "Invalid KICK TR:\n";
  730. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  731. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  732. }
  733. next;
  734. }
  735. # Skip empty results
  736. if ( $tds[1] =~ /\<h2\>Nothing found\!\<\/h2\>/i
  737. || $tds[1] =~ /\<p\>\<strong\>Page not found\<\/strong\>\<\/p\>/i)
  738. {
  739. if ($DEBUG) {
  740. print STDERR "Skipping empty TD\n";
  741. }
  742. next;
  743. }
  744. # Find the torrent title
  745. my ($title) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href\=\"[^\"]+\?title\=\[[^\]\"]+\]([^\"]+)\"/i;
  746. if (!defined($title) || length($title) < 1) {
  747. if ($DEBUG) {
  748. print STDERR "Unable to find torrent title in TD:\n\t" . $tds[1] . "\n";
  749. }
  750. next;
  751. }
  752. $title =~ s/^\s+//;
  753. # Find the torrent URL
  754. my ($url) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href=\"(magnet:[^\"]+)\"/i;
  755. if (!defined($url) || length($url) < 1) {
  756. if ($DEBUG) {
  757. print STDERR "Unable to find torrent URL in TD:\n\t" . $tds[1] . "\n";
  758. }
  759. next;
  760. }
  761. $url = decode_entities($url);
  762. # Extract the season and episode numbers
  763. my ($fileSeason, $episode) = findSE($title);
  764. # Extract the size
  765. my $size = 0;
  766. my $unit = 'M';
  767. if ($tds[2] =~ m/(\d+(?:\.\d+)?)\s+\<span\>(G|M)B\<\/span\>\<\/td\>/i) {
  768. $size = $1;
  769. $unit = $2;
  770. }
  771. if ($unit eq 'G') {
  772. $size *= 1024;
  773. }
  774. $size = int($size);
  775. # Count the sum of seeders and leachers
  776. my $seeds = 0;
  777. if ($tds[4] =~ /(\d+)\<\/td\>/i) {
  778. $seeds = $1;
  779. }
  780. my $leaches = 0;
  781. if ($tds[5] =~ /(\d+)\<\/td\>/i) {
  782. $leaches = $1;
  783. }
  784. if ($DEBUG) {
  785. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  786. }
  787. # Save the extracted data
  788. my %tor = (
  789. 'title' => $title,
  790. 'season' => $fileSeason,
  791. 'episode' => $episode,
  792. 'seeds' => $seeds,
  793. 'leaches' => $leaches,
  794. 'size' => $size,
  795. 'url' => $url,
  796. 'source' => 'KICK'
  797. );
  798. push(@tors, \%tor);
  799. }
  800. } elsif ($content =~ /\<a(?:\s+[^\>]*)?\>Torrentz\<\/a\>/i) {
  801. # We'll need to strip HTML
  802. my $hs = HTML::Strip->new();
  803. # Find each DL element from Torrentz
  804. my @dls = splitTags($content, 'DL', undef(), undef());
  805. foreach my $dl (@dls) {
  806. # Skip ads
  807. if ($dl =~ /\s+rel\=\"nofollow/i) {
  808. next;
  809. }
  810. # Find the torrent title
  811. my ($hash, $title) = $dl =~ /\<a href\=\"\/(\w+)\">(.*)\<\/a\>/i;
  812. if (!defined($hash) || length($hash) < 1 || !defined($title) || length($title) < 1) {
  813. if ($DEBUG) {
  814. print STDERR "Unable to find torrent title in DL:\n\t" . $dl . "\n";
  815. }
  816. next;
  817. }
  818. $hash = lc($hash);
  819. $title = $hs->parse($title);
  820. $hs->eof;
  821. $title =~ s/^\s+//;
  822. # Extract the season and episode numbers
  823. my ($fileSeason, $episode) = findSE($title);
  824. # Extract the size
  825. my $size = 0;
  826. if ($dl =~ /\<span class\=\"s\">(\d+(?:\.\d+)?) (G|M)B\<\/span\>/i) {
  827. $size = $1;
  828. if ($2 eq 'G') {
  829. $size *= 1024;
  830. }
  831. $size = int($size);
  832. }
  833. # Count the sum of seeders and leachers
  834. my $seeds = 0;
  835. if ($dl =~ /<span class\=\"u\">(\d+)\<\/span\>/i) {
  836. $seeds = $1;
  837. }
  838. my $leaches = 0;
  839. if ($dl =~ /<span class\=\"d\">(\d+)\<\/span\>/i) {
  840. $leaches = $1;
  841. }
  842. # Construct a magnet URL from the hash
  843. # Assume the seconary lookup will append a list of trackers
  844. my $url = 'magnet:?xt=urn:btih:' . $hash . '&dn=' . uri_encode($title, { 'encode_reserved' => 1 });
  845. if ($DEBUG) {
  846. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  847. }
  848. # Save the extracted data
  849. my %tor = (
  850. 'title' => $title,
  851. 'season' => $fileSeason,
  852. 'episode' => $episode,
  853. 'seeds' => $seeds,
  854. 'leaches' => $leaches,
  855. 'size' => $size,
  856. 'url' => $url,
  857. 'source' => 'Z'
  858. );
  859. push(@tors, \%tor);
  860. }
  861. } elsif ($content =~ /ExtraTorrent\.cc The World\'s Largest BitTorrent System\<\/title\>/i) {
  862. # Find TR elements from ET
  863. my @trs = splitTags($content, 'TR', undef(), 'magnet\:');
  864. foreach my $tr (@trs) {
  865. # Find the show title
  866. my ($title) = $tr =~ /title\=\"Download\s+([^\"]*)\s+torrent\"/i;
  867. if (!defined($title) || length($title) < 1) {
  868. if ($DEBUG) {
  869. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  870. }
  871. next;
  872. }
  873. $title =~ s/^\s+//;
  874. # Find the magnet URL
  875. my ($url) = $tr =~ /\<a\s+href=\"(magnet\:[^\"]+)\"/i;
  876. if (!defined($url) || length($url) < 1) {
  877. if ($DEBUG) {
  878. print STDERR "Skipping TR with no magnet URL\n";
  879. }
  880. next;
  881. }
  882. $url = decode_entities($url);
  883. # Extract the season and episode numbers
  884. my ($fileSeason, $episode) = findSE($title);
  885. # Count the sum of seeders and leachers
  886. my $seeds = 0;
  887. my $leaches = 0;
  888. if ($tr =~ /\<td\s+class=\"sy\">(\d+)\<\/td\>/) {
  889. $seeds = $1;
  890. }
  891. if ($tr =~ /\<td\s+class=\"ly\">(\d+)\<\/td\>/) {
  892. $leaches = $1;
  893. }
  894. # Extract the size (from separate column or inline)
  895. my $size = 0;
  896. my $unit = 'M';
  897. if ($tr =~ /\<td\>(\d+(?:\.\d+)?)\&nbsp\;(G|M)B\<\/td\>/) {
  898. $size = $1;
  899. $unit = $2;
  900. }
  901. if ($unit eq 'G') {
  902. $size *= 1024;
  903. }
  904. $size = int($size);
  905. if ($DEBUG) {
  906. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  907. }
  908. # Save the extracted data
  909. my %tor = (
  910. 'title' => $title,
  911. 'season' => $fileSeason,
  912. 'episode' => $episode,
  913. 'seeds' => $seeds,
  914. 'leaches' => $leaches,
  915. 'size' => $size,
  916. 'url' => $url,
  917. 'source' => 'ET'
  918. );
  919. push(@tors, \%tor);
  920. }
  921. } elsif ($content =~ /\<title\>Sorry/) {
  922. if ($DEBUG) {
  923. print STDERR "ISO offline\n";
  924. }
  925. } elsif ($content =~ /Database\s+maintenance/i) {
  926. if ($DEBUG) {
  927. print STDERR "TPB offline\n";
  928. }
  929. } elsif ($content =~ /^\s*$/) {
  930. if ($DEBUG) {
  931. print STDERR "Source returned no content\n";
  932. }
  933. } else {
  934. print STDERR "Unknown HTML content:\n" . $content . "\n\n";
  935. }
  936. }
  937. # Handle JSON content
  938. foreach my $content (@json_content) {
  939. if (exists($content->{'title'}) && $content->{'title'} =~ /NZBCat/i) {
  940. my $list = $content->{'item'};
  941. if (!$list || ref($list) ne 'ARRAY') {
  942. warn("Invalid NCAT list\n");
  943. next;
  944. }
  945. foreach my $item (@{$list}) {
  946. # Ensure the record is sensible
  947. my $id = undef();
  948. if ( ref($item) eq 'HASH'
  949. && exists($item->{'guid'})
  950. && ref($item->{'guid'}) eq 'HASH'
  951. && exists($item->{'guid'}->{'text'}))
  952. {
  953. $id = $item->{'guid'}->{'text'};
  954. }
  955. if (!$id) {
  956. warn("Unable to parse NCAT list item\n");
  957. next;
  958. }
  959. # NZB URL
  960. my $url = undef();
  961. if (exists($item->{'link'}) && $item->{'link'}) {
  962. $url = $item->{'link'};
  963. $url =~ s/^(https?)\:/${PROTOCOL}\:/;
  964. }
  965. # Title
  966. my $title = undef();
  967. if (exists($item->{'title'}) && $item->{'title'}) {
  968. $title = $item->{'title'};
  969. } elsif (exists($item->{'description'}) && $item->{'description'}) {
  970. $title = $item->{'description'};
  971. }
  972. if ($title) {
  973. $url .= '#' . $title;
  974. }
  975. # Extract the season and episode numbers
  976. my ($season, $episode) = findSE($title);
  977. # Size
  978. my $size = 0;
  979. if (exists($item->{'newznab:attr'}) && $item->{'newznab:attr'} && ref($item->{'newznab:attr'}) eq 'ARRAY') {
  980. foreach my $hash (@{ $item->{'newznab:attr'} }) {
  981. if (ref($hash) eq 'HASH' && exists($hash->{'_name'}) && exists($hash->{'_value'}) && $hash->{'_name'} eq 'size')
  982. {
  983. $size = int($hash->{'_value'}) / 1024;
  984. }
  985. }
  986. }
  987. # Date
  988. my $date = undef();
  989. if (exists($item->{'pubDate'}) && $item->{'pubDate'}) {
  990. $date = str2time($item->{'pubDate'});
  991. }
  992. # Sanity checks
  993. if (!$url) {
  994. warn('No URL in NCAT item: ' . $id . "\n");
  995. next;
  996. }
  997. if (!$title) {
  998. warn('No title in NCAT item: ' . $id . "\n");
  999. next;
  1000. }
  1001. if (!$size) {
  1002. warn('No size in NCAT item: ' . $id . "\n");
  1003. next;
  1004. }
  1005. if ($DEBUG) {
  1006. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  1007. }
  1008. # Save the extracted data
  1009. my %tor = (
  1010. 'title' => $title,
  1011. 'season' => $season,
  1012. 'episode' => $episode,
  1013. 'date' => $date,
  1014. 'size' => $size,
  1015. 'url' => $url,
  1016. 'source' => 'NCAT'
  1017. );
  1018. push(@tors, \%tor);
  1019. }
  1020. } else {
  1021. print STDERR "Unknown JSON content:\n" . $content . "\n\n";
  1022. }
  1023. }
  1024. # Filter for size/count/etc.
  1025. my %tors = ();
  1026. my $showRegex = undef();
  1027. {
  1028. my $showClean = $show;
  1029. $showClean =~ s/[\"\']//g;
  1030. $showClean =~ s/[\W_]+/\[\\W_\].*/g;
  1031. $showRegex = qr/^${showClean}[\W_]/i;
  1032. }
  1033. foreach my $tor (@tors) {
  1034. # Extract the BTIH, if available
  1035. getHash($tor);
  1036. # Skip files that are in the torrent excludes list
  1037. if ($tor->{'hash'} && $EXCLUDES{ $tor->{'hash'} }) {
  1038. if ($DEBUG) {
  1039. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1040. }
  1041. next;
  1042. }
  1043. # Skip files that don't start with our show title
  1044. if (!($tor->{'title'} =~ $showRegex)) {
  1045. if ($DEBUG) {
  1046. print STDERR 'Skipping file: Title does not match (' . $showRegex . '): ' . $tor->{'title'} . "\n";
  1047. }
  1048. next;
  1049. # Skip pre-air files
  1050. } elsif ($tor->{'title'} =~ /preair/i) {
  1051. if ($DEBUG) {
  1052. print STDERR 'Skipping file: Title contains "preair": ' . $tor->{'title'} . "\n";
  1053. }
  1054. next;
  1055. # Skip SWESUB files
  1056. } elsif ($tor->{'title'} =~ /swesub/i) {
  1057. if ($DEBUG) {
  1058. print STDERR 'Skipping file: Title contains "SWESUB": ' . $tor->{'title'} . "\n";
  1059. }
  1060. next;
  1061. # Skip .rus. files
  1062. } elsif ($tor->{'title'} =~ /\.rus\./i) {
  1063. if ($DEBUG) {
  1064. print STDERR 'Skipping file: Title contains ".rus.": ' . $tor->{'title'} . "\n";
  1065. }
  1066. next;
  1067. # Skip German files
  1068. } elsif ($tor->{'title'} =~ /german/i) {
  1069. if ($DEBUG) {
  1070. print STDERR 'Skipping file: Title contains "german": ' . $tor->{'title'} . "\n";
  1071. }
  1072. next;
  1073. # Skip French files
  1074. } elsif ($tor->{'title'} =~ /french/i) {
  1075. if ($DEBUG) {
  1076. print STDERR 'Skipping file: Title contains "french": ' . $tor->{'title'} . "\n";
  1077. }
  1078. next;
  1079. # Skip Spanish files
  1080. } elsif ($tor->{'title'} =~ /Español/i) {
  1081. if ($DEBUG) {
  1082. print STDERR 'Skipping file: Title contains "Español": ' . $tor->{'title'} . "\n";
  1083. }
  1084. next;
  1085. # Skip Castellano files
  1086. } elsif ($tor->{'title'} =~ /Castellano/i) {
  1087. if ($DEBUG) {
  1088. print STDERR 'Skipping file: Title contains "Castellano": ' . $tor->{'title'} . "\n";
  1089. }
  1090. next;
  1091. # Skip unedited files
  1092. } elsif ($tor->{'title'} =~ /unedited/i) {
  1093. if ($DEBUG) {
  1094. print STDERR 'Skipping file: Title contains "unedited": ' . $tor->{'title'} . "\n";
  1095. }
  1096. next;
  1097. }
  1098. # Skip files that contain a word from the series exclude list
  1099. my $exclude = undef();
  1100. foreach my $ex (keys(%TITLE_EXCLUDES)) {
  1101. $ex = quotemeta($ex);
  1102. if ($tor->{'title'} =~ m/${ex}/i) {
  1103. $exclude = $ex;
  1104. last;
  1105. }
  1106. }
  1107. if (defined($exclude)) {
  1108. if ($DEBUG) {
  1109. print STDERR 'Skipping file: Title contains "' . $exclude . '": ' . $tor->{'title'} . "\n";
  1110. }
  1111. next;
  1112. }
  1113. # Enforce season and episode number matches for standard searches, or CUSTOM_SEARCH matching (if it's a regex)
  1114. if (!$CUSTOM_SEARCH) {
  1115. # Skip files that don't contain the right season number
  1116. if (!defined($tor->{'season'}) || $tor->{'season'} != $season) {
  1117. if ($DEBUG) {
  1118. print STDERR 'Skipping file: No match for season number (' . $season . '): ' . $tor->{'title'} . "\n";
  1119. }
  1120. next;
  1121. # Skip files that don't contain a needed episode number, unless there is no episode number and NO_QUALITY_CHECKS is set
  1122. } elsif ((defined($tor->{'episode'}) && !$need{ $tor->{'episode'} })
  1123. || (!defined($tor->{'episode'}) && !$NO_QUALITY_CHECKS))
  1124. {
  1125. if ($DEBUG) {
  1126. print STDERR 'Skipping file: No match for episode number (' . $tor->{'episode'} . '): ' . $tor->{'title'} . "\n";
  1127. }
  1128. next;
  1129. }
  1130. } elsif (ref($CUSTOM_SEARCH) eq 'Regexp') {
  1131. # Skip files that don't match the regex
  1132. if (!($tor->{'title'} =~ $CUSTOM_SEARCH)) {
  1133. if ($DEBUG) {
  1134. print STDERR 'Skipping file: No match for CUSTOM_SEARCH regex (' . $CUSTOM_SEARCH . '): ' . $tor->{'title'} . "\n";
  1135. }
  1136. next;
  1137. }
  1138. }
  1139. # Only apply the quality rules if NO_QUALITY_CHECKS is not in effect
  1140. if (!$NO_QUALITY_CHECKS) {
  1141. # Cap seeds/leaches/date
  1142. if (!exists($tor->{'seeds'}) || !$tor->{'seeds'}) {
  1143. $tor->{'seeds'} = 0;
  1144. }
  1145. if ($tor->{'seeds'} > $MAX_SEEDS) {
  1146. $tor->{'seeds'} = $MAX_SEEDS;
  1147. }
  1148. if (!exists($tor->{'leaches'}) || !$tor->{'leaches'}) {
  1149. $tor->{'leaches'} = 0;
  1150. }
  1151. if ($tor->{'leaches'} > $MAX_SEEDS) {
  1152. $tor->{'leaches'} = $MAX_SEEDS;
  1153. }
  1154. # Proxy publication date to seeder/leacher quality
  1155. if ($tor->{'date'} && !$tor->{'seeds'}) {
  1156. my $min = ($MIN_COUNT / 2) - 1;
  1157. my $penalty = (time() - $tor->{'date'}) * $AGE_PENALTY;
  1158. $tor->{'seeds'} = $MAX_SEEDS - $penalty;
  1159. if ($tor->{'seeds'} < $min) {
  1160. $tor->{'seeds'} = $min;
  1161. }
  1162. $tor->{'leaches'} = $tor->{'seeds'};
  1163. }
  1164. # Skip torrents with too few seeders/leachers
  1165. if (($tor->{'seeds'} + $tor->{'leaches'}) * $SOURCES->{ $tor->{'source'} }->{'weight'} < $MIN_COUNT) {
  1166. if ($DEBUG) {
  1167. print STDERR 'Skipping file: Insufficient seeder/leacher count ('
  1168. . $tor->{'seeds'} . '/'
  1169. . $tor->{'leaches'} . '): '
  1170. . $tor->{'title'} . "\n";
  1171. }
  1172. next;
  1173. # Skip torrents with unusual seed/leach ratios
  1174. } elsif ($tor->{'seeds'} > 1
  1175. && $tor->{'seeds'} < $SEED_RATIO_COUNT
  1176. && $tor->{'seeds'} > $tor->{'leaches'} * $MAX_SEED_RATIO)
  1177. {
  1178. if ($DEBUG) {
  1179. print STDERR 'Skipping file: Unusual seeder/leacher ratio ('
  1180. . $tor->{'seeds'} . '/'
  1181. . $tor->{'leaches'} . '): '
  1182. . $tor->{'title'} . "\n";
  1183. }
  1184. next;
  1185. # Skip torrents that are too small
  1186. } elsif ($tor->{'size'} < $MIN_SIZE) {
  1187. if ($DEBUG) {
  1188. print STDERR 'Skipping file: Insufficient size (' . $tor->{'size'} . ' MiB): ' . $tor->{'title'} . "\n";
  1189. }
  1190. next;
  1191. }
  1192. }
  1193. # Save good torrents
  1194. push(@{ $tors{ $tor->{'episode'} } }, $tor);
  1195. if ($DEBUG) {
  1196. print STDERR 'Possible URL ('
  1197. . $tor->{'seeds'} . '/'
  1198. . $tor->{'leaches'}
  1199. . ' seeds/leaches, '
  1200. . $tor->{'size'}
  1201. . ' MiB): '
  1202. . $tor->{'title'} . "\n";
  1203. }
  1204. }
  1205. # Find the average torrent size for each episode
  1206. my %size = ();
  1207. {
  1208. my %max = ();
  1209. my %avg = ();
  1210. foreach my $episode (keys(%tors)) {
  1211. my $count = 0;
  1212. $avg{$episode} = 0.001;
  1213. $max{$episode} = 0.001;
  1214. foreach my $tor (@{ $tors{$episode} }) {
  1215. $count++;
  1216. $avg{$episode} += $tor->{'size'};
  1217. if ($tor->{'size'} > $max{$episode}) {
  1218. $max{$episode} = $tor->{'size'};
  1219. }
  1220. }
  1221. if ($count > 0) {
  1222. $avg{$episode} /= $count;
  1223. }
  1224. $size{$episode} = ($max{$episode} + $avg{$episode}) / 2;
  1225. if ($DEBUG) {
  1226. print STDERR 'Episode '
  1227. . $episode
  1228. . ' max/avg/cmp size: '
  1229. . int($max{$episode}) . '/'
  1230. . int($avg{$episode}) . '/'
  1231. . int($size{$episode})
  1232. . " MiB\n";
  1233. }
  1234. }
  1235. }
  1236. # Calculate an adjusted count based on the peer count, relative file size, and title contents
  1237. foreach my $episode (keys(%tors)) {
  1238. foreach my $tor (@{ $tors{$episode} }) {
  1239. my $count = $tor->{'seeds'} + $tor->{'leaches'};
  1240. # Start with the peer count
  1241. $tor->{'adj_count'} = $count;
  1242. # Adjust based on file size
  1243. {
  1244. my $size_ratio = $tor->{'size'} / $size{$episode};
  1245. if ($tor->{'size'} >= $size{$episode}) {
  1246. $tor->{'adj_count'} *= $SIZE_BONUS * $size_ratio;
  1247. } else {
  1248. $tor->{'adj_count'} *= (1 / $SIZE_PENALTY) * $size_ratio;
  1249. }
  1250. }
  1251. # Adjust based on title contents
  1252. if ($tor->{'title'} =~ /Subtitulado/i) {
  1253. $tor->{'adj_count'} *= 1 / $TITLE_PENALTY;
  1254. }
  1255. # Truncate to an integer
  1256. $tor->{'adj_count'} = int($tor->{'adj_count'});
  1257. if ($DEBUG) {
  1258. print STDERR 'Possible URL (' . $tor->{'adj_count'} . ' size-adjusted sources): ' . $tor->{'url'} . "\n";
  1259. }
  1260. }
  1261. }
  1262. # Pick the best-adjusted-count torrent for each episode
  1263. my %urls = ();
  1264. foreach my $episode (keys(%tors)) {
  1265. my @sorted = sort { $b->{'adj_count'} <=> $a->{'adj_count'} } @{ $tors{$episode} };
  1266. my $max = undef();
  1267. foreach my $tor (@sorted) {
  1268. if (!defined($max) || $tor->{'adj_count'} > $max) {
  1269. if ($urls{$episode} = resolveSecondary($tor)) {
  1270. if (!$tor->{'hash'} || $EXCLUDES{ $tor->{'hash'} }) {
  1271. print STDERR 'Double-skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1272. next;
  1273. }
  1274. $max = $tor->{'adj_count'};
  1275. }
  1276. if ($DEBUG) {
  1277. print STDERR 'Semi-final URL (adjusted count: ' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  1278. }
  1279. } elsif ($DEBUG) {
  1280. print STDERR 'Skipping for lesser adjusted count (' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  1281. }
  1282. }
  1283. }
  1284. # Output
  1285. foreach my $episode (keys(%tors)) {
  1286. if (defined($urls{$episode})) {
  1287. $urls{$episode} = resolveTrackers($urls{$episode});
  1288. print $urls{$episode} . "\n";
  1289. if ($SYSLOG) {
  1290. syslog(LOG_NOTICE, $show . ': ' . getHash($urls{$episode}));
  1291. }
  1292. if ($DEBUG) {
  1293. print STDERR 'Final URL: ' . $urls{$episode} . "\n";
  1294. }
  1295. } elsif ($DEBUG) {
  1296. print STDERR 'No URL for: ' . $episode . "\n";
  1297. }
  1298. }
  1299. # Cleanup
  1300. if ($COOKIES) {
  1301. unlink($COOKIES);
  1302. }
  1303. if ($SCRIPT) {
  1304. unlink($SCRIPT);
  1305. }
  1306. if ($SYSLOG) {
  1307. closelog();
  1308. }
  1309. exit(0);
  1310. sub getHash($) {
  1311. my ($tor) = @_;
  1312. my $url;
  1313. # Allow either a URL or a tor hashref
  1314. if (ref($tor) eq 'HASH') {
  1315. $url = $tor->{'url'};
  1316. } else {
  1317. $url = $tor;
  1318. undef($tor);
  1319. }
  1320. # Extract the BTIH hash or NZB ID, if available
  1321. my $hash = undef();
  1322. if ($url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1323. $hash = lc($1);
  1324. } elsif ($url =~ /\/getnzb\/(\w+)\.nzb\&/i) {
  1325. $hash = lc($1);
  1326. }
  1327. # Save back to the tor, if provided
  1328. if (defined($tor) && defined($hash)) {
  1329. $tor->{'hash'} = lc($hash);
  1330. }
  1331. # Always return the hash, or undef() if none can be found
  1332. return $hash;
  1333. }
  1334. # Resolve secondary URLs
  1335. sub resolveSecondary($) {
  1336. my ($tor) = @_;
  1337. # Fetch the torrent-specific page and extract the magent link
  1338. if ($tor->{'source'} eq 'ISO') {
  1339. my ($content, $code) = fetch($tor->{'url'}, 'secondary.html');
  1340. if ($code != 200) {
  1341. print STDERR 'Error fetching secondary URL: ' . $tor->{'url'} . "\n";
  1342. goto OUT;
  1343. }
  1344. my ($magnet) = $content =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  1345. if (!defined($magnet) || length($magnet) < 1) {
  1346. if ($DEBUG) {
  1347. print STDERR 'No secondary URL available from: ' . $tor->{'url'} . "\n";
  1348. }
  1349. goto OUT;
  1350. }
  1351. $tor->{'url'} = decode_entities($magnet);
  1352. }
  1353. # Extract the hash
  1354. getHash($tor);
  1355. if (!$tor->{'hash'} || $EXCLUDES{ $tor->{'hash'} }) {
  1356. if ($DEBUG) {
  1357. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1358. }
  1359. goto OUT;
  1360. }
  1361. OUT:
  1362. return $tor->{'url'};
  1363. }
  1364. sub resolveTrackers($) {
  1365. my ($url) = @_;
  1366. # Always append a few static trackers to the URI
  1367. if ($url =~ /^magnet\:/i) {
  1368. foreach my $tracker (@TRACKERS) {
  1369. my $arg = '&tr=' . uri_encode($tracker, { 'encode_reserved' => 1 });
  1370. my $match = quotemeta($arg);
  1371. if (!($url =~ /${match}/)) {
  1372. $url .= $arg;
  1373. }
  1374. }
  1375. }
  1376. # Fetch a dynamic tracker list for any hashinfo from TorrentZ
  1377. if ($TRACKER_LOOKUP && defined($SOURCES->{'Z'}) && $url =~ /^magnet\:/ && $url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1378. my $hash = lc($1);
  1379. my $baseURL = $SOURCES->{'Z'}->{'protocol'} . '://' . $SOURCES->{'Z'}->{'host'};
  1380. my $hashURL = $baseURL . '/' . $hash;
  1381. my ($content, $code) = fetch($hashURL, 'hashinfo.html');
  1382. if ($code != 200) {
  1383. print STDERR 'Error fetching hashinfo URL: ' . $hashURL . "\n";
  1384. return $url;
  1385. }
  1386. # Parse out and fetch the tracker list URL
  1387. my ($list) = $content =~ /\<a rel\=\"nofollow\" href\=\"(\/announcelist_\d+)\"\>/;
  1388. if (!defined($list)) {
  1389. if ($DEBUG) {
  1390. print STDERR 'Unable to find tracker list link at: ' . $hashURL . "\n";
  1391. }
  1392. return $url;
  1393. }
  1394. my $listURL = $baseURL . $list;
  1395. ($content, $code) = fetch($listURL, 'tracker.html');
  1396. if ($code != 200) {
  1397. print STDERR 'Error fetching tracker list URL: ' . $listURL . "\n";
  1398. return $url;
  1399. }
  1400. # Parse the tracker list and append our URL
  1401. foreach my $line ($content) {
  1402. if ($line =~ /^\s*$/) {
  1403. next;
  1404. }
  1405. $line =~ s/^\s+//;
  1406. $line =~ s/\s+$//;
  1407. if ($line =~ /^(?:http|udp)\:/i) {
  1408. my $arg = '&tr=' . uri_encode($line, { 'encode_reserved' => 1 });
  1409. my $match = quotemeta($arg);
  1410. if (!($url =~ /${match}/)) {
  1411. if ($DEBUG) {
  1412. print STDERR 'Adding tracker: ' . $line . "\n";
  1413. }
  1414. $url .= $arg;
  1415. }
  1416. } else {
  1417. print STDERR 'Unknown tracker type: ' . $line . "\n";
  1418. }
  1419. }
  1420. }
  1421. return $url;
  1422. }
  1423. sub splitTags($$$$) {
  1424. my ($content, $tag, $header, $match) = @_;
  1425. # Find each tag element
  1426. my @out = ();
  1427. my @parts = split(/\<${tag}(?:\s+[^\>]*)?\>/i, $content);
  1428. foreach my $part (@parts) {
  1429. # Trim trailing tags and skip things that aren't complete tags
  1430. if (!($part =~ s/\<\/${tag}\>.*$//is)) {
  1431. if ($DEBUG > 1) {
  1432. print STDERR 'Skipping non-tag line: ' . $part . "\n\n";
  1433. }
  1434. next;
  1435. }
  1436. # Skip headers
  1437. if (defined($header) && $part =~ /<${header}(?:\s+[^\>]*)?\>/i) {
  1438. if ($DEBUG > 2) {
  1439. print STDERR 'Skipping header line: ' . $part . "\n\n";
  1440. }
  1441. next;
  1442. }
  1443. # Skip non-matches
  1444. if (defined($match) && !($part =~ m/${match}/i)) {
  1445. if ($DEBUG > 2) {
  1446. print STDERR 'Skipping non-match line: ' . $part . "\n\n";
  1447. }
  1448. next;
  1449. }
  1450. # Save good tags
  1451. push(@out, $part);
  1452. }
  1453. return @out;
  1454. }
  1455. # Extract season
  1456. sub findSE($) {
  1457. my ($name) = @_;
  1458. my $season = 0;
  1459. my $episode = 0;
  1460. my $seasonBlock = '';
  1461. if ($name =~ /(?:\b|_)(20\d\d(?:\.|\-)[01]?\d(?:\.|\-)[0-3]?\d)(?:\b|_)/) {
  1462. $seasonBlock = $1;
  1463. my ($month, $day);
  1464. ($season, $month, $day) = $seasonBlock =~ /(20\d\d)(?:\.|\-)([01]?\d)(?:\.|\-)([0-3]?\d)/;
  1465. $episode = sprintf('%04d-%02d-%02d', $season, $month, $day);
  1466. } elsif ($name =~ /(?:\b|_)(S(?:eason)?[_\s\.\-]*\d{1,2}[_\s\.\-]*E(?:pisode)?[_\s\.]*\d{1,3}(?:E\d{1,3})?)(?:\b|_)/i) {
  1467. $seasonBlock = $1;
  1468. ($season, $episode) = $seasonBlock =~ /S(?:eason)?[_\s\.\-]*(\d{1,2})[_\s\.\-]*E(?:pisode)?[_\s\.\-]*(\d{1,3})/i;
  1469. } elsif ($name =~ /[\[\_\.](\d{1,2}x\d{2,3})[\]\_\.]/i) {
  1470. $seasonBlock = $1;
  1471. ($season, $episode) = $seasonBlock =~ /(\d+)x(\d+)/i;
  1472. } elsif ($name =~ /(?:\b|_)([01]?\d[_\s\.]?[0-3]\d)(?:\b|_)/i) {
  1473. $seasonBlock = $1;
  1474. ($season, $episode) = $seasonBlock =~ /(?:\b|_)([01]?\d)[_\s\.]?([0-3]\d)/i;
  1475. }
  1476. # Return something valid and INTy or UNDEF
  1477. if (!defined($seasonBlock) || $season < 1 || length($episode) < 1 || $episode eq '0') {
  1478. if ($DEBUG) {
  1479. print STDERR 'Could not find seasonBlock in: ' . $name . "\n";
  1480. }
  1481. $season = undef();
  1482. $episode = undef();
  1483. } elsif ($episode =~ /^20\d{2}\-\d{2}\-\d{2}$/) {
  1484. # Episode-by-date
  1485. $season = int($season);
  1486. } else {
  1487. # Catchall
  1488. # This will warn and return undef() (which is valid) on error
  1489. $season = int($season);
  1490. $episode = int($episode);
  1491. }
  1492. return ($season, $episode);
  1493. }
  1494. sub initSources() {
  1495. my %sources = ();
  1496. # NZB Cat
  1497. if ($ENABLE_SOURCE{'NCAT'}) {
  1498. my @proxies = ('nzb.cat/api');
  1499. my $source = findProxy(\@proxies, '\bforgottenpassword\b');
  1500. if ($source && exists($CONFIG{'NCAT_APIKEY'}) && $CONFIG{'NCAT_APIKEY'}) {
  1501. $source->{'weight'} = 1.00;
  1502. $source->{'quote'} = 0;
  1503. $source->{'search_exclude'} = 0;
  1504. $source->{'search_suffix'} = '';
  1505. $source->{'custom_search'} = sub ($$$$) {
  1506. my ($urls, $series, $season, $episode) = @_;
  1507. my $url =
  1508. $source->{'search_url'}
  1509. . '?o=json&t=tvsearch&q='
  1510. . $series
  1511. . '&season='
  1512. . $season . '&ep='
  1513. . $episode
  1514. . '&apikey='
  1515. . $CONFIG{'NCAT_APIKEY'};
  1516. push(@{$urls}, $url);
  1517. };
  1518. $sources{'NCAT'} = $source;
  1519. }
  1520. }
  1521. # The Pirate Bay
  1522. if ($ENABLE_SOURCE{'TPB'}) {
  1523. my @proxies = ('thepiratebay.org/search/', 'tpb.unblocked.co/search/');
  1524. my $source = findProxy(\@proxies, '\bPirate Search\b');
  1525. if ($source) {
  1526. $source->{'weight'} = 1.00;
  1527. $source->{'quote'} = 0;
  1528. $source->{'search_exclude'} = 1;
  1529. $source->{'search_suffix'} = '/0/7/0';
  1530. $sources{'TPB'} = $source;
  1531. }
  1532. }
  1533. # ISOhunt
  1534. if ($ENABLE_SOURCE{'ISO'}) {
  1535. my @proxies = ('isohunt.to/torrents/?ihq=', 'isohunters.net/torrents/?ihq=');
  1536. my $source = findProxy(\@proxies, 'Last\s+\d+\s+files\s+indexed');
  1537. if ($source) {
  1538. $source->{'weight'} = 0.75;
  1539. $source->{'quote'} = 1;
  1540. $source->{'search_exclude'} = 1;
  1541. $source->{'search_suffix'} = '';
  1542. $sources{'ISO'} = $source;
  1543. }
  1544. }
  1545. # Kickass
  1546. if ($ENABLE_SOURCE{'KICK'}) {
  1547. my @proxies = ('kickass.cd/search.php?q=', 'kickass.mx/search.php?q=');
  1548. my $source = findProxy(\@proxies, '/search.php');
  1549. if ($source) {
  1550. $source->{'weight'} = 1.00;
  1551. $source->{'quote'} = 1;
  1552. $source->{'search_exclude'} = 1;
  1553. $source->{'search_suffix'} = '/';
  1554. $sources{'KICK'} = $source;
  1555. }
  1556. }
  1557. # Torrentz
  1558. if ($ENABLE_SOURCE{'Z'}) {
  1559. my @proxies = ('torrentz.eu/search?q=', 'torrentz.me/search?q=', 'torrentz.ch/search?q=', 'torrentz.in/search?q=');
  1560. my $source = findProxy(\@proxies, 'Indexing [\d\,]+ active torrents');
  1561. if ($source) {
  1562. $source->{'weight'} = 1.00;
  1563. $source->{'quote'} = 1;
  1564. $source->{'search_exclude'} = 1;
  1565. $source->{'search_suffix'} = '';
  1566. if (!$NO_QUALITY_CHECKS) {
  1567. $source->{'search_suffix'} = '+peer+%3E+' . $MIN_COUNT,;
  1568. }
  1569. $sources{'Z'} = $source;
  1570. }
  1571. }
  1572. # ExtraTorrent
  1573. if ($ENABLE_SOURCE{'ET'}) {
  1574. my @proxies = (
  1575. 'extra.to/search/?search=', 'etmirror.com/search/?search=',
  1576. 'etproxy.com/search/?search=', 'extratorrentlive.com/search/?search='
  1577. );
  1578. my $source = findProxy(\@proxies, '/search/');
  1579. if ($source) {
  1580. $source->{'weight'} = 1.00;
  1581. $source->{'quote'} = 1;
  1582. $source->{'search_exclude'} = 0;
  1583. $source->{'search_suffix'} = '&new=1&x=0&y=0&srt=seeds&order=desc';
  1584. $source->{'search_url'} = 'phantomjs:' . $source->{'search_url'};
  1585. $sources{'ET'} = $source;
  1586. }
  1587. }
  1588. # Sanity check
  1589. if (scalar(keys(%sources)) < 1) {
  1590. die("No sources available\n");
  1591. }
  1592. return \%sources;
  1593. }
  1594. sub findProxy($$) {
  1595. my ($proxies, $match) = @_;
  1596. # Automatically select a proxy
  1597. my $host = '';
  1598. my $protocol = '';
  1599. my $search_url = '';
  1600. foreach my $url (@{$proxies}) {
  1601. my $path;
  1602. ($protocol, $host, $path) = $url =~ /^(?:(https?)\:\/\/)?([^\/]+)(\/.*)?$/i;
  1603. if (!$protocol) {
  1604. $protocol = $PROTOCOL;
  1605. }
  1606. if (!$host) {
  1607. print STDERR 'Could not parse proxy URL: ' . $url . "\n";
  1608. next;
  1609. }
  1610. my ($content, $code) = fetch($protocol . '://' . $host, undef(), 1);
  1611. if ($code == 200 && $content =~ m/${match}/i) {
  1612. $search_url = $protocol . '://' . $host . $path;
  1613. last;
  1614. } elsif ($DEBUG) {
  1615. print STDERR 'Proxy not available: ' . $host . "\n";
  1616. if ($DEBUG > 1) {
  1617. print STDERR $content . "\n";
  1618. }
  1619. }
  1620. }
  1621. # Return the search data if at least one proxy is up
  1622. if ($search_url) {
  1623. my %tmp = (
  1624. 'protocol' => $protocol,
  1625. 'host' => $host,
  1626. 'search_url' => $search_url,
  1627. );
  1628. return \%tmp;
  1629. }
  1630. # Return undef if we didn't find any working proxies
  1631. return undef();
  1632. }
  1633. sub fetch($;$$) {
  1634. my ($url, $file, $nocheck) = @_;
  1635. if ($DEBUG) {
  1636. print STDERR 'Fetch: ' . $url . "\n";
  1637. }
  1638. # Allow usage with short names
  1639. if ($file && !($file =~ /^\//)) {
  1640. $file = '/tmp/findTorrent-' . $file;
  1641. }
  1642. # Random delay
  1643. if ($DEBUG > 1) {
  1644. sleep($SLEEP * 0.25);
  1645. } else {
  1646. sleep($DELAY * (rand($SLEEP) + ($SLEEP / 2)));
  1647. }
  1648. # Allow JS processing
  1649. if ($url =~ /^phantomjs\:\s*(\S.*)$/i) {
  1650. return phantomFetch($1, $file);
  1651. }
  1652. # Fetch object
  1653. if (!$FETCH) {
  1654. if (!$COOKIES) {
  1655. $COOKIES = mktemp('/tmp/findTorrent.cookies.XXXXXXXX');
  1656. }
  1657. $FETCH = Fetch->new(
  1658. 'cookiefile' => $COOKIES,
  1659. 'timeout' => $TIMEOUT,
  1660. 'uas' => $UA
  1661. );
  1662. }
  1663. # Debug as requested
  1664. if ($DEBUG > 1 && $file) {
  1665. $FETCH->file($file);
  1666. } else {
  1667. $FETCH->file('');
  1668. }
  1669. # Standard fetch
  1670. $FETCH->url($url);
  1671. if ($nocheck) {
  1672. $FETCH->fetch('nocheck' => 1);
  1673. } else {
  1674. $FETCH->fetch();
  1675. }
  1676. return (scalar($FETCH->content()), $FETCH->status_code());
  1677. }
  1678. sub phantomFetch($$) {
  1679. my ($url, $file) = @_;
  1680. # Init
  1681. if (!$SCRIPT) {
  1682. $SCRIPT = mktemp('/tmp/findTorrent.phantom.XXXXXXXX');
  1683. open(my $fh, '>', $SCRIPT)
  1684. or die('Unable to open PhantomJS script file: ' . $! . "\n");
  1685. print $fh $PHANTOM_CONFIG;
  1686. close($fh);
  1687. }
  1688. # Capture STDOUT, drop STDERR
  1689. my @cmd = ($PHANTOM, $SCRIPT, $url);
  1690. my ($content, undef(), undef()) = capture { system(@cmd); };
  1691. # Debug as requested
  1692. if ($DEBUG > 1 && $file) {
  1693. open(my $fh, '>', $file)
  1694. or warn('Unable to open phantom debug file: ' . $! . "\n");
  1695. if ($fh) {
  1696. print $fh $content;
  1697. close($fh);
  1698. }
  1699. }
  1700. # Try to get the response status code, fake one if we cannot
  1701. my ($code) = $content =~ /^(\d\d\d)$/m;
  1702. if (!defined($code)) {
  1703. warn("Unable to retrive HTTP status code. Faking success...\n");
  1704. $code = 200;
  1705. }
  1706. return ($content, $code);
  1707. }