findTorrent.pl 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. # Includes
  5. use Date::Parse;
  6. use URI::Encode qw(uri_encode);
  7. use HTML::Entities;
  8. use HTML::Strip;
  9. use JSON;
  10. use Sys::Syslog qw(:standard :macros);
  11. use Capture::Tiny ':all';
  12. use File::Temp;
  13. use File::Spec;
  14. use File::Basename;
  15. use FindBin qw($Bin);
  16. use lib $Bin;
  17. use Fetch;
  18. # Prototypes
  19. sub confDefault($;$);
  20. sub readGlobalExcludes($);
  21. sub readInputDir($);
  22. sub parseNZB($);
  23. sub badTor($);
  24. sub seedCleanup($);
  25. sub lowQualityTor($);
  26. sub getHash($);
  27. sub resolveSecondary($);
  28. sub resolveTrackers($);
  29. sub splitTags($$$$);
  30. sub findSE($);
  31. sub initSources();
  32. sub findProxy($$);
  33. sub fetch($;$$);
  34. sub phantomFetch($$);
  35. # Globals
  36. my %CONFIG = ();
  37. my $SOURCES = undef();
  38. my $COOKIES = undef();
  39. my $FETCH = undef();
  40. my $SCRIPT = undef();
  41. # Command line
  42. my ($dir, $search) = @ARGV;
  43. if (!defined($dir)) {
  44. die('Usage: ' . basename($0) . " input_directory [search_string]\n");
  45. }
  46. # Pre-config environment
  47. my $DEBUG = 0;
  48. if ($ENV{'DEBUG'}) {
  49. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  50. $DEBUG = $1;
  51. } else {
  52. $DEBUG = 1;
  53. }
  54. }
  55. my $CONF_FILE = $ENV{'HOME'} . '/.findTorrent.config';
  56. if (defined($ENV{'CONF_FILE'})) {
  57. $CONF_FILE = $ENV{'CONF_FILE'};
  58. }
  59. # Read the config file
  60. if ($CONF_FILE && -r $CONF_FILE) {
  61. my $fh;
  62. open($fh, '<', $CONF_FILE)
  63. or die('Unable to open config file: ' . $CONF_FILE . ': ' . $! . "\n");
  64. while (<$fh>) {
  65. # Skip blank lines and comments
  66. if (/^\s*#/ || /^\s*$/) {
  67. next;
  68. }
  69. # Assume everything else is key = val pairs
  70. if (my ($key, $val) = $_ =~ /^\s*(\S[^\=]+)\=\>?\s*(\S.*)$/) {
  71. $key =~ s/^\s+//;
  72. $key =~ s/\s+$//;
  73. $val =~ s/^\s+//;
  74. $val =~ s/\s+$//;
  75. $CONFIG{$key} = $val;
  76. if ($DEBUG > 1) {
  77. print STDERR 'Adding config: ' . $key . ' => ' . $val . "\n";
  78. }
  79. } else {
  80. warn('Ignoring config line: ' . $_ . "\n");
  81. }
  82. }
  83. close($fh);
  84. }
  85. # Config defaults
  86. confDefault('EXCLUDES_FILE', $ENV{'HOME'} . '/.findTorrent.exclude');
  87. confDefault('TV_DIR', `~/bin/video/mediaPath` . '/TV');
  88. confDefault('PHANTOM_BIN', '/usr/local/bin/phantomjs');
  89. confDefault('UA',
  90. 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/538.39.41 (KHTML, like Gecko) Version/8.0 Safari/538.39.41');
  91. confDefault('PROTOCOL', 'https');
  92. confDefault('SLEEP', 1);
  93. confDefault('DELAY', 5);
  94. confDefault('TIMEOUT', 15);
  95. confDefault('ERR_DELAY', $CONFIG{'TIMEOUT'} * 2);
  96. confDefault('ERR_RETRIES', 3);
  97. confDefault('MORE_NUMBER_FORMATS', 0);
  98. confDefault('NO_QUALITY_CHECKS', 0);
  99. confDefault('NEXT_EPISODES', 3);
  100. confDefault('MIN_DAYS_BACK', 0);
  101. confDefault('MAX_DAYS_BACK', 3);
  102. confDefault('SYSLOG', 1);
  103. confDefault('MIN_COUNT', 10);
  104. confDefault('MIN_SIZE', 100);
  105. confDefault('SIZE_BONUS', 5);
  106. confDefault('SIZE_PENALTY', $CONFIG{'SIZE_BONUS'});
  107. confDefault('TITLE_PENALTY', $CONFIG{'SIZE_BONUS'} / 2);
  108. confDefault('MAX_SEEDS', 500);
  109. confDefault('MAX_SEED_RATIO', 0.25);
  110. confDefault('SEED_RATIO_COUNT', $CONFIG{'MIN_COUNT'});
  111. confDefault('TRACKER_LOOKUP', 1);
  112. confDefault(
  113. 'PHANTOM_CONFIG', "var system = require('system');
  114. var page = require('webpage').create();
  115. var resources = [];
  116. page.open(system.args[1], function(status)
  117. {
  118. console.log(resources[0].status);
  119. console.log(page.content);
  120. phantom.exit();
  121. });
  122. // Add responses for completed text/html resources
  123. page.onResourceReceived = function(response) {
  124. if (response.stage !== 'end') return;
  125. if (response.headers.filter(function(header) {
  126. if (header.name == 'Content-Type' && header.value.indexOf('text/html') == 0) {
  127. return true;
  128. }
  129. return false;
  130. }).length > 0)
  131. resources.push(response);
  132. };"
  133. );
  134. # Re-parse a few config items
  135. if (exists($CONFIG{'TRACKERS'})) {
  136. my @trackers = ();
  137. foreach my $tracker (split(/\s*,\s*/, $CONFIG{'TRACKERS'})) {
  138. $tracker =~ s/^\s+//;
  139. $tracker =~ s/\s+$//;
  140. push(@trackers, $tracker);
  141. }
  142. $CONFIG{'TRACKERS'} = \@trackers;
  143. }
  144. if (exists($CONFIG{'SOURCES'})) {
  145. my %sources = ();
  146. foreach my $source (split(/\s*,\s*/, $CONFIG{'SOURCES'})) {
  147. $source =~ s/^\s+//;
  148. $source =~ s/\s+$//;
  149. $source = uc($source);
  150. $sources{$source} = 1;
  151. }
  152. $CONFIG{'SOURCES'} = \%sources;
  153. }
  154. # Sanity checks
  155. if (ref($CONFIG{'SOURCES'}) ne 'HASH' || !exists($CONFIG{'SOURCES'}) || scalar(keys(%{ $CONFIG{'SOURCES'} })) < 1) {
  156. die("No sources configured\n");
  157. }
  158. if (ref($CONFIG{'TRACKERS'}) ne 'ARRAY' || !exists($CONFIG{'TRACKERS'}) || scalar(@{ $CONFIG{'TRACKERS'} }) < 1) {
  159. warn("No trackers configured\n");
  160. }
  161. if (!exists($CONFIG{'NZB_AGE_GOOD'}) || !exists($CONFIG{'NZB_AGE_MAX'})) {
  162. warn("No NZB good/max age provided\n");
  163. }
  164. # Open the log if enabled
  165. if ($CONFIG{'SYSLOG'}) {
  166. openlog(basename($0), '', LOG_DAEMON);
  167. }
  168. # Read the torrent excludes list
  169. $CONFIG{'EXCLUDES'} = readGlobalExcludes($CONFIG{'EXCLUDES_FILE'});
  170. # Setup our sources
  171. $SOURCES = initSources();
  172. $CONFIG{'DELAY'} /= scalar(keys(%{$SOURCES})) / 2;
  173. # Figure out what we're searching for
  174. my @urls = ();
  175. my $CUSTOM_SEARCH = 0;
  176. my @need = ();
  177. my %need = ();
  178. my $SERIES = readInputDir($dir);
  179. # Handle custom searches
  180. if ((scalar(@urls) < 1) && defined($search) && length($search) > 0) {
  181. # Note the custom search string
  182. $CUSTOM_SEARCH = 1;
  183. if ($DEBUG) {
  184. print STDERR "Custom search\n";
  185. }
  186. # Create the relevent search strings
  187. foreach my $key (keys(%{$SOURCES})) {
  188. my $source = $SOURCES->{$key};
  189. push(@urls, $source->{'search_url'} . $search . $SERIES->{'exclude'} . $source->{'search_suffix'});
  190. }
  191. }
  192. # Handle search-by-date series
  193. if ((scalar(@urls) < 1) && -e $SERIES->{'path'} . '/search_by_date') {
  194. # Note the search-by-date status
  195. # Set the CUSTOM_SERACH flag to skip season/epsiode matching
  196. $CUSTOM_SEARCH = 1;
  197. # Read the find-by-date string
  198. my $search_by_date = '';
  199. local ($/, *FH);
  200. open(FH, $SERIES->{'path'} . '/search_by_date')
  201. or die('Unable to read search_by_date for show: ' . $SERIES->{'name'} . ': ' . $! . "\n");
  202. my $text = <FH>;
  203. close(FH);
  204. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  205. $search_by_date = $1;
  206. } else {
  207. die('Skipping invalid search_by_date for show: ' . $SERIES->{'name'} . ': ' . $text . "\n");
  208. }
  209. # Create search strings for each date in the range, unless the related file already exists
  210. my (%years, %months, %days) = ();
  211. for (my $days_back = $CONFIG{'MIN_DAYS_BACK'} ; $days_back <= $CONFIG{'MAX_DAYS_BACK'} ; $days_back++) {
  212. # Calculate the date
  213. my (undef(), undef(), undef(), $day, $month, $year) = localtime(time() - (86400 * $days_back));
  214. # Format as strings
  215. $year = sprintf('%04d', $year + 1900);
  216. $month = sprintf('%02d', $month + 1);
  217. $day = sprintf('%02d', $day);
  218. # Check for an existing file
  219. my $exists = 0;
  220. {
  221. my $season_dir = $SERIES->{'path'} . '/Season ' . $year;
  222. my $prefix = qr/${year}\-${month}\-${day}\s*\-\s*/;
  223. opendir(SEASON, $season_dir)
  224. or die('Unable to open season directory (' . $season_dir . '): ' . $! . "\n");
  225. while (my $file = readdir(SEASON)) {
  226. if ($file =~ $prefix) {
  227. $exists = 1;
  228. last;
  229. }
  230. }
  231. closedir(SEASON);
  232. }
  233. if ($exists) {
  234. next;
  235. }
  236. # Save all the date string components
  237. $years{$year} = 1;
  238. $months{$month} = 1;
  239. $days{$day} = 1;
  240. # Create the relevent search strings
  241. my $search_str = $search_by_date;
  242. $search_str =~ s/%Y/${year}/g;
  243. $search_str =~ s/%m/${month}/g;
  244. $search_str =~ s/%d/${day}/g;
  245. foreach my $key (keys(%{$SOURCES})) {
  246. my $source = $SOURCES->{$key};
  247. push(@urls, $source->{'search_url'} . $search_str . $SERIES->{'exclude'} . $source->{'search_suffix'});
  248. }
  249. }
  250. # Build a date string matching regex
  251. my $str = '\b(?:' . join('|', keys(%years)) . ')\b';
  252. $str .= '.*';
  253. $str .= '\b(?:' . join('|', keys(%months)) . ')\b';
  254. $str .= '.*';
  255. $str .= '\b(?:' . join('|', keys(%days)) . ')\b';
  256. $CUSTOM_SEARCH = qr/${str}/;
  257. # Debug
  258. if ($DEBUG) {
  259. print STDERR 'Searching with date template: ' . $str . "\n";
  260. }
  261. }
  262. # Handle standard series
  263. if (scalar(@urls) < 1) {
  264. # Validate the season number
  265. if (!defined($SERIES->{'season'}) || $SERIES->{'season'} < 1 || $SERIES->{'season'} > 2000) {
  266. die('Invalid season number: ' . $SERIES->{'name'} . ' => ' . $SERIES->{'season'} . "\n");
  267. }
  268. # Get the last episode number
  269. my $no_next = 0;
  270. my %episodes = ();
  271. my $highest = 0;
  272. opendir(SEASON, $SERIES->{'path'} . '/Season ' . $SERIES->{'season'})
  273. or die("Unable to open season directory: ${!}\n");
  274. while (my $file = readdir(SEASON)) {
  275. # Skip support files
  276. if ($file =~ /\.(?:png|xml|jpg|gif|tbn|txt|nfo|torrent)\s*$/i) {
  277. next;
  278. }
  279. # Check for a season_done file
  280. if ($file eq 'season_done') {
  281. $no_next = 1;
  282. next;
  283. }
  284. # Extract the episode number
  285. my ($num) = $file =~ /^\s*\.?(\d+)\s*\-\s*/i;
  286. if (defined($num)) {
  287. $num = int($num);
  288. if ($DEBUG) {
  289. print STDERR 'Found episode number: ' . $num . ' in file: ' . $file . "\n";
  290. }
  291. # Record it
  292. $episodes{$num} = 1;
  293. # Track the highest episode number
  294. if ($num > $highest) {
  295. $highest = $num;
  296. }
  297. }
  298. }
  299. close(SEASON);
  300. # Assume we need the next 2 episodes, unless no_next is set (i.e. season_done)
  301. if (!$no_next) {
  302. for (my $i = 1 ; $i <= $CONFIG{'NEXT_EPISODES'} ; $i++) {
  303. push(@need, $highest + $i);
  304. }
  305. }
  306. # Find any missing episodes
  307. for (my $i = 1 ; $i <= $highest ; $i++) {
  308. if (!$episodes{$i}) {
  309. push(@need, $i);
  310. }
  311. }
  312. if ($DEBUG) {
  313. print STDERR 'Needed episodes: ' . join(', ', @need) . "\n";
  314. }
  315. # Reverse the array for later matching
  316. foreach my $episode (@need) {
  317. $need{$episode} = 1;
  318. }
  319. # Construct a URL-friendly show name
  320. my $safeShow = $SERIES->{'name'};
  321. $safeShow =~ s/\s+\&\s+/ and /i;
  322. $safeShow =~ s/^\s*The\b//i;
  323. $safeShow =~ s/\s+\-\s+/ /g;
  324. $safeShow =~ s/[\'\:]//g;
  325. $safeShow =~ s/[^\w\"\-]+/ /g;
  326. $safeShow =~ s/^\s+//;
  327. $safeShow =~ s/\s+$//;
  328. $safeShow =~ s/\s\s+/ /g;
  329. $safeShow =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
  330. $safeShow =~ s/\%20/\+/g;
  331. # Calculate possible name variations
  332. my @urlShowVarients = ();
  333. {
  334. # Default name
  335. push(@urlShowVarients, $safeShow);
  336. # Search for both "and" and "&"
  337. if ($safeShow =~ /\+and\+/i) {
  338. my $tmp = $safeShow;
  339. $tmp =~ s/\+and\+/\+%26\+/;
  340. push(@urlShowVarients, $tmp);
  341. }
  342. }
  343. # Construct the URL for each title varient of each needed episode
  344. foreach my $urlShow (@urlShowVarients) {
  345. foreach my $episode (@need) {
  346. my $episode_long = sprintf('%02d', $episode);
  347. my $season_long = sprintf('%02d', $SERIES->{'season'});
  348. foreach my $source (values(%{$SOURCES})) {
  349. # Allow custom handling
  350. if (defined($source->{'custom_search'}) && ref($source->{'custom_search'}) eq 'CODE') {
  351. $source->{'custom_search'}->(\@urls, $urlShow, $SERIES->{'season'}, $episode);
  352. next;
  353. }
  354. # Use quotes around the show name if the source needs them
  355. my $quote = '%22';
  356. if (!$source->{'quote'}) {
  357. $quote = '';
  358. }
  359. # Calculate the compete search prefix and suffix to simplify later concatenations
  360. my $prefix = $source->{'search_url'} . $quote . $urlShow . $quote;
  361. my $suffix = '';
  362. if ($source->{'search_exclude'}) {
  363. $suffix .= $SERIES->{'exclude'};
  364. }
  365. if ($source->{'search_suffix'}) {
  366. $suffix .= $source->{'search_suffix'};
  367. }
  368. # SXXEYY
  369. my $url = $prefix . '+s' . $season_long . 'e' . $episode_long . $suffix;
  370. push(@urls, $url);
  371. # Extra searches for shows that have lazy/non-standard number formats
  372. if ($CONFIG{'MORE_NUMBER_FORMATS'}) {
  373. # SXEY
  374. if ($season_long ne $SERIES->{'season'} || $episode_long ne $episode) {
  375. $url = $prefix . '+s' . $SERIES->{'season'} . 'e' . $episode . $suffix;
  376. push(@urls, $url);
  377. }
  378. # SXX EYY
  379. $url = $prefix . '+s' . $season_long . '+e' . $episode_long . $suffix;
  380. push(@urls, $url);
  381. # Season XX Episode YY
  382. $url = $prefix . '+season+' . $season_long . '+episode+' . $episode_long . $suffix;
  383. push(@urls, $url);
  384. # Series X Episode Y
  385. $url = $prefix . '+series+' . $SERIES->{'season'} . '+episode+' . $episode . $suffix;
  386. push(@urls, $url);
  387. # SxEE
  388. $url = $prefix . '+' . $SERIES->{'season'} . 'x' . $episode_long . $suffix;
  389. push(@urls, $url);
  390. # Season X
  391. if ($CONFIG{'NO_QUALITY_CHECKS'}) {
  392. $url = $prefix . '+Season+' . $SERIES->{'season'} . $suffix;
  393. push(@urls, $url);
  394. }
  395. }
  396. }
  397. }
  398. }
  399. }
  400. my @html_content = ();
  401. my @json_content = ();
  402. foreach my $url (@urls) {
  403. # Fetch the page
  404. my $errCount = 0;
  405. my $content = '';
  406. HTTP_ERR_LOOP: {
  407. my $code = 0;
  408. if ($DEBUG) {
  409. print STDERR 'Searching with URL: ' . $url . "\n";
  410. }
  411. # Fetch
  412. ($content, $code) = fetch($url, 'primary.html');
  413. # Check for useful errors
  414. if ($code == 404) {
  415. if ($DEBUG) {
  416. print STDERR "Skipping content from 404 response\n";
  417. }
  418. next;
  419. }
  420. # Check for less useful errors
  421. if ($code != 200) {
  422. if ($code >= 500 && $code <= 599) {
  423. if ($errCount > $CONFIG{'ERR_RETRIES'}) {
  424. print STDERR 'Unable to fetch URL: ' . $url . "\n";
  425. $errCount = 0;
  426. next;
  427. }
  428. if ($DEBUG) {
  429. print STDERR 'Retrying URL (' . $code . '): ' . $url . "\n";
  430. }
  431. $errCount++;
  432. sleep($CONFIG{'ERR_DELAY'});
  433. redo HTTP_ERR_LOOP;
  434. } else {
  435. print STDERR 'Error fetching URL (' . $code . '): ' . $url . "\n";
  436. }
  437. next;
  438. }
  439. }
  440. # Save the content, discriminating by data type
  441. if ($DEBUG > 1) {
  442. print STDERR 'Fetched ' . length($content) . " bytes\n";
  443. }
  444. if ($content =~ /^\s*[\{\[]/) {
  445. my $json = eval { decode_json($content); };
  446. if (defined($json) && ref($json)) {
  447. push(@json_content, $json);
  448. } elsif ($DEBUG) {
  449. print STDERR 'JSON parsing failure on: ' . $content . "\n";
  450. }
  451. } else {
  452. push(@html_content, scalar($content));
  453. }
  454. }
  455. # We need someplace to store parsed torrent data from the various sources
  456. my @tors = ();
  457. # Handle HTML content
  458. foreach my $content (@html_content) {
  459. if ($content =~ /\<title\>The Pirate Bay/i) {
  460. # Find TR elements from TPB
  461. my @trs = splitTags($content, 'TR', 'TH', undef());
  462. foreach my $tr (@trs) {
  463. # Find the show title
  464. my ($title) = $tr =~ /title\=\"Details\s+for\s+([^\"]*)\"/i;
  465. if (!defined($title) || length($title) < 1) {
  466. if ($DEBUG) {
  467. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  468. }
  469. next;
  470. }
  471. $title =~ s/^\s+//;
  472. # Extract the season and episode numbers
  473. my ($fileSeason, $episode) = findSE($title);
  474. # Extract the URL
  475. my ($url) = $tr =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  476. if (!defined($url) || length($url) < 1) {
  477. if ($DEBUG) {
  478. print STDERR "Skipping TR with no magnet URL\n";
  479. }
  480. next;
  481. }
  482. $url = decode_entities($url);
  483. # Count the sum of seeders and leachers
  484. my $seeds = 0;
  485. my $leaches = 0;
  486. if ($tr =~ /\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*$/i) {
  487. $seeds = $1;
  488. $leaches = $2;
  489. }
  490. # Extract the size (from separate column or inline)
  491. my $size = 0;
  492. my $unit = 'M';
  493. if ($tr =~ m/Size (\d+(?:\.\d+)?)\&nbsp\;(G|M)iB/) {
  494. $size = $1;
  495. $unit = $2;
  496. } elsif ($tr =~ m/(\d+(?:\.\d+)?)\&nbsp\;(G|M)iB\<\/[tT][dD]\>/) {
  497. $size = $1;
  498. $unit = $2;
  499. }
  500. if ($unit eq 'G') {
  501. $size *= 1024;
  502. }
  503. $size = int($size);
  504. if ($DEBUG) {
  505. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  506. }
  507. # Save the extracted data
  508. my %tor = (
  509. 'title' => $title,
  510. 'season' => $fileSeason,
  511. 'episode' => $episode,
  512. 'seeds' => $seeds,
  513. 'leaches' => $leaches,
  514. 'size' => $size,
  515. 'url' => $url,
  516. 'source' => 'TPB'
  517. );
  518. push(@tors, \%tor);
  519. }
  520. } elsif ($content =~ /\<title\>[^\<\>]*\bisoHunt\b[^\<\>]*\<\/title\>/i) {
  521. # Find each TR element from ISOHunt
  522. my @trs = splitTags($content, 'TR', 'TH', undef());
  523. foreach my $tr (@trs) {
  524. # Split each TD element from the row
  525. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  526. if (scalar(@tds) > 10 || scalar(@tds) < 9) {
  527. if ($tds[1] =~ /No results found/) {
  528. next;
  529. }
  530. print STDERR 'Skipping invalid ISO TR: ' . $tr . "\n";
  531. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  532. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  533. }
  534. next;
  535. }
  536. # Skip sponsored results
  537. if ($tds[1] =~ /\bSponsored\b/i) {
  538. if ($DEBUG > 1) {
  539. print STDERR "Skipping sponsored result\n";
  540. }
  541. next;
  542. }
  543. # Find the torrent ID and show title
  544. my ($id, $title) = $tds[2] =~ /\<a(?:\s+[^\>]*)?href=\"\/torrent_details\/(\d+\/[^\"]+)\"\>\<span\>([^\>]+)\<\/span\>/i;
  545. if (!defined($id) || length($id) < 1 || !defined($title) || length($title) < 1) {
  546. if ($DEBUG > 1) {
  547. print STDERR "Unable to find show ID or title in TD:\n\t" . $tds[2] . "\n";
  548. }
  549. next;
  550. }
  551. $title =~ s/^\s+//;
  552. # Extract the season and episode numbers
  553. my ($fileSeason, $episode) = findSE($title);
  554. # Extract the size
  555. my $size = 0;
  556. my $unit = 'M';
  557. if ($tds[6] =~ m/(\d+(?:\.\d+)?) (G|M)B<\/td\>/i) {
  558. $size = $1;
  559. $unit = $2;
  560. }
  561. if ($unit eq 'G') {
  562. $size *= 1024;
  563. }
  564. $size = int($size);
  565. # Count the sum of seeders and leachers
  566. my $seeds = 0;
  567. if ($tds[7] =~ /(\d+)\<\/td\>/i) {
  568. $seeds = $1;
  569. }
  570. my $leaches = 0;
  571. if ($tds[8] =~ /(\d+)\<\/td\>/i) {
  572. $leaches = $1;
  573. }
  574. # Build the detail page URL
  575. my $url = $SOURCES->{'ISO'}->{'protocol'} . '://' . $SOURCES->{'ISO'}->{'host'} . '/torrent_details/' . $id;
  576. if ($DEBUG) {
  577. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  578. }
  579. # Save the extracted data
  580. my %tor = (
  581. 'title' => $title,
  582. 'season' => $fileSeason,
  583. 'episode' => $episode,
  584. 'seeds' => $seeds,
  585. 'leaches' => $leaches,
  586. 'size' => $size,
  587. 'url' => $url,
  588. 'source' => 'ISO'
  589. );
  590. push(@tors, \%tor);
  591. }
  592. } elsif ($content =~ /Kickass\s*Torrents\<\/title\>/i) {
  593. # Find each TR element from Kickass
  594. my @trs = splitTags($content, 'TR', 'TH', undef());
  595. foreach my $tr (@trs) {
  596. # Split each TD element from the row
  597. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  598. if (scalar(@tds) != 6) {
  599. print STDERR "Invalid KICK TR:\n";
  600. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  601. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  602. }
  603. next;
  604. }
  605. # Skip empty results
  606. if ( $tds[1] =~ /\<h2\>Nothing found\!\<\/h2\>/i
  607. || $tds[1] =~ /\<p\>\<strong\>Page not found\<\/strong\>\<\/p\>/i)
  608. {
  609. if ($DEBUG) {
  610. print STDERR "Skipping empty TD\n";
  611. }
  612. next;
  613. }
  614. # Find the torrent title
  615. my ($title) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href\=\"[^\"]+\?title\=\[[^\]\"]+\]([^\"]+)\"/i;
  616. if (!defined($title) || length($title) < 1) {
  617. if ($DEBUG) {
  618. print STDERR "Unable to find torrent title in TD:\n\t" . $tds[1] . "\n";
  619. }
  620. next;
  621. }
  622. $title =~ s/^\s+//;
  623. # Find the torrent URL
  624. my ($url) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href=\"(magnet:[^\"]+)\"/i;
  625. if (!defined($url) || length($url) < 1) {
  626. if ($DEBUG) {
  627. print STDERR "Unable to find torrent URL in TD:\n\t" . $tds[1] . "\n";
  628. }
  629. next;
  630. }
  631. $url = decode_entities($url);
  632. # Extract the season and episode numbers
  633. my ($fileSeason, $episode) = findSE($title);
  634. # Extract the size
  635. my $size = 0;
  636. my $unit = 'M';
  637. if ($tds[2] =~ m/(\d+(?:\.\d+)?)\s+\<span\>(G|M)B\<\/span\>\<\/td\>/i) {
  638. $size = $1;
  639. $unit = $2;
  640. }
  641. if ($unit eq 'G') {
  642. $size *= 1024;
  643. }
  644. $size = int($size);
  645. # Count the sum of seeders and leachers
  646. my $seeds = 0;
  647. if ($tds[4] =~ /(\d+)\<\/td\>/i) {
  648. $seeds = $1;
  649. }
  650. my $leaches = 0;
  651. if ($tds[5] =~ /(\d+)\<\/td\>/i) {
  652. $leaches = $1;
  653. }
  654. if ($DEBUG) {
  655. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  656. }
  657. # Save the extracted data
  658. my %tor = (
  659. 'title' => $title,
  660. 'season' => $fileSeason,
  661. 'episode' => $episode,
  662. 'seeds' => $seeds,
  663. 'leaches' => $leaches,
  664. 'size' => $size,
  665. 'url' => $url,
  666. 'source' => 'KICK'
  667. );
  668. push(@tors, \%tor);
  669. }
  670. } elsif ($content =~ /\<a(?:\s+[^\>]*)?\>Torrentz\<\/a\>/i) {
  671. # We'll need to strip HTML
  672. my $hs = HTML::Strip->new();
  673. # Find each DL element from Torrentz
  674. my @dls = splitTags($content, 'DL', undef(), undef());
  675. foreach my $dl (@dls) {
  676. # Skip ads
  677. if ($dl =~ /\s+rel\=\"nofollow/i) {
  678. next;
  679. }
  680. # Find the torrent title
  681. my ($hash, $title) = $dl =~ /\<a href\=\"\/(\w+)\">(.*)\<\/a\>/i;
  682. if (!defined($hash) || length($hash) < 1 || !defined($title) || length($title) < 1) {
  683. if ($DEBUG) {
  684. print STDERR "Unable to find torrent title in DL:\n\t" . $dl . "\n";
  685. }
  686. next;
  687. }
  688. $hash = lc($hash);
  689. $title = $hs->parse($title);
  690. $hs->eof;
  691. $title =~ s/^\s+//;
  692. # Extract the season and episode numbers
  693. my ($fileSeason, $episode) = findSE($title);
  694. # Extract the size
  695. my $size = 0;
  696. if ($dl =~ /\<span class\=\"s\">(\d+(?:\.\d+)?) (G|M)B\<\/span\>/i) {
  697. $size = $1;
  698. if ($2 eq 'G') {
  699. $size *= 1024;
  700. }
  701. $size = int($size);
  702. }
  703. # Count the sum of seeders and leachers
  704. my $seeds = 0;
  705. if ($dl =~ /<span class\=\"u\">(\d+)\<\/span\>/i) {
  706. $seeds = $1;
  707. }
  708. my $leaches = 0;
  709. if ($dl =~ /<span class\=\"d\">(\d+)\<\/span\>/i) {
  710. $leaches = $1;
  711. }
  712. # Construct a magnet URL from the hash
  713. # Assume the seconary lookup will append a list of trackers
  714. my $url = 'magnet:?xt=urn:btih:' . $hash . '&dn=' . uri_encode($title, { 'encode_reserved' => 1 });
  715. if ($DEBUG) {
  716. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  717. }
  718. # Save the extracted data
  719. my %tor = (
  720. 'title' => $title,
  721. 'season' => $fileSeason,
  722. 'episode' => $episode,
  723. 'seeds' => $seeds,
  724. 'leaches' => $leaches,
  725. 'size' => $size,
  726. 'url' => $url,
  727. 'source' => 'Z'
  728. );
  729. push(@tors, \%tor);
  730. }
  731. } elsif ($content =~ /ExtraTorrent\.cc The World\'s Largest BitTorrent System\<\/title\>/i) {
  732. # Find TR elements from ET
  733. my @trs = splitTags($content, 'TR', undef(), 'magnet\:');
  734. foreach my $tr (@trs) {
  735. # Find the show title
  736. my ($title) = $tr =~ /title\=\"Download\s+([^\"]*)\s+torrent\"/i;
  737. if (!defined($title) || length($title) < 1) {
  738. if ($DEBUG) {
  739. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  740. }
  741. next;
  742. }
  743. $title =~ s/^\s+//;
  744. # Find the magnet URL
  745. my ($url) = $tr =~ /\<a\s+href=\"(magnet\:[^\"]+)\"/i;
  746. if (!defined($url) || length($url) < 1) {
  747. if ($DEBUG) {
  748. print STDERR "Skipping TR with no magnet URL\n";
  749. }
  750. next;
  751. }
  752. $url = decode_entities($url);
  753. # Extract the season and episode numbers
  754. my ($fileSeason, $episode) = findSE($title);
  755. # Count the sum of seeders and leachers
  756. my $seeds = 0;
  757. my $leaches = 0;
  758. if ($tr =~ /\<td\s+class=\"sy\">(\d+)\<\/td\>/) {
  759. $seeds = $1;
  760. }
  761. if ($tr =~ /\<td\s+class=\"ly\">(\d+)\<\/td\>/) {
  762. $leaches = $1;
  763. }
  764. # Extract the size (from separate column or inline)
  765. my $size = 0;
  766. my $unit = 'M';
  767. if ($tr =~ /\<td\>(\d+(?:\.\d+)?)\&nbsp\;(G|M)B\<\/td\>/) {
  768. $size = $1;
  769. $unit = $2;
  770. }
  771. if ($unit eq 'G') {
  772. $size *= 1024;
  773. }
  774. $size = int($size);
  775. if ($DEBUG) {
  776. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  777. }
  778. # Save the extracted data
  779. my %tor = (
  780. 'title' => $title,
  781. 'season' => $fileSeason,
  782. 'episode' => $episode,
  783. 'seeds' => $seeds,
  784. 'leaches' => $leaches,
  785. 'size' => $size,
  786. 'url' => $url,
  787. 'source' => 'ET'
  788. );
  789. push(@tors, \%tor);
  790. }
  791. } elsif ($content =~ /\<title\>Sorry/) {
  792. if ($DEBUG) {
  793. print STDERR "ISO offline\n";
  794. }
  795. } elsif ($content =~ /Database\s+maintenance/i) {
  796. if ($DEBUG) {
  797. print STDERR "TPB offline\n";
  798. }
  799. } elsif ($content =~ /^\s*$/) {
  800. if ($DEBUG) {
  801. print STDERR "Source returned no content\n";
  802. }
  803. } elsif ($content =~ /limit\s*reached/i) {
  804. if ($DEBUG) {
  805. print STDERR "Source request limit reached\n";
  806. }
  807. } elsif ($content =~ /Not\s+Found\<\/title\>/i) {
  808. if ($DEBUG) {
  809. print STDERR "No results\n";
  810. }
  811. } else {
  812. print STDERR "Unknown HTML content:\n" . $content . "\n\n";
  813. }
  814. }
  815. # Handle JSON content
  816. foreach my $content (@json_content) {
  817. if (ref($content) eq 'ARRAY' && scalar(@{$content}) == 0) {
  818. if ($DEBUG > 1) {
  819. print STDERR "Empty JSON array\n";
  820. }
  821. } elsif (exists($content->{'channel'})
  822. && ref($content->{'channel'}) eq 'HASH'
  823. && exists($content->{'channel'}->{'title'})
  824. && $content->{'channel'}->{'title'} =~ /nzbplanet/i)
  825. {
  826. print STDERR "Index: NZBplanet\n";
  827. } elsif (exists($content->{'channel'})
  828. && ref($content->{'channel'}) eq 'HASH'
  829. && exists($content->{'channel'}->{'title'})
  830. && $content->{'channel'}->{'title'} =~ /usenet\-crawler/i)
  831. {
  832. print STDERR "Index: usenet-crawler\n";
  833. } elsif (ref($content) eq 'ARRAY'
  834. && scalar(@{$content}) > 0
  835. && ref($content->[0]) eq 'HASH'
  836. && exists($content->[0]->{'guid'}))
  837. {
  838. print STDERR "Index: NZB.is\n";
  839. } elsif (exists($content->{'title'}) && $content->{'title'} =~ /NZBCat/i) {
  840. my $list = $content->{'item'};
  841. if (!$list || ref($list) ne 'ARRAY') {
  842. if ($DEBUG) {
  843. warn("Empty/invalid NCAT list\n");
  844. }
  845. next;
  846. }
  847. foreach my $item (@{$list}) {
  848. my $tor = parseNZB($item);
  849. if ($tor) {
  850. $tor->{'source'} = 'NCAT';
  851. push(@tors, $tor);
  852. }
  853. }
  854. } elsif (ref($content) eq 'HASH'
  855. && exists($content->{'channel'})
  856. && ref($content->{'channel'}) eq 'HASH'
  857. && exists($content->{'channel'}->{'title'})
  858. && $content->{'channel'}->{'title'} =~ /\bNzb\s+Planet\b/i)
  859. {
  860. if (!exists($content->{'channel'}->{'item'}) || ref($content->{'channel'}->{'item'}) ne 'ARRAY') {
  861. if ($DEBUG) {
  862. print STDERR "Empty/invalid NZB Planet result\n";
  863. }
  864. next;
  865. }
  866. foreach my $item (@{ $content->{'channel'}->{'item'} }) {
  867. my $tor = parseNZB($item);
  868. if ($tor) {
  869. $tor->{'source'} = 'PLANET';
  870. push(@tors, $tor);
  871. }
  872. }
  873. } else {
  874. print STDERR "Unknown JSON content:\n" . $content . "\n\n";
  875. }
  876. }
  877. # Filter for size/count/etc.
  878. my %tors = ();
  879. my $showRegex = undef();
  880. {
  881. my $showClean = $SERIES->{'name'};
  882. $showClean =~ s/[\"\']//g;
  883. $showClean =~ s/[\W_]+/\[\\W_\].*/g;
  884. $showRegex = qr/^${showClean}[\W_]/i;
  885. }
  886. foreach my $tor (@tors) {
  887. # Extract the BTIH/GUID, if available
  888. getHash($tor);
  889. # Skip bad TORs based on name/etc.
  890. if (badTor($tor)) {
  891. next;
  892. }
  893. # Skip files that contain a word from the series exclude list
  894. my $exclude = undef();
  895. foreach my $ex (keys(%{ $SERIES->{'exclude_hash'} })) {
  896. $ex = quotemeta($ex);
  897. if ($tor->{'title'} =~ m/${ex}/i) {
  898. $exclude = $ex;
  899. last;
  900. }
  901. }
  902. if (defined($exclude)) {
  903. if ($DEBUG) {
  904. print STDERR 'Skipping file: Title contains "' . $exclude . '": ' . $tor->{'title'} . "\n";
  905. }
  906. next;
  907. }
  908. # Enforce season and episode number matches for standard searches, or CUSTOM_SEARCH matching (if it's a regex)
  909. if (!$CUSTOM_SEARCH) {
  910. # Skip files that don't contain the right season number
  911. if (!defined($tor->{'season'}) || $tor->{'season'} != $SERIES->{'season'}) {
  912. if ($DEBUG) {
  913. print STDERR 'Skipping file: No match for season number (' . $SERIES->{'season'} . '): ' . $tor->{'title'} . "\n";
  914. }
  915. next;
  916. # Skip files that don't contain a needed episode number, unless there is no episode number and NO_QUALITY_CHECKS is set
  917. } elsif ((defined($tor->{'episode'}) && !$need{ $tor->{'episode'} })
  918. || (!defined($tor->{'episode'}) && !$CONFIG{'NO_QUALITY_CHECKS'}))
  919. {
  920. if ($DEBUG) {
  921. print STDERR 'Skipping file: No match for episode number (' . $tor->{'episode'} . '): ' . $tor->{'title'} . "\n";
  922. }
  923. next;
  924. }
  925. } elsif (ref($CUSTOM_SEARCH) eq 'Regexp') {
  926. # Skip files that don't match the regex
  927. if (!($tor->{'title'} =~ $CUSTOM_SEARCH)) {
  928. if ($DEBUG) {
  929. print STDERR 'Skipping file: No match for CUSTOM_SEARCH regex (' . $CUSTOM_SEARCH . '): ' . $tor->{'title'} . "\n";
  930. }
  931. next;
  932. }
  933. }
  934. # Ensure the seed/leach count is usable
  935. seedCleanup($tor);
  936. # Only apply the quality rules if NO_QUALITY_CHECKS is not in effect
  937. if (!$CONFIG{'NO_QUALITY_CHECKS'} && lowQualityTor($tor)) {
  938. next;
  939. }
  940. # Save good torrents
  941. push(@{ $tors{ $tor->{'episode'} } }, $tor);
  942. if ($DEBUG) {
  943. print STDERR 'Possible URL ('
  944. . $tor->{'seeds'} . '/'
  945. . $tor->{'leaches'}
  946. . ' seeds/leaches, '
  947. . $tor->{'size'}
  948. . ' MiB): '
  949. . $tor->{'title'} . "\n";
  950. }
  951. }
  952. # Find the average torrent size for each episode
  953. my %size = ();
  954. {
  955. my %max = ();
  956. my %avg = ();
  957. foreach my $episode (keys(%tors)) {
  958. my $count = 0;
  959. $avg{$episode} = 0.001;
  960. $max{$episode} = 0.001;
  961. foreach my $tor (@{ $tors{$episode} }) {
  962. $count++;
  963. $avg{$episode} += $tor->{'size'};
  964. if ($tor->{'size'} > $max{$episode}) {
  965. $max{$episode} = $tor->{'size'};
  966. }
  967. }
  968. if ($count > 0) {
  969. $avg{$episode} /= $count;
  970. }
  971. if (!defined($max{$episode})) {
  972. warn("Invalid max episode\n");
  973. }
  974. if (!defined($avg{$episode})) {
  975. warn("Invalid avg episode\n");
  976. }
  977. $size{$episode} = ($max{$episode} + $avg{$episode}) / 2;
  978. if ($DEBUG) {
  979. print STDERR 'Episode '
  980. . $episode
  981. . ' max/avg/cmp size: '
  982. . int($max{$episode}) . '/'
  983. . int($avg{$episode}) . '/'
  984. . int($size{$episode})
  985. . " MiB\n";
  986. }
  987. }
  988. }
  989. # Calculate an adjusted count based on the peer count, relative file size, and title contents
  990. foreach my $episode (keys(%tors)) {
  991. foreach my $tor (@{ $tors{$episode} }) {
  992. my $count = $tor->{'seeds'} + $tor->{'leaches'};
  993. # Start with the peer count
  994. $tor->{'adj_count'} = $count;
  995. # Adjust based on file size
  996. {
  997. my $size_ratio = $tor->{'size'} / $size{$episode};
  998. if ($tor->{'size'} >= $size{$episode}) {
  999. $tor->{'adj_count'} *= $CONFIG{'SIZE_BONUS'} * $size_ratio;
  1000. } else {
  1001. $tor->{'adj_count'} *= (1 / $CONFIG{'SIZE_PENALTY'}) * $size_ratio;
  1002. }
  1003. }
  1004. # Adjust based on title contents
  1005. if ($tor->{'title'} =~ /Subtitulado/i) {
  1006. $tor->{'adj_count'} *= 1 / $CONFIG{'TITLE_PENALTY'};
  1007. }
  1008. # Truncate to an integer
  1009. $tor->{'adj_count'} = int($tor->{'adj_count'});
  1010. if ($DEBUG) {
  1011. print STDERR 'Possible URL (' . $tor->{'adj_count'} . ' size-adjusted sources): ' . $tor->{'url'} . "\n";
  1012. }
  1013. }
  1014. }
  1015. # Pick the best-adjusted-count torrent for each episode
  1016. my %urls = ();
  1017. foreach my $episode (keys(%tors)) {
  1018. my @sorted = sort { $b->{'adj_count'} <=> $a->{'adj_count'} } @{ $tors{$episode} };
  1019. my $max = undef();
  1020. foreach my $tor (@sorted) {
  1021. if (!defined($max) || $tor->{'adj_count'} > $max) {
  1022. if ($urls{$episode} = resolveSecondary($tor)) {
  1023. if (!$tor->{'hash'} || $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  1024. print STDERR 'Double-skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1025. next;
  1026. }
  1027. $max = $tor->{'adj_count'};
  1028. }
  1029. if ($DEBUG) {
  1030. print STDERR 'Semi-final URL (adjusted count: ' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  1031. }
  1032. } elsif ($DEBUG) {
  1033. print STDERR 'Skipping for lesser adjusted count (' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  1034. }
  1035. }
  1036. }
  1037. # Output
  1038. foreach my $episode (keys(%tors)) {
  1039. if (defined($urls{$episode})) {
  1040. $urls{$episode} = resolveTrackers($urls{$episode});
  1041. print $urls{$episode} . "\n";
  1042. if ($CONFIG{'SYSLOG'}) {
  1043. syslog(LOG_NOTICE, $SERIES->{'name'} . ': ' . getHash($urls{$episode}));
  1044. }
  1045. if ($DEBUG) {
  1046. print STDERR 'Final URL: ' . $urls{$episode} . "\n";
  1047. }
  1048. } elsif ($DEBUG) {
  1049. print STDERR 'No URL for: ' . $episode . "\n";
  1050. }
  1051. }
  1052. # Cleanup
  1053. if ($COOKIES) {
  1054. unlink($COOKIES);
  1055. }
  1056. if ($SCRIPT) {
  1057. unlink($SCRIPT);
  1058. }
  1059. if ($CONFIG{'SYSLOG'}) {
  1060. closelog();
  1061. }
  1062. exit(0);
  1063. sub confDefault($;$) {
  1064. my ($name, $default) = @_;
  1065. if (!$name) {
  1066. return;
  1067. }
  1068. # Enviornment, config file, default, undef()
  1069. if (defined($ENV{$name})) {
  1070. $CONFIG{$name} = $ENV{$name};
  1071. } elsif (!defined($CONFIG{$name})) {
  1072. if ($default) {
  1073. $CONFIG{$name} = $default;
  1074. } else {
  1075. $CONFIG{$name} = undef();
  1076. }
  1077. }
  1078. }
  1079. sub readGlobalExcludes($) {
  1080. my ($file) = @_;
  1081. my %excludes = ();
  1082. if (!$file || !-r $file) {
  1083. warn('No excludes files available' . $file . "\n");
  1084. return \%excludes;
  1085. }
  1086. my $fh;
  1087. open($fh, '<', $file)
  1088. or die('Unable to open excludes file: ' . $file . ': ' . $! . "\n");
  1089. while (<$fh>) {
  1090. # Skip blank lines and comments
  1091. if (/^\s*$/ || /^\s*#/) {
  1092. next;
  1093. }
  1094. # Assume everything else is one BT hash/NZB GUID per line
  1095. if (/^\s*(\w+)\s*$/) {
  1096. my $hash = lc($1);
  1097. if ($DEBUG > 1) {
  1098. print STDERR 'Excluding hash: ' . $hash . "\n";
  1099. }
  1100. $excludes{$hash} = 1;
  1101. } else {
  1102. die('Invalid exclude line: ' . $_ . "\n");
  1103. }
  1104. }
  1105. close($fh);
  1106. return \%excludes;
  1107. }
  1108. # Find an input directory and read the local config
  1109. sub readInputDir($) {
  1110. my ($path) = @_;
  1111. my %series = ('path' => undef(), 'name' => '', 'season' => 0, 'exclude' => '', 'exclude_hash' => {});
  1112. # Allow use of the raw series name
  1113. if (!($path =~ /\//)) {
  1114. $series{'name'} = $path;
  1115. $series{'path'} = $CONFIG{'TV_DIR'} . '/' . $path;
  1116. } else {
  1117. $series{'path'} = $path;
  1118. }
  1119. # Allow use of relative paths
  1120. $series{'path'} = File::Spec->rel2abs($series{'path'});
  1121. # Isolate the season from the path, if provided
  1122. if ($series{'path'} =~ /\/Season\s+(\d+)\/?$/i) {
  1123. $series{'season'} = $1;
  1124. $series{'path'} = dirname($series{'path'});
  1125. }
  1126. # Sanity check
  1127. if ($DEBUG) {
  1128. print STDERR 'Checking directory: ' . $series{'path'} . "\n";
  1129. }
  1130. if (!-d $series{'path'}) {
  1131. die('Invalid input directory: ' . $series{'path'} . "\n");
  1132. }
  1133. # Allow the series name to be overriden
  1134. my $search_name = $series{'path'} . '/search_name';
  1135. if (-e $search_name) {
  1136. local ($/, *FH);
  1137. open(FH, $search_name)
  1138. or die('Unable to read search_name for series: ' . $series{'name'} . ': ' . $! . "\n");
  1139. my $text = <FH>;
  1140. close(FH);
  1141. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  1142. $series{'name'} = $1;
  1143. } else {
  1144. warn('Skipping invalid search_name for series: ' . $series{'name'} . ': ' . $text . "\n");
  1145. }
  1146. }
  1147. if ($DEBUG) {
  1148. print STDERR 'Searching with series title: ' . $series{'name'} . "\n";
  1149. }
  1150. # Final series name
  1151. if (!$series{'name'}) {
  1152. $series{'name'} = basename($series{'path'});
  1153. }
  1154. $series{'name'} =~ s/[\'\"\.]//g;
  1155. # If no explicit season is provided find the latest
  1156. if (!$series{'season'}) {
  1157. opendir(SERIES, $series{'path'})
  1158. or die("Unable to open series directory: ${!}\n");
  1159. while (my $file = readdir(SERIES)) {
  1160. if ($file =~ /^Season\s+(\d+)$/i) {
  1161. if (!$series{'season'} || $series{'season'} < $1) {
  1162. $series{'season'} = $1;
  1163. }
  1164. }
  1165. }
  1166. closedir(SERIES);
  1167. }
  1168. # Allow quality checks to be disabled
  1169. if (-e $series{'path'} . '/no_quality_checks') {
  1170. $CONFIG{'NO_QUALITY_CHECKS'} = 1;
  1171. if ($DEBUG) {
  1172. print STDERR 'Searching with no quality checks: ' . $series{'name'} . "\n";
  1173. }
  1174. }
  1175. # Allow use of more number formats
  1176. if (-e $series{'path'} . '/more_number_formats') {
  1177. $CONFIG{'MORE_NUMBER_FORMATS'} = 1;
  1178. if ($DEBUG) {
  1179. print STDERR 'Searching with more number formats: ' . $series{'name'} . "\n";
  1180. }
  1181. }
  1182. # Read the search excludes file, if any
  1183. if (-e $series{'path'} . '/excludes') {
  1184. local $/ = undef;
  1185. open(EX, $series{'path'} . '/excludes')
  1186. or die("Unable to open series excludes file: ${!}\n");
  1187. my $ex = <EX>;
  1188. close(EX);
  1189. $ex =~ s/^\s+//;
  1190. $ex =~ s/\s+$//;
  1191. my @excludes = split(/\s*,\s*/, $ex);
  1192. foreach my $ex (@excludes) {
  1193. $series{'excludes_hash'}->{$ex} = 1;
  1194. if (length($series{'exclude'})) {
  1195. $series{'exclude'} .= ' ';
  1196. }
  1197. $series{'exclude'} .= '-"' . $ex . '"';
  1198. }
  1199. $series{'exclude'} = uri_encode(' ' . $series{'exclude'});
  1200. }
  1201. # Log and return
  1202. if ($CONFIG{'SYSLOG'}) {
  1203. syslog(LOG_NOTICE, $series{'name'});
  1204. }
  1205. return \%series;
  1206. }
  1207. sub parseNZB($) {
  1208. my ($item) = @_;
  1209. if (!$item || ref($item) ne 'HASH' || !exists($item->{'guid'})) {
  1210. warn("Unable to parse NZB item\n");
  1211. return undef();
  1212. }
  1213. # Ensure the record is sensible
  1214. my $id = undef();
  1215. if (ref($item->{'guid'}) eq 'HASH' && exists($item->{'guid'}->{'text'})) {
  1216. $id = $item->{'guid'}->{'text'};
  1217. } elsif (!ref($item->{'guid'})) {
  1218. $id = $item->{'guid'};
  1219. }
  1220. if (!$id) {
  1221. warn("Unable to parse NZB item GUID\n");
  1222. return undef();
  1223. }
  1224. # NZB URL
  1225. my $url = undef();
  1226. if (exists($item->{'link'}) && $item->{'link'}) {
  1227. $url = $item->{'link'};
  1228. $url =~ s/^(https?)\:/${CONFIG{'PROTOCOL'}}\:/;
  1229. }
  1230. # Title
  1231. my $title = undef();
  1232. if (exists($item->{'title'}) && $item->{'title'}) {
  1233. $title = $item->{'title'};
  1234. } elsif (exists($item->{'description'}) && $item->{'description'}) {
  1235. $title = $item->{'description'};
  1236. }
  1237. if ($title) {
  1238. $url .= '#' . $title;
  1239. }
  1240. # Extract the season and episode numbers
  1241. my ($season, $episode) = findSE($title);
  1242. # Size
  1243. my $size = 0;
  1244. if (exists($item->{'newznab:attr'}) && $item->{'newznab:attr'} && ref($item->{'newznab:attr'}) eq 'ARRAY') {
  1245. foreach my $hash (@{ $item->{'newznab:attr'} }) {
  1246. if (ref($hash) eq 'HASH' && exists($hash->{'_name'}) && exists($hash->{'_value'}) && $hash->{'_name'} eq 'size') {
  1247. $size = int($hash->{'_value'}) / 1024;
  1248. last;
  1249. }
  1250. }
  1251. } elsif (exists($item->{'attr'}) && $item->{'attr'} && ref($item->{'attr'}) eq 'ARRAY') {
  1252. foreach my $hash (@{ $item->{'attr'} }) {
  1253. if (!exists($hash->{'@attributes'}) || ref($hash->{'@attributes'}) ne 'HASH') {
  1254. next;
  1255. }
  1256. $hash = $hash->{'@attributes'};
  1257. if (exists($hash->{'name'}) && $hash->{'name'} eq 'size' && exists($hash->{'value'})) {
  1258. $size = int($hash->{'value'}) / 1024;
  1259. last;
  1260. }
  1261. }
  1262. }
  1263. # Date
  1264. my $date = undef();
  1265. if (exists($item->{'pubDate'}) && $item->{'pubDate'}) {
  1266. $date = str2time($item->{'pubDate'});
  1267. }
  1268. # Sanity checks
  1269. if (!$url) {
  1270. warn('No URL in NZB item: ' . $id . "\n");
  1271. return undef();
  1272. }
  1273. if (!$title) {
  1274. warn('No title in NZB item: ' . $id . "\n");
  1275. return undef();
  1276. }
  1277. if (!$size) {
  1278. warn('No size in NZB item: ' . $id . "\n");
  1279. return undef();
  1280. }
  1281. if ($DEBUG) {
  1282. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  1283. }
  1284. # Save the extracted data
  1285. my %tor = (
  1286. 'title' => $title,
  1287. 'season' => $season,
  1288. 'episode' => $episode,
  1289. 'date' => $date,
  1290. 'size' => $size,
  1291. 'url' => $url,
  1292. );
  1293. return \%tor;
  1294. }
  1295. sub badTor($) {
  1296. my ($tor) = @_;
  1297. # Skip files that are in the torrent excludes list
  1298. if ($tor->{'hash'} && $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  1299. if ($DEBUG) {
  1300. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1301. }
  1302. return 1;
  1303. }
  1304. # Skip files that don't start with our show title
  1305. if (!($tor->{'title'} =~ $showRegex)) {
  1306. if ($DEBUG) {
  1307. print STDERR 'Skipping file: Title does not match (' . $showRegex . '): ' . $tor->{'title'} . "\n";
  1308. }
  1309. return 1;
  1310. # Skip leaked files
  1311. } elsif ($tor->{'title'} =~ /leaked/i) {
  1312. if ($DEBUG) {
  1313. print STDERR 'Skipping file: Title contains "leaked": ' . $tor->{'title'} . "\n";
  1314. }
  1315. return 1;
  1316. # Skip pre-air files
  1317. } elsif ($tor->{'title'} =~ /preair/i) {
  1318. if ($DEBUG) {
  1319. print STDERR 'Skipping file: Title contains "preair": ' . $tor->{'title'} . "\n";
  1320. }
  1321. return 1;
  1322. # Skip SWESUB files
  1323. } elsif ($tor->{'title'} =~ /swesub/i) {
  1324. if ($DEBUG) {
  1325. print STDERR 'Skipping file: Title contains "SWESUB": ' . $tor->{'title'} . "\n";
  1326. }
  1327. return 1;
  1328. # Skip .rus. files
  1329. } elsif ($tor->{'title'} =~ /\.rus\./i) {
  1330. if ($DEBUG) {
  1331. print STDERR 'Skipping file: Title contains ".rus.": ' . $tor->{'title'} . "\n";
  1332. }
  1333. return 1;
  1334. # Skip German files
  1335. } elsif ($tor->{'title'} =~ /german/i) {
  1336. if ($DEBUG) {
  1337. print STDERR 'Skipping file: Title contains "german": ' . $tor->{'title'} . "\n";
  1338. }
  1339. return 1;
  1340. # Skip French files
  1341. } elsif ($tor->{'title'} =~ /french/i) {
  1342. if ($DEBUG) {
  1343. print STDERR 'Skipping file: Title contains "french": ' . $tor->{'title'} . "\n";
  1344. }
  1345. return 1;
  1346. # Skip Spanish files
  1347. } elsif ($tor->{'title'} =~ /Español/i) {
  1348. if ($DEBUG) {
  1349. print STDERR 'Skipping file: Title contains "Español": ' . $tor->{'title'} . "\n";
  1350. }
  1351. return 1;
  1352. # Skip Castellano files
  1353. } elsif ($tor->{'title'} =~ /Castellano/i) {
  1354. if ($DEBUG) {
  1355. print STDERR 'Skipping file: Title contains "Castellano": ' . $tor->{'title'} . "\n";
  1356. }
  1357. return 1;
  1358. # Skip unedited files
  1359. } elsif ($tor->{'title'} =~ /unedited/i) {
  1360. if ($DEBUG) {
  1361. print STDERR 'Skipping file: Title contains "unedited": ' . $tor->{'title'} . "\n";
  1362. }
  1363. return 1;
  1364. }
  1365. # If we're still around the TOR is good
  1366. return 0;
  1367. }
  1368. sub seedCleanup($) {
  1369. my ($tor) = @_;
  1370. # Cap seeds/leaches/date and ensure they are always defined
  1371. if (!exists($tor->{'seeds'}) || !$tor->{'seeds'}) {
  1372. $tor->{'seeds'} = 0;
  1373. }
  1374. if ($tor->{'seeds'} > $CONFIG{'MAX_SEEDS'}) {
  1375. $tor->{'seeds'} = $CONFIG{'MAX_SEEDS'};
  1376. }
  1377. if (!exists($tor->{'leaches'}) || !$tor->{'leaches'}) {
  1378. $tor->{'leaches'} = 0;
  1379. }
  1380. if ($tor->{'leaches'} > $CONFIG{'MAX_SEEDS'}) {
  1381. $tor->{'leaches'} = $CONFIG{'MAX_SEEDS'};
  1382. }
  1383. # Proxy publication date to seeder/leacher quality
  1384. if ($tor->{'date'} && !$tor->{'seeds'}) {
  1385. my $age = time() - $tor->{'date'};
  1386. if ($age < 0) {
  1387. warn('Invalid date (' . $tor->{'date'} . '): ' . $tor->{'name'} . "\n");
  1388. $age = 0;
  1389. }
  1390. $age /= 86400;
  1391. if ($age > $CONFIG{'NZB_AGE_GOOD'}) {
  1392. $tor->{'seeds'} = $CONFIG{'MIN_COUNT'};
  1393. if ($age > $CONFIG{'NZB_AGE_MAX'}) {
  1394. $tor->{'seeds'} = ($CONFIG{'MIN_COUNT'} / 2) - 1;
  1395. }
  1396. } else {
  1397. my $penalty = ($CONFIG{'MAX_SEEDS'} / 2) / $CONFIG{'NZB_AGE_GOOD'};
  1398. $tor->{'seeds'} = $CONFIG{'MAX_SEEDS'} - ($age * $penalty);
  1399. }
  1400. $tor->{'seeds'} = int($tor->{'seeds'});
  1401. $tor->{'leaches'} = $tor->{'seeds'};
  1402. }
  1403. }
  1404. sub lowQualityTor($) {
  1405. my ($tor) = @_;
  1406. # Skip torrents with too few seeders/leachers
  1407. if (($tor->{'seeds'} + $tor->{'leaches'}) * $SOURCES->{ $tor->{'source'} }->{'weight'} < $CONFIG{'MIN_COUNT'}) {
  1408. if ($DEBUG) {
  1409. print STDERR 'Skipping file: Insufficient seeder/leacher count ('
  1410. . $tor->{'seeds'} . '/'
  1411. . $tor->{'leaches'} . '): '
  1412. . $tor->{'title'} . "\n";
  1413. }
  1414. return 1;
  1415. # Skip torrents with unusual seed/leach ratios
  1416. } elsif ($tor->{'seeds'} > 1
  1417. && $tor->{'seeds'} < $CONFIG{'SEED_RATIO_COUNT'}
  1418. && $tor->{'seeds'} > $tor->{'leaches'} * $CONFIG{'MAX_SEED_RATIO'})
  1419. {
  1420. if ($DEBUG) {
  1421. print STDERR 'Skipping file: Unusual seeder/leacher ratio ('
  1422. . $tor->{'seeds'} . '/'
  1423. . $tor->{'leaches'} . '): '
  1424. . $tor->{'title'} . "\n";
  1425. }
  1426. return 1;
  1427. # Skip torrents that are too small
  1428. } elsif ($tor->{'size'} < $CONFIG{'MIN_SIZE'}) {
  1429. if ($DEBUG) {
  1430. print STDERR 'Skipping file: Insufficient size (' . $tor->{'size'} . ' MiB): ' . $tor->{'title'} . "\n";
  1431. }
  1432. return 1;
  1433. }
  1434. # If we're still around the TOR is good
  1435. return 0;
  1436. }
  1437. sub getHash($) {
  1438. my ($tor) = @_;
  1439. my $url;
  1440. # Allow either a URL or a tor hashref
  1441. if (ref($tor) eq 'HASH') {
  1442. $url = $tor->{'url'};
  1443. } else {
  1444. $url = $tor;
  1445. undef($tor);
  1446. }
  1447. # Extract the BTIH hash or NZB ID, if available
  1448. my $hash = undef();
  1449. if ($url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1450. $hash = lc($1);
  1451. } elsif ($url =~ /\/getnzb\/(\w+)\.nzb\&/i) {
  1452. $hash = lc($1);
  1453. }
  1454. # Save back to the tor, if provided
  1455. if (defined($tor) && defined($hash)) {
  1456. $tor->{'hash'} = lc($hash);
  1457. }
  1458. # Always return the hash, or undef() if none can be found
  1459. return $hash;
  1460. }
  1461. # Resolve secondary URLs
  1462. sub resolveSecondary($) {
  1463. my ($tor) = @_;
  1464. # Fetch the torrent-specific page and extract the magent link
  1465. if ($tor->{'source'} eq 'ISO') {
  1466. my ($content, $code) = fetch($tor->{'url'}, 'secondary.html');
  1467. if ($code != 200) {
  1468. print STDERR 'Error fetching secondary URL: ' . $tor->{'url'} . "\n";
  1469. goto OUT;
  1470. }
  1471. my ($magnet) = $content =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  1472. if (!defined($magnet) || length($magnet) < 1) {
  1473. if ($DEBUG) {
  1474. print STDERR 'No secondary URL available from: ' . $tor->{'url'} . "\n";
  1475. }
  1476. goto OUT;
  1477. }
  1478. $tor->{'url'} = decode_entities($magnet);
  1479. }
  1480. # Extract the hash
  1481. getHash($tor);
  1482. if (!$tor->{'hash'} || $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  1483. if ($DEBUG) {
  1484. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1485. }
  1486. goto OUT;
  1487. }
  1488. OUT:
  1489. return $tor->{'url'};
  1490. }
  1491. sub resolveTrackers($) {
  1492. my ($url) = @_;
  1493. # Always append a few static trackers to the URI
  1494. if ($url =~ /^magnet\:/i) {
  1495. foreach my $tracker (@{ $CONFIG{'TRACKERS'} }) {
  1496. my $arg = '&tr=' . uri_encode($tracker, { 'encode_reserved' => 1 });
  1497. my $match = quotemeta($arg);
  1498. if (!($url =~ /${match}/)) {
  1499. $url .= $arg;
  1500. }
  1501. }
  1502. }
  1503. # Fetch a dynamic tracker list for any hashinfo from TorrentZ
  1504. if ($CONFIG{'TRACKER_LOOKUP'} && defined($SOURCES->{'Z'}) && $url =~ /^magnet\:/ && $url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1505. my $hash = lc($1);
  1506. my $baseURL = $SOURCES->{'Z'}->{'protocol'} . '://' . $SOURCES->{'Z'}->{'host'};
  1507. my $hashURL = $baseURL . '/' . $hash;
  1508. my ($content, $code) = fetch($hashURL, 'hashinfo.html');
  1509. if ($code != 200) {
  1510. print STDERR 'Error fetching hashinfo URL: ' . $hashURL . "\n";
  1511. return $url;
  1512. }
  1513. # Parse out and fetch the tracker list URL
  1514. my ($list) = $content =~ /\<a rel\=\"nofollow\" href\=\"(\/announcelist_\d+)\"\>/;
  1515. if (!defined($list)) {
  1516. if ($DEBUG) {
  1517. print STDERR 'Unable to find tracker list link at: ' . $hashURL . "\n";
  1518. }
  1519. return $url;
  1520. }
  1521. my $listURL = $baseURL . $list;
  1522. ($content, $code) = fetch($listURL, 'tracker.html');
  1523. if ($code != 200) {
  1524. print STDERR 'Error fetching tracker list URL: ' . $listURL . "\n";
  1525. return $url;
  1526. }
  1527. # Parse the tracker list and append our URL
  1528. foreach my $line ($content) {
  1529. if ($line =~ /^\s*$/) {
  1530. next;
  1531. }
  1532. $line =~ s/^\s+//;
  1533. $line =~ s/\s+$//;
  1534. if ($line =~ /^(?:http|udp)\:/i) {
  1535. my $arg = '&tr=' . uri_encode($line, { 'encode_reserved' => 1 });
  1536. my $match = quotemeta($arg);
  1537. if (!($url =~ /${match}/)) {
  1538. if ($DEBUG) {
  1539. print STDERR 'Adding tracker: ' . $line . "\n";
  1540. }
  1541. $url .= $arg;
  1542. }
  1543. } else {
  1544. print STDERR 'Unknown tracker type: ' . $line . "\n";
  1545. }
  1546. }
  1547. }
  1548. return $url;
  1549. }
  1550. sub splitTags($$$$) {
  1551. my ($content, $tag, $header, $match) = @_;
  1552. # Find each tag element
  1553. my @out = ();
  1554. my @parts = split(/\<${tag}(?:\s+[^\>]*)?\>/i, $content);
  1555. foreach my $part (@parts) {
  1556. # Trim trailing tags and skip things that aren't complete tags
  1557. if (!($part =~ s/\<\/${tag}\>.*$//is)) {
  1558. if ($DEBUG > 2) {
  1559. print STDERR 'Skipping non-tag line: ' . $part . "\n\n";
  1560. }
  1561. next;
  1562. }
  1563. # Skip headers
  1564. if (defined($header) && $part =~ /<${header}(?:\s+[^\>]*)?\>/i) {
  1565. if ($DEBUG > 2) {
  1566. print STDERR 'Skipping header line: ' . $part . "\n\n";
  1567. }
  1568. next;
  1569. }
  1570. # Skip non-matches
  1571. if (defined($match) && !($part =~ m/${match}/i)) {
  1572. if ($DEBUG > 2) {
  1573. print STDERR 'Skipping non-match line: ' . $part . "\n\n";
  1574. }
  1575. next;
  1576. }
  1577. # Save good tags
  1578. push(@out, $part);
  1579. }
  1580. return @out;
  1581. }
  1582. # Extract season
  1583. sub findSE($) {
  1584. my ($name) = @_;
  1585. my $season = 0;
  1586. my $episode = 0;
  1587. my $seasonBlock = '';
  1588. if ($name =~ /(?:\b|_)(20\d\d(?:\.|\-)[01]?\d(?:\.|\-)[0-3]?\d)(?:\b|_)/) {
  1589. $seasonBlock = $1;
  1590. my ($month, $day);
  1591. ($season, $month, $day) = $seasonBlock =~ /(20\d\d)(?:\.|\-)([01]?\d)(?:\.|\-)([0-3]?\d)/;
  1592. $episode = sprintf('%04d-%02d-%02d', $season, $month, $day);
  1593. } elsif ($name =~ /(?:\b|_)(S(?:eason)?[_\s\.\-]*\d{1,2}[_\s\.\-]*E(?:pisode)?[_\s\.]*\d{1,3}(?:E\d{1,3})?)(?:\b|_)/i) {
  1594. $seasonBlock = $1;
  1595. ($season, $episode) = $seasonBlock =~ /S(?:eason)?[_\s\.\-]*(\d{1,2})[_\s\.\-]*E(?:pisode)?[_\s\.\-]*(\d{1,3})/i;
  1596. } elsif ($name =~ /[\[\_\.](\d{1,2}x\d{2,3})[\]\_\.]/i) {
  1597. $seasonBlock = $1;
  1598. ($season, $episode) = $seasonBlock =~ /(\d+)x(\d+)/i;
  1599. } elsif ($name =~ /(?:\b|_)([01]?\d[_\s\.]?[0-3]\d)(?:\b|_)/i) {
  1600. $seasonBlock = $1;
  1601. ($season, $episode) = $seasonBlock =~ /(?:\b|_)([01]?\d)[_\s\.]?([0-3]\d)/i;
  1602. }
  1603. # Return something valid and INTy or UNDEF
  1604. if (!defined($seasonBlock) || $season < 1 || length($episode) < 1 || $episode eq '0') {
  1605. if ($DEBUG) {
  1606. print STDERR 'Could not find seasonBlock in: ' . $name . "\n";
  1607. }
  1608. $season = undef();
  1609. $episode = undef();
  1610. } elsif ($episode =~ /^20\d{2}\-\d{2}\-\d{2}$/) {
  1611. # Episode-by-date
  1612. $season = int($season);
  1613. } else {
  1614. # Catchall
  1615. # This will warn and return undef() (which is valid) on error
  1616. $season = int($season);
  1617. $episode = int($episode);
  1618. }
  1619. return ($season, $episode);
  1620. }
  1621. sub initSources() {
  1622. my %sources = ();
  1623. # NZBplanet.net
  1624. if ($CONFIG{'SOURCES'}->{'PLANET'}) {
  1625. my @proxies = ('api.nzbplanet.net/api');
  1626. my $source = findProxy(\@proxies, '\bNzbplanet\b');
  1627. if ($source && exists($CONFIG{'PLANET_APIKEY'}) && $CONFIG{'PLANET_APIKEY'}) {
  1628. $source->{'weight'} = 1.00;
  1629. $source->{'quote'} = 0;
  1630. $source->{'search_exclude'} = 0;
  1631. $source->{'search_suffix'} = '';
  1632. $source->{'custom_search'} = sub ($$$$) {
  1633. my ($urls, $series, $season, $episode) = @_;
  1634. my $url =
  1635. $source->{'search_url'}
  1636. . '?o=json&t=tvsearch&q='
  1637. . $series
  1638. . '&season='
  1639. . $season . '&ep='
  1640. . $episode
  1641. . '&apikey='
  1642. . $CONFIG{'PLANET_APIKEY'};
  1643. push(@{$urls}, $url);
  1644. };
  1645. $sources{'PLANET'} = $source;
  1646. }
  1647. }
  1648. # NZB.is
  1649. if ($CONFIG{'SOURCES'}->{'IS'}) {
  1650. my @proxies = ('nzb.is/api');
  1651. my $source = findProxy(\@proxies, '\bnzb\.is\b');
  1652. if ($source && exists($CONFIG{'IS_APIKEY'}) && $CONFIG{'IS_APIKEY'}) {
  1653. $source->{'weight'} = 1.00;
  1654. $source->{'quote'} = 0;
  1655. $source->{'search_exclude'} = 0;
  1656. $source->{'search_suffix'} = '';
  1657. $source->{'custom_search'} = sub ($$$$) {
  1658. my ($urls, $series, $season, $episode) = @_;
  1659. my $url =
  1660. $source->{'search_url'}
  1661. . '?o=json&t=tvsearch&q='
  1662. . $series
  1663. . '&season='
  1664. . $season . '&ep='
  1665. . $episode
  1666. . '&apikey='
  1667. . $CONFIG{'IS_APIKEY'};
  1668. push(@{$urls}, $url);
  1669. };
  1670. $sources{'IS'} = $source;
  1671. }
  1672. }
  1673. # Usenet Crawler
  1674. if ($CONFIG{'SOURCES'}->{'UC'}) {
  1675. my @proxies = ('www.usenet-crawler.com/api');
  1676. my $source = findProxy(\@proxies, '\busenet-crawler\b');
  1677. if ($source && exists($CONFIG{'UC_APIKEY'}) && $CONFIG{'UC_APIKEY'}) {
  1678. $source->{'weight'} = 1.00;
  1679. $source->{'quote'} = 0;
  1680. $source->{'search_exclude'} = 0;
  1681. $source->{'search_suffix'} = '';
  1682. $source->{'custom_search'} = sub ($$$$) {
  1683. my ($urls, $series, $season, $episode) = @_;
  1684. my $url =
  1685. $source->{'search_url'}
  1686. . '?o=json&t=tvsearch&q='
  1687. . $series
  1688. . '&season='
  1689. . $season . '&ep='
  1690. . $episode
  1691. . '&apikey='
  1692. . $CONFIG{'UC_APIKEY'};
  1693. push(@{$urls}, $url);
  1694. };
  1695. $sources{'UC'} = $source;
  1696. }
  1697. }
  1698. # NZB Cat
  1699. if ($CONFIG{'SOURCES'}->{'NCAT'}) {
  1700. my @proxies = ('nzb.cat/api');
  1701. my $source = findProxy(\@proxies, '\bforgottenpassword\b');
  1702. if ($source && exists($CONFIG{'NCAT_APIKEY'}) && $CONFIG{'NCAT_APIKEY'}) {
  1703. $source->{'weight'} = 1.00;
  1704. $source->{'quote'} = 0;
  1705. $source->{'search_exclude'} = 0;
  1706. $source->{'search_suffix'} = '';
  1707. $source->{'custom_search'} = sub ($$$$) {
  1708. my ($urls, $series, $season, $episode) = @_;
  1709. my $url =
  1710. $source->{'search_url'}
  1711. . '?o=json&t=tvsearch&q='
  1712. . $series
  1713. . '&season='
  1714. . $season . '&ep='
  1715. . $episode
  1716. . '&apikey='
  1717. . $CONFIG{'NCAT_APIKEY'};
  1718. push(@{$urls}, $url);
  1719. };
  1720. $sources{'NCAT'} = $source;
  1721. }
  1722. }
  1723. # The Pirate Bay
  1724. if ($CONFIG{'SOURCES'}->{'TPB'}) {
  1725. my @proxies = ('thepiratebay.org/search/', 'tpb.unblocked.co/search/');
  1726. my $source = findProxy(\@proxies, '\bPirate Search\b');
  1727. if ($source) {
  1728. $source->{'weight'} = 1.00;
  1729. $source->{'quote'} = 0;
  1730. $source->{'search_exclude'} = 1;
  1731. $source->{'search_suffix'} = '/0/7/0';
  1732. $sources{'TPB'} = $source;
  1733. }
  1734. }
  1735. # ISOhunt
  1736. if ($CONFIG{'SOURCES'}->{'ISO'}) {
  1737. my @proxies = ('isohunt.to/torrents/?ihq=', 'isohunters.net/torrents/?ihq=');
  1738. my $source = findProxy(\@proxies, 'Last\s+\d+\s+files\s+indexed');
  1739. if ($source) {
  1740. $source->{'weight'} = 0.75;
  1741. $source->{'quote'} = 1;
  1742. $source->{'search_exclude'} = 1;
  1743. $source->{'search_suffix'} = '';
  1744. $sources{'ISO'} = $source;
  1745. }
  1746. }
  1747. # Kickass
  1748. if ($CONFIG{'SOURCES'}->{'KICK'}) {
  1749. my @proxies = ('kickass.cd/search.php?q=', 'kickass.mx/search.php?q=');
  1750. my $source = findProxy(\@proxies, '/search.php');
  1751. if ($source) {
  1752. $source->{'weight'} = 1.00;
  1753. $source->{'quote'} = 1;
  1754. $source->{'search_exclude'} = 1;
  1755. $source->{'search_suffix'} = '/';
  1756. $sources{'KICK'} = $source;
  1757. }
  1758. }
  1759. # Torrentz
  1760. if ($CONFIG{'SOURCES'}->{'Z'}) {
  1761. my @proxies = ('torrentz.eu/search?q=', 'torrentz.me/search?q=', 'torrentz.ch/search?q=', 'torrentz.in/search?q=');
  1762. my $source = findProxy(\@proxies, 'Indexing [\d\,]+ active torrents');
  1763. if ($source) {
  1764. $source->{'weight'} = 1.00;
  1765. $source->{'quote'} = 1;
  1766. $source->{'search_exclude'} = 1;
  1767. $source->{'search_suffix'} = '';
  1768. if (!$CONFIG{'NO_QUALITY_CHECKS'}) {
  1769. $source->{'search_suffix'} = '+peer+%3E+' . $CONFIG{'MIN_COUNT'};
  1770. }
  1771. $sources{'Z'} = $source;
  1772. }
  1773. }
  1774. # ExtraTorrent
  1775. if ($CONFIG{'SOURCES'}->{'ET'}) {
  1776. my @proxies = (
  1777. 'extra.to/search/?search=', 'etmirror.com/search/?search=',
  1778. 'etproxy.com/search/?search=', 'extratorrentlive.com/search/?search='
  1779. );
  1780. my $source = findProxy(\@proxies, '/search/');
  1781. if ($source) {
  1782. $source->{'weight'} = 1.00;
  1783. $source->{'quote'} = 1;
  1784. $source->{'search_exclude'} = 0;
  1785. $source->{'search_suffix'} = '&new=1&x=0&y=0&srt=seeds&order=desc';
  1786. $source->{'search_url'} = 'phantomjs:' . $source->{'search_url'};
  1787. $sources{'ET'} = $source;
  1788. }
  1789. }
  1790. # Sanity check
  1791. if (scalar(keys(%sources)) < 1) {
  1792. die("No sources available\n");
  1793. }
  1794. return \%sources;
  1795. }
  1796. sub findProxy($$) {
  1797. my ($proxies, $match) = @_;
  1798. # Automatically select a proxy
  1799. my $host = '';
  1800. my $protocol = '';
  1801. my $search_url = '';
  1802. foreach my $url (@{$proxies}) {
  1803. my $path;
  1804. ($protocol, $host, $path) = $url =~ /^(?:(https?)\:\/\/)?([^\/]+)(\/.*)?$/i;
  1805. if (!$protocol) {
  1806. $protocol = $CONFIG{'PROTOCOL'};
  1807. }
  1808. if (!$host) {
  1809. print STDERR 'Could not parse proxy URL: ' . $url . "\n";
  1810. next;
  1811. }
  1812. my ($content, $code) = fetch($protocol . '://' . $host, undef(), 1);
  1813. if ($code == 200 && $content =~ m/${match}/i) {
  1814. $search_url = $protocol . '://' . $host . $path;
  1815. last;
  1816. } elsif ($DEBUG) {
  1817. print STDERR 'Proxy not available: ' . $host . "\n";
  1818. if ($DEBUG > 1) {
  1819. print STDERR $content . "\n";
  1820. }
  1821. }
  1822. }
  1823. # Return the search data if at least one proxy is up
  1824. if ($search_url) {
  1825. my %tmp = (
  1826. 'protocol' => $protocol,
  1827. 'host' => $host,
  1828. 'search_url' => $search_url,
  1829. );
  1830. return \%tmp;
  1831. }
  1832. # Return undef if we didn't find any working proxies
  1833. return undef();
  1834. }
  1835. sub fetch($;$$) {
  1836. my ($url, $file, $nocheck) = @_;
  1837. if ($DEBUG > 1) {
  1838. print STDERR 'Fetch: ' . $url . "\n";
  1839. }
  1840. # Allow usage with short names
  1841. if ($file && !($file =~ /^\//)) {
  1842. $file = '/tmp/findTorrent-' . $file;
  1843. }
  1844. # Random delay
  1845. if ($DEBUG > 1) {
  1846. sleep($CONFIG{'SLEEP'} * 0.25);
  1847. } else {
  1848. sleep($CONFIG{'DELAY'} * (rand($CONFIG{'SLEEP'}) + ($CONFIG{'SLEEP'} / 2)));
  1849. }
  1850. # Allow JS processing
  1851. if ($url =~ /^phantomjs\:\s*(\S.*)$/i) {
  1852. return phantomFetch($1, $file);
  1853. }
  1854. # Fetch object
  1855. if (!$FETCH) {
  1856. if (!$COOKIES) {
  1857. $COOKIES = mktemp('/tmp/findTorrent.cookies.XXXXXXXX');
  1858. }
  1859. $FETCH = Fetch->new(
  1860. 'cookiefile' => $COOKIES,
  1861. 'timeout' => $CONFIG{'TIMEOUT'},
  1862. 'uas' => $CONFIG{'UA'}
  1863. );
  1864. }
  1865. # Debug as requested
  1866. if ($DEBUG > 1 && $file) {
  1867. $FETCH->file($file);
  1868. } else {
  1869. $FETCH->file('');
  1870. }
  1871. # Standard fetch
  1872. $FETCH->url($url);
  1873. if ($nocheck) {
  1874. $FETCH->fetch('nocheck' => 1);
  1875. } else {
  1876. $FETCH->fetch();
  1877. }
  1878. return (scalar($FETCH->content()), $FETCH->status_code());
  1879. }
  1880. sub phantomFetch($$) {
  1881. my ($url, $file) = @_;
  1882. # Init
  1883. if (!$SCRIPT) {
  1884. $SCRIPT = mktemp('/tmp/findTorrent.phantom.XXXXXXXX');
  1885. open(my $fh, '>', $SCRIPT)
  1886. or die('Unable to open PhantomJS script file: ' . $! . "\n");
  1887. print $fh $CONFIG{'PHANTOM_CONFIG'};
  1888. close($fh);
  1889. }
  1890. # Capture STDOUT, drop STDERR
  1891. my @cmd = ($CONFIG{'PHANTOM_BIN'}, $SCRIPT, $url);
  1892. my ($content, undef(), undef()) = capture { system(@cmd); };
  1893. # Debug as requested
  1894. if ($DEBUG > 1 && $file) {
  1895. open(my $fh, '>', $file)
  1896. or warn('Unable to open phantom debug file: ' . $! . "\n");
  1897. if ($fh) {
  1898. print $fh $content;
  1899. close($fh);
  1900. }
  1901. }
  1902. # Try to get the response status code, fake one if we cannot
  1903. my ($code) = $content =~ /^(\d\d\d)$/m;
  1904. if (!defined($code)) {
  1905. warn("Unable to retrive HTTP status code. Faking success...\n");
  1906. $code = 200;
  1907. }
  1908. return ($content, $code);
  1909. }