findTorrent.pl 56 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. # Includes
  5. use Date::Parse;
  6. use URI::Encode qw(uri_encode);
  7. use HTML::Entities;
  8. use HTML::Strip;
  9. use JSON;
  10. use Sys::Syslog qw(:standard :macros);
  11. use Capture::Tiny ':all';
  12. use File::Temp;
  13. use File::Spec;
  14. use File::Basename;
  15. use FindBin qw($Bin);
  16. use lib $Bin;
  17. use Fetch;
  18. # Prototypes
  19. sub confDefault($;$);
  20. sub readGlobalExcludes($);
  21. sub readInputDir($);
  22. sub parseNZB($);
  23. sub badTor($);
  24. sub seedCleanup($);
  25. sub lowQualityTor($);
  26. sub getHash($);
  27. sub resolveSecondary($);
  28. sub resolveTrackers($);
  29. sub splitTags($$$$);
  30. sub findSE($);
  31. sub initSources();
  32. sub findProxy($$);
  33. sub fetch($;$$);
  34. sub phantomFetch($$);
  35. # Globals
  36. my %CONFIG = ();
  37. my $SOURCES = undef();
  38. my $COOKIES = undef();
  39. my $FETCH = undef();
  40. my $SCRIPT = undef();
  41. # Command line
  42. my ($dir, $search) = @ARGV;
  43. if (!defined($dir)) {
  44. die('Usage: ' . basename($0) . " input_directory [search_string]\n");
  45. }
  46. # Pre-config environment
  47. my $DEBUG = 0;
  48. if ($ENV{'DEBUG'}) {
  49. if ($ENV{'DEBUG'} =~ /(\d+)/) {
  50. $DEBUG = $1;
  51. } else {
  52. $DEBUG = 1;
  53. }
  54. }
  55. my $CONF_FILE = $ENV{'HOME'} . '/.findTorrent.config';
  56. if (defined($ENV{'CONF_FILE'})) {
  57. $CONF_FILE = $ENV{'CONF_FILE'};
  58. }
  59. # Read the config file
  60. if ($CONF_FILE && -r $CONF_FILE) {
  61. my $fh;
  62. open($fh, '<', $CONF_FILE)
  63. or die('Unable to open config file: ' . $CONF_FILE . ': ' . $! . "\n");
  64. while (<$fh>) {
  65. # Skip blank lines and comments
  66. if (/^\s*#/ || /^\s*$/) {
  67. next;
  68. }
  69. # Assume everything else is key = val pairs
  70. if (my ($key, $val) = $_ =~ /^\s*(\S[^\=]+)\=\>?\s*(\S.*)$/) {
  71. $key =~ s/^\s+//;
  72. $key =~ s/\s+$//;
  73. $val =~ s/^\s+//;
  74. $val =~ s/\s+$//;
  75. $CONFIG{$key} = $val;
  76. if ($DEBUG > 1) {
  77. print STDERR 'Adding config: ' . $key . ' => ' . $val . "\n";
  78. }
  79. } else {
  80. warn('Ignoring config line: ' . $_ . "\n");
  81. }
  82. }
  83. close($fh);
  84. }
  85. # Config defaults
  86. confDefault('EXCLUDES_FILE', $ENV{'HOME'} . '/.findTorrent.exclude');
  87. confDefault('TV_DIR', `~/bin/video/mediaPath` . '/TV');
  88. confDefault('PHANTOM_BIN', '/usr/local/bin/phantomjs');
  89. confDefault('UA',
  90. 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/538.39.41 (KHTML, like Gecko) Version/8.0 Safari/538.39.41');
  91. confDefault('PROTOCOL', 'https');
  92. confDefault('SLEEP', 1);
  93. confDefault('DELAY', 5);
  94. confDefault('TIMEOUT', 15);
  95. confDefault('ERR_DELAY', $CONFIG{'TIMEOUT'} * 2);
  96. confDefault('ERR_RETRIES', 3);
  97. confDefault('MORE_NUMBER_FORMATS', 0);
  98. confDefault('NO_QUALITY_CHECKS', 0);
  99. confDefault('NEXT_EPISODES', 3);
  100. confDefault('MIN_DAYS_BACK', 0);
  101. confDefault('MAX_DAYS_BACK', 3);
  102. confDefault('SYSLOG', 1);
  103. confDefault('MIN_COUNT', 10);
  104. confDefault('MIN_SIZE', 100);
  105. confDefault('SIZE_BONUS', 5);
  106. confDefault('SIZE_PENALTY', $CONFIG{'SIZE_BONUS'});
  107. confDefault('TITLE_PENALTY', $CONFIG{'SIZE_BONUS'} / 2);
  108. confDefault('MAX_SEEDS', 500);
  109. confDefault('MAX_SEED_RATIO', 0.25);
  110. confDefault('SEED_RATIO_COUNT', $CONFIG{'MIN_COUNT'});
  111. confDefault('TRACKER_LOOKUP', 1);
  112. confDefault(
  113. 'PHANTOM_CONFIG', "var system = require('system');
  114. var page = require('webpage').create();
  115. var resources = [];
  116. page.open(system.args[1], function(status)
  117. {
  118. console.log(resources[0].status);
  119. console.log(page.content);
  120. phantom.exit();
  121. });
  122. // Add responses for completed text/html resources
  123. page.onResourceReceived = function(response) {
  124. if (response.stage !== 'end') return;
  125. if (response.headers.filter(function(header) {
  126. if (header.name == 'Content-Type' && header.value.indexOf('text/html') == 0) {
  127. return true;
  128. }
  129. return false;
  130. }).length > 0)
  131. resources.push(response);
  132. };"
  133. );
  134. # Re-parse a few config items
  135. if (exists($CONFIG{'TRACKERS'})) {
  136. my @trackers = ();
  137. foreach my $tracker (split(/\s*,\s*/, $CONFIG{'TRACKERS'})) {
  138. $tracker =~ s/^\s+//;
  139. $tracker =~ s/\s+$//;
  140. push(@trackers, $tracker);
  141. }
  142. $CONFIG{'TRACKERS'} = \@trackers;
  143. }
  144. if (exists($CONFIG{'SOURCES'})) {
  145. my %sources = ();
  146. foreach my $source (split(/\s*,\s*/, $CONFIG{'SOURCES'})) {
  147. $source =~ s/^\s+//;
  148. $source =~ s/\s+$//;
  149. $source = uc($source);
  150. $sources{$source} = 1;
  151. }
  152. $CONFIG{'SOURCES'} = \%sources;
  153. }
  154. # Sanity checks
  155. if (ref($CONFIG{'SOURCES'}) ne 'HASH' || !exists($CONFIG{'SOURCES'}) || scalar(keys(%{ $CONFIG{'SOURCES'} })) < 1) {
  156. die("No sources configured\n");
  157. }
  158. if (ref($CONFIG{'TRACKERS'}) ne 'ARRAY' || !exists($CONFIG{'TRACKERS'}) || scalar(@{ $CONFIG{'TRACKERS'} }) < 1) {
  159. warn("No trackers configured\n");
  160. }
  161. if (!exists($CONFIG{'NZB_AGE_GOOD'}) || !exists($CONFIG{'NZB_AGE_MAX'})) {
  162. warn("No NZB good/max age provided\n");
  163. }
  164. # Open the log if enabled
  165. if ($CONFIG{'SYSLOG'}) {
  166. openlog(basename($0), '', LOG_DAEMON);
  167. }
  168. # Read the torrent excludes list
  169. $CONFIG{'EXCLUDES'} = readGlobalExcludes($CONFIG{'EXCLUDES_FILE'});
  170. # Setup our sources
  171. $SOURCES = initSources();
  172. $CONFIG{'DELAY'} /= scalar(keys(%{$SOURCES})) / 2;
  173. # Figure out what we're searching for
  174. my @urls = ();
  175. my $CUSTOM_SEARCH = 0;
  176. my $SERIES = readInputDir($dir);
  177. # Handle custom searches
  178. if ((scalar(@urls) < 1) && defined($search) && length($search) > 0) {
  179. # Note the custom search string
  180. $CUSTOM_SEARCH = 1;
  181. if ($DEBUG) {
  182. print STDERR "Custom search\n";
  183. }
  184. # Create the relevent search strings
  185. foreach my $key (keys(%{$SOURCES})) {
  186. my $source = $SOURCES->{$key};
  187. push(@urls, $source->{'search_url'} . $search . $SERIES->{'exclude'} . $source->{'search_suffix'});
  188. }
  189. }
  190. # Handle search-by-date series
  191. if ((scalar(@urls) < 1) && -e $SERIES->{'path'} . '/search_by_date') {
  192. # Note the search-by-date status
  193. # Set the CUSTOM_SERACH flag to skip season/epsiode matching
  194. $CUSTOM_SEARCH = 1;
  195. # Read the find-by-date string
  196. my $search_by_date = '';
  197. local ($/, *FH);
  198. open(FH, $SERIES->{'path'} . '/search_by_date')
  199. or die('Unable to read search_by_date for show: ' . $SERIES->{'name'} . ': ' . $! . "\n");
  200. my $text = <FH>;
  201. close(FH);
  202. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  203. $search_by_date = $1;
  204. } else {
  205. die('Skipping invalid search_by_date for show: ' . $SERIES->{'name'} . ': ' . $text . "\n");
  206. }
  207. # Create search strings for each date in the range, unless the related file already exists
  208. my (%years, %months, %days) = ();
  209. for (my $days_back = $CONFIG{'MIN_DAYS_BACK'} ; $days_back <= $CONFIG{'MAX_DAYS_BACK'} ; $days_back++) {
  210. # Calculate the date
  211. my (undef(), undef(), undef(), $day, $month, $year) = localtime(time() - (86400 * $days_back));
  212. # Format as strings
  213. $year = sprintf('%04d', $year + 1900);
  214. $month = sprintf('%02d', $month + 1);
  215. $day = sprintf('%02d', $day);
  216. # Check for an existing file
  217. my $exists = 0;
  218. {
  219. my $season_dir = $SERIES->{'path'} . '/Season ' . $year;
  220. my $prefix = qr/${year}\-${month}\-${day}\s*\-\s*/;
  221. opendir(SEASON, $season_dir)
  222. or die('Unable to open season directory (' . $season_dir . '): ' . $! . "\n");
  223. while (my $file = readdir(SEASON)) {
  224. if ($file =~ $prefix) {
  225. $exists = 1;
  226. last;
  227. }
  228. }
  229. closedir(SEASON);
  230. }
  231. if ($exists) {
  232. next;
  233. }
  234. # Save all the date string components
  235. $years{$year} = 1;
  236. $months{$month} = 1;
  237. $days{$day} = 1;
  238. # Create the relevent search strings
  239. my $search_str = $search_by_date;
  240. $search_str =~ s/%Y/${year}/g;
  241. $search_str =~ s/%m/${month}/g;
  242. $search_str =~ s/%d/${day}/g;
  243. foreach my $key (keys(%{$SOURCES})) {
  244. my $source = $SOURCES->{$key};
  245. push(@urls, $source->{'search_url'} . $search_str . $SERIES->{'exclude'} . $source->{'search_suffix'});
  246. }
  247. }
  248. # Build a date string matching regex
  249. my $str = '\b(?:' . join('|', keys(%years)) . ')\b';
  250. $str .= '.*';
  251. $str .= '\b(?:' . join('|', keys(%months)) . ')\b';
  252. $str .= '.*';
  253. $str .= '\b(?:' . join('|', keys(%days)) . ')\b';
  254. $CUSTOM_SEARCH = qr/${str}/;
  255. # Debug
  256. if ($DEBUG) {
  257. print STDERR 'Searching with date template: ' . $str . "\n";
  258. }
  259. }
  260. # Handle standard series
  261. if (scalar(@urls) < 1) {
  262. # Construct a URL-friendly show name
  263. my $safeShow = $SERIES->{'name'};
  264. $safeShow =~ s/\s+\&\s+/ and /i;
  265. $safeShow =~ s/^\s*The\b//i;
  266. $safeShow =~ s/\s+\-\s+/ /g;
  267. $safeShow =~ s/[\'\:]//g;
  268. $safeShow =~ s/[^\w\"\-]+/ /g;
  269. $safeShow =~ s/^\s+//;
  270. $safeShow =~ s/\s+$//;
  271. $safeShow =~ s/\s\s+/ /g;
  272. $safeShow =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
  273. $safeShow =~ s/\%20/\+/g;
  274. # Calculate possible name variations
  275. my @urlShowVarients = ();
  276. {
  277. # Default name
  278. push(@urlShowVarients, $safeShow);
  279. # Search for both "and" and "&"
  280. if ($safeShow =~ /\+and\+/i) {
  281. my $tmp = $safeShow;
  282. $tmp =~ s/\+and\+/\+%26\+/;
  283. push(@urlShowVarients, $tmp);
  284. }
  285. }
  286. # Construct the URL for each title varient of each needed episode
  287. foreach my $urlShow (@urlShowVarients) {
  288. foreach my $episode ($SERIES->{'need'}) {
  289. my $episode_long = sprintf('%02d', $episode);
  290. my $season_long = sprintf('%02d', $SERIES->{'season'});
  291. foreach my $source (values(%{$SOURCES})) {
  292. # Allow custom handling
  293. if (defined($source->{'custom_search'}) && ref($source->{'custom_search'}) eq 'CODE') {
  294. $source->{'custom_search'}->(\@urls, $urlShow, $SERIES->{'season'}, $episode);
  295. next;
  296. }
  297. # Use quotes around the show name if the source needs them
  298. my $quote = '%22';
  299. if (!$source->{'quote'}) {
  300. $quote = '';
  301. }
  302. # Calculate the compete search prefix and suffix to simplify later concatenations
  303. my $prefix = $source->{'search_url'} . $quote . $urlShow . $quote;
  304. my $suffix = '';
  305. if ($source->{'search_exclude'}) {
  306. $suffix .= $SERIES->{'exclude'};
  307. }
  308. if ($source->{'search_suffix'}) {
  309. $suffix .= $source->{'search_suffix'};
  310. }
  311. # SXXEYY
  312. my $url = $prefix . '+s' . $season_long . 'e' . $episode_long . $suffix;
  313. push(@urls, $url);
  314. # Extra searches for shows that have lazy/non-standard number formats
  315. if ($CONFIG{'MORE_NUMBER_FORMATS'}) {
  316. # SXEY
  317. if ($season_long ne $SERIES->{'season'} || $episode_long ne $episode) {
  318. $url = $prefix . '+s' . $SERIES->{'season'} . 'e' . $episode . $suffix;
  319. push(@urls, $url);
  320. }
  321. # SXX EYY
  322. $url = $prefix . '+s' . $season_long . '+e' . $episode_long . $suffix;
  323. push(@urls, $url);
  324. # Season XX Episode YY
  325. $url = $prefix . '+season+' . $season_long . '+episode+' . $episode_long . $suffix;
  326. push(@urls, $url);
  327. # Series X Episode Y
  328. $url = $prefix . '+series+' . $SERIES->{'season'} . '+episode+' . $episode . $suffix;
  329. push(@urls, $url);
  330. # SxEE
  331. $url = $prefix . '+' . $SERIES->{'season'} . 'x' . $episode_long . $suffix;
  332. push(@urls, $url);
  333. # Season X
  334. if ($CONFIG{'NO_QUALITY_CHECKS'}) {
  335. $url = $prefix . '+Season+' . $SERIES->{'season'} . $suffix;
  336. push(@urls, $url);
  337. }
  338. }
  339. }
  340. }
  341. }
  342. }
  343. my @html_content = ();
  344. my @json_content = ();
  345. foreach my $url (@urls) {
  346. # Fetch the page
  347. my $errCount = 0;
  348. my $content = '';
  349. HTTP_ERR_LOOP: {
  350. my $code = 0;
  351. if ($DEBUG) {
  352. print STDERR 'Searching with URL: ' . $url . "\n";
  353. }
  354. # Fetch
  355. ($content, $code) = fetch($url, 'primary.html');
  356. # Check for useful errors
  357. if ($code == 404) {
  358. if ($DEBUG) {
  359. print STDERR "Skipping content from 404 response\n";
  360. }
  361. next;
  362. }
  363. # Check for less useful errors
  364. if ($code != 200) {
  365. if ($code >= 500 && $code <= 599) {
  366. if ($errCount > $CONFIG{'ERR_RETRIES'}) {
  367. print STDERR 'Unable to fetch URL: ' . $url . "\n";
  368. $errCount = 0;
  369. next;
  370. }
  371. if ($DEBUG) {
  372. print STDERR 'Retrying URL (' . $code . '): ' . $url . "\n";
  373. }
  374. $errCount++;
  375. sleep($CONFIG{'ERR_DELAY'});
  376. redo HTTP_ERR_LOOP;
  377. } else {
  378. print STDERR 'Error fetching URL (' . $code . '): ' . $url . "\n";
  379. }
  380. next;
  381. }
  382. }
  383. # Save the content, discriminating by data type
  384. if ($DEBUG > 1) {
  385. print STDERR 'Fetched ' . length($content) . " bytes\n";
  386. }
  387. if ($content =~ /^\s*[\{\[]/) {
  388. my $json = eval { decode_json($content); };
  389. if (defined($json) && ref($json)) {
  390. push(@json_content, $json);
  391. } elsif ($DEBUG) {
  392. print STDERR 'JSON parsing failure on: ' . $content . "\n";
  393. }
  394. } else {
  395. push(@html_content, scalar($content));
  396. }
  397. }
  398. # We need someplace to store parsed torrent data from the various sources
  399. my @tors = ();
  400. # Handle HTML content
  401. foreach my $content (@html_content) {
  402. if ($content =~ /\<title\>The Pirate Bay/i) {
  403. # Find TR elements from TPB
  404. my @trs = splitTags($content, 'TR', 'TH', undef());
  405. foreach my $tr (@trs) {
  406. # Find the show title
  407. my ($title) = $tr =~ /title\=\"Details\s+for\s+([^\"]*)\"/i;
  408. if (!defined($title) || length($title) < 1) {
  409. if ($DEBUG) {
  410. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  411. }
  412. next;
  413. }
  414. $title =~ s/^\s+//;
  415. # Extract the season and episode numbers
  416. my ($fileSeason, $episode) = findSE($title);
  417. # Extract the URL
  418. my ($url) = $tr =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  419. if (!defined($url) || length($url) < 1) {
  420. if ($DEBUG) {
  421. print STDERR "Skipping TR with no magnet URL\n";
  422. }
  423. next;
  424. }
  425. $url = decode_entities($url);
  426. # Count the sum of seeders and leachers
  427. my $seeds = 0;
  428. my $leaches = 0;
  429. if ($tr =~ /\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*$/i) {
  430. $seeds = $1;
  431. $leaches = $2;
  432. }
  433. # Extract the size (from separate column or inline)
  434. my $size = 0;
  435. my $unit = 'M';
  436. if ($tr =~ m/Size (\d+(?:\.\d+)?)\&nbsp\;(G|M)iB/) {
  437. $size = $1;
  438. $unit = $2;
  439. } elsif ($tr =~ m/(\d+(?:\.\d+)?)\&nbsp\;(G|M)iB\<\/[tT][dD]\>/) {
  440. $size = $1;
  441. $unit = $2;
  442. }
  443. if ($unit eq 'G') {
  444. $size *= 1024;
  445. }
  446. $size = int($size);
  447. if ($DEBUG) {
  448. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  449. }
  450. # Save the extracted data
  451. my %tor = (
  452. 'title' => $title,
  453. 'season' => $fileSeason,
  454. 'episode' => $episode,
  455. 'seeds' => $seeds,
  456. 'leaches' => $leaches,
  457. 'size' => $size,
  458. 'url' => $url,
  459. 'source' => 'TPB'
  460. );
  461. push(@tors, \%tor);
  462. }
  463. } elsif ($content =~ /\<title\>[^\<\>]*\bisoHunt\b[^\<\>]*\<\/title\>/i) {
  464. # Find each TR element from ISOHunt
  465. my @trs = splitTags($content, 'TR', 'TH', undef());
  466. foreach my $tr (@trs) {
  467. # Split each TD element from the row
  468. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  469. if (scalar(@tds) > 10 || scalar(@tds) < 9) {
  470. if ($tds[1] =~ /No results found/) {
  471. next;
  472. }
  473. print STDERR 'Skipping invalid ISO TR: ' . $tr . "\n";
  474. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  475. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  476. }
  477. next;
  478. }
  479. # Skip sponsored results
  480. if ($tds[1] =~ /\bSponsored\b/i) {
  481. if ($DEBUG > 1) {
  482. print STDERR "Skipping sponsored result\n";
  483. }
  484. next;
  485. }
  486. # Find the torrent ID and show title
  487. my ($id, $title) = $tds[2] =~ /\<a(?:\s+[^\>]*)?href=\"\/torrent_details\/(\d+\/[^\"]+)\"\>\<span\>([^\>]+)\<\/span\>/i;
  488. if (!defined($id) || length($id) < 1 || !defined($title) || length($title) < 1) {
  489. if ($DEBUG > 1) {
  490. print STDERR "Unable to find show ID or title in TD:\n\t" . $tds[2] . "\n";
  491. }
  492. next;
  493. }
  494. $title =~ s/^\s+//;
  495. # Extract the season and episode numbers
  496. my ($fileSeason, $episode) = findSE($title);
  497. # Extract the size
  498. my $size = 0;
  499. my $unit = 'M';
  500. if ($tds[6] =~ m/(\d+(?:\.\d+)?) (G|M)B<\/td\>/i) {
  501. $size = $1;
  502. $unit = $2;
  503. }
  504. if ($unit eq 'G') {
  505. $size *= 1024;
  506. }
  507. $size = int($size);
  508. # Count the sum of seeders and leachers
  509. my $seeds = 0;
  510. if ($tds[7] =~ /(\d+)\<\/td\>/i) {
  511. $seeds = $1;
  512. }
  513. my $leaches = 0;
  514. if ($tds[8] =~ /(\d+)\<\/td\>/i) {
  515. $leaches = $1;
  516. }
  517. # Build the detail page URL
  518. my $url = $SOURCES->{'ISO'}->{'protocol'} . '://' . $SOURCES->{'ISO'}->{'host'} . '/torrent_details/' . $id;
  519. if ($DEBUG) {
  520. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  521. }
  522. # Save the extracted data
  523. my %tor = (
  524. 'title' => $title,
  525. 'season' => $fileSeason,
  526. 'episode' => $episode,
  527. 'seeds' => $seeds,
  528. 'leaches' => $leaches,
  529. 'size' => $size,
  530. 'url' => $url,
  531. 'source' => 'ISO'
  532. );
  533. push(@tors, \%tor);
  534. }
  535. } elsif ($content =~ /Kickass\s*Torrents\<\/title\>/i) {
  536. # Find each TR element from Kickass
  537. my @trs = splitTags($content, 'TR', 'TH', undef());
  538. foreach my $tr (@trs) {
  539. # Split each TD element from the row
  540. my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
  541. if (scalar(@tds) != 6) {
  542. print STDERR "Invalid KICK TR:\n";
  543. for (my $i = 0 ; $i < scalar(@tds) ; $i++) {
  544. print STDERR 'TD' . $i . ': ' . $tds[$i] . "\n";
  545. }
  546. next;
  547. }
  548. # Skip empty results
  549. if ( $tds[1] =~ /\<h2\>Nothing found\!\<\/h2\>/i
  550. || $tds[1] =~ /\<p\>\<strong\>Page not found\<\/strong\>\<\/p\>/i)
  551. {
  552. if ($DEBUG) {
  553. print STDERR "Skipping empty TD\n";
  554. }
  555. next;
  556. }
  557. # Find the torrent title
  558. my ($title) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href\=\"[^\"]+\?title\=\[[^\]\"]+\]([^\"]+)\"/i;
  559. if (!defined($title) || length($title) < 1) {
  560. if ($DEBUG) {
  561. print STDERR "Unable to find torrent title in TD:\n\t" . $tds[1] . "\n";
  562. }
  563. next;
  564. }
  565. $title =~ s/^\s+//;
  566. # Find the torrent URL
  567. my ($url) = $tds[1] =~ /\<a(?:\s+[^\>]*)?href=\"(magnet:[^\"]+)\"/i;
  568. if (!defined($url) || length($url) < 1) {
  569. if ($DEBUG) {
  570. print STDERR "Unable to find torrent URL in TD:\n\t" . $tds[1] . "\n";
  571. }
  572. next;
  573. }
  574. $url = decode_entities($url);
  575. # Extract the season and episode numbers
  576. my ($fileSeason, $episode) = findSE($title);
  577. # Extract the size
  578. my $size = 0;
  579. my $unit = 'M';
  580. if ($tds[2] =~ m/(\d+(?:\.\d+)?)\s+\<span\>(G|M)B\<\/span\>\<\/td\>/i) {
  581. $size = $1;
  582. $unit = $2;
  583. }
  584. if ($unit eq 'G') {
  585. $size *= 1024;
  586. }
  587. $size = int($size);
  588. # Count the sum of seeders and leachers
  589. my $seeds = 0;
  590. if ($tds[4] =~ /(\d+)\<\/td\>/i) {
  591. $seeds = $1;
  592. }
  593. my $leaches = 0;
  594. if ($tds[5] =~ /(\d+)\<\/td\>/i) {
  595. $leaches = $1;
  596. }
  597. if ($DEBUG) {
  598. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  599. }
  600. # Save the extracted data
  601. my %tor = (
  602. 'title' => $title,
  603. 'season' => $fileSeason,
  604. 'episode' => $episode,
  605. 'seeds' => $seeds,
  606. 'leaches' => $leaches,
  607. 'size' => $size,
  608. 'url' => $url,
  609. 'source' => 'KICK'
  610. );
  611. push(@tors, \%tor);
  612. }
  613. } elsif ($content =~ /\<a(?:\s+[^\>]*)?\>Torrentz\<\/a\>/i) {
  614. # We'll need to strip HTML
  615. my $hs = HTML::Strip->new();
  616. # Find each DL element from Torrentz
  617. my @dls = splitTags($content, 'DL', undef(), undef());
  618. foreach my $dl (@dls) {
  619. # Skip ads
  620. if ($dl =~ /\s+rel\=\"nofollow/i) {
  621. next;
  622. }
  623. # Find the torrent title
  624. my ($hash, $title) = $dl =~ /\<a href\=\"\/(\w+)\">(.*)\<\/a\>/i;
  625. if (!defined($hash) || length($hash) < 1 || !defined($title) || length($title) < 1) {
  626. if ($DEBUG) {
  627. print STDERR "Unable to find torrent title in DL:\n\t" . $dl . "\n";
  628. }
  629. next;
  630. }
  631. $hash = lc($hash);
  632. $title = $hs->parse($title);
  633. $hs->eof;
  634. $title =~ s/^\s+//;
  635. # Extract the season and episode numbers
  636. my ($fileSeason, $episode) = findSE($title);
  637. # Extract the size
  638. my $size = 0;
  639. if ($dl =~ /\<span class\=\"s\">(\d+(?:\.\d+)?) (G|M)B\<\/span\>/i) {
  640. $size = $1;
  641. if ($2 eq 'G') {
  642. $size *= 1024;
  643. }
  644. $size = int($size);
  645. }
  646. # Count the sum of seeders and leachers
  647. my $seeds = 0;
  648. if ($dl =~ /<span class\=\"u\">(\d+)\<\/span\>/i) {
  649. $seeds = $1;
  650. }
  651. my $leaches = 0;
  652. if ($dl =~ /<span class\=\"d\">(\d+)\<\/span\>/i) {
  653. $leaches = $1;
  654. }
  655. # Construct a magnet URL from the hash
  656. # Assume the seconary lookup will append a list of trackers
  657. my $url = 'magnet:?xt=urn:btih:' . $hash . '&dn=' . uri_encode($title, { 'encode_reserved' => 1 });
  658. if ($DEBUG) {
  659. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  660. }
  661. # Save the extracted data
  662. my %tor = (
  663. 'title' => $title,
  664. 'season' => $fileSeason,
  665. 'episode' => $episode,
  666. 'seeds' => $seeds,
  667. 'leaches' => $leaches,
  668. 'size' => $size,
  669. 'url' => $url,
  670. 'source' => 'Z'
  671. );
  672. push(@tors, \%tor);
  673. }
  674. } elsif ($content =~ /ExtraTorrent\.cc The World\'s Largest BitTorrent System\<\/title\>/i) {
  675. # Find TR elements from ET
  676. my @trs = splitTags($content, 'TR', undef(), 'magnet\:');
  677. foreach my $tr (@trs) {
  678. # Find the show title
  679. my ($title) = $tr =~ /title\=\"Download\s+([^\"]*)\s+torrent\"/i;
  680. if (!defined($title) || length($title) < 1) {
  681. if ($DEBUG) {
  682. print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
  683. }
  684. next;
  685. }
  686. $title =~ s/^\s+//;
  687. # Find the magnet URL
  688. my ($url) = $tr =~ /\<a\s+href=\"(magnet\:[^\"]+)\"/i;
  689. if (!defined($url) || length($url) < 1) {
  690. if ($DEBUG) {
  691. print STDERR "Skipping TR with no magnet URL\n";
  692. }
  693. next;
  694. }
  695. $url = decode_entities($url);
  696. # Extract the season and episode numbers
  697. my ($fileSeason, $episode) = findSE($title);
  698. # Count the sum of seeders and leachers
  699. my $seeds = 0;
  700. my $leaches = 0;
  701. if ($tr =~ /\<td\s+class=\"sy\">(\d+)\<\/td\>/) {
  702. $seeds = $1;
  703. }
  704. if ($tr =~ /\<td\s+class=\"ly\">(\d+)\<\/td\>/) {
  705. $leaches = $1;
  706. }
  707. # Extract the size (from separate column or inline)
  708. my $size = 0;
  709. my $unit = 'M';
  710. if ($tr =~ /\<td\>(\d+(?:\.\d+)?)\&nbsp\;(G|M)B\<\/td\>/) {
  711. $size = $1;
  712. $unit = $2;
  713. }
  714. if ($unit eq 'G') {
  715. $size *= 1024;
  716. }
  717. $size = int($size);
  718. if ($DEBUG) {
  719. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  720. }
  721. # Save the extracted data
  722. my %tor = (
  723. 'title' => $title,
  724. 'season' => $fileSeason,
  725. 'episode' => $episode,
  726. 'seeds' => $seeds,
  727. 'leaches' => $leaches,
  728. 'size' => $size,
  729. 'url' => $url,
  730. 'source' => 'ET'
  731. );
  732. push(@tors, \%tor);
  733. }
  734. } elsif ($content =~ /\<title\>Sorry/) {
  735. if ($DEBUG) {
  736. print STDERR "ISO offline\n";
  737. }
  738. } elsif ($content =~ /Database\s+maintenance/i) {
  739. if ($DEBUG) {
  740. print STDERR "TPB offline\n";
  741. }
  742. } elsif ($content =~ /^\s*$/) {
  743. if ($DEBUG) {
  744. print STDERR "Source returned no content\n";
  745. }
  746. } elsif ($content =~ /limit\s*reached/i) {
  747. if ($DEBUG) {
  748. print STDERR "Source request limit reached\n";
  749. }
  750. } elsif ($content =~ /Not\s+Found\<\/title\>/i) {
  751. if ($DEBUG) {
  752. print STDERR "No results\n";
  753. }
  754. } else {
  755. print STDERR "Unknown HTML content:\n" . $content . "\n\n";
  756. }
  757. }
  758. # Handle JSON content
  759. foreach my $content (@json_content) {
  760. if (ref($content) eq 'ARRAY' && scalar(@{$content}) == 0) {
  761. if ($DEBUG > 1) {
  762. print STDERR "Empty JSON array\n";
  763. }
  764. } elsif (exists($content->{'channel'})
  765. && ref($content->{'channel'}) eq 'HASH'
  766. && exists($content->{'channel'}->{'title'})
  767. && $content->{'channel'}->{'title'} =~ /nzbplanet/i)
  768. {
  769. print STDERR "Index: NZBplanet\n";
  770. } elsif (exists($content->{'channel'})
  771. && ref($content->{'channel'}) eq 'HASH'
  772. && exists($content->{'channel'}->{'title'})
  773. && $content->{'channel'}->{'title'} =~ /usenet\-crawler/i)
  774. {
  775. print STDERR "Index: usenet-crawler\n";
  776. } elsif (ref($content) eq 'ARRAY'
  777. && scalar(@{$content}) > 0
  778. && ref($content->[0]) eq 'HASH'
  779. && exists($content->[0]->{'guid'}))
  780. {
  781. print STDERR "Index: NZB.is\n";
  782. } elsif (exists($content->{'title'}) && $content->{'title'} =~ /NZBCat/i) {
  783. my $list = $content->{'item'};
  784. if (!$list || ref($list) ne 'ARRAY') {
  785. if ($DEBUG) {
  786. warn("Empty/invalid NCAT list\n");
  787. }
  788. next;
  789. }
  790. foreach my $item (@{$list}) {
  791. my $tor = parseNZB($item);
  792. if ($tor) {
  793. $tor->{'source'} = 'NCAT';
  794. push(@tors, $tor);
  795. }
  796. }
  797. } elsif (ref($content) eq 'HASH'
  798. && exists($content->{'channel'})
  799. && ref($content->{'channel'}) eq 'HASH'
  800. && exists($content->{'channel'}->{'title'})
  801. && $content->{'channel'}->{'title'} =~ /\bNzb\s+Planet\b/i)
  802. {
  803. if (!exists($content->{'channel'}->{'item'}) || ref($content->{'channel'}->{'item'}) ne 'ARRAY') {
  804. if ($DEBUG) {
  805. print STDERR "Empty/invalid NZB Planet result\n";
  806. }
  807. next;
  808. }
  809. foreach my $item (@{ $content->{'channel'}->{'item'} }) {
  810. my $tor = parseNZB($item);
  811. if ($tor) {
  812. $tor->{'source'} = 'PLANET';
  813. push(@tors, $tor);
  814. }
  815. }
  816. } else {
  817. print STDERR "Unknown JSON content:\n" . $content . "\n\n";
  818. }
  819. }
  820. # Filter for size/count/etc.
  821. my %tors = ();
  822. my $showRegex = undef();
  823. {
  824. my $showClean = $SERIES->{'name'};
  825. $showClean =~ s/[\"\']//g;
  826. $showClean =~ s/[\W_]+/\[\\W_\].*/g;
  827. $showRegex = qr/^${showClean}[\W_]/i;
  828. }
  829. foreach my $tor (@tors) {
  830. # Extract the BTIH/GUID, if available
  831. getHash($tor);
  832. # Skip bad TORs based on name/etc.
  833. if (badTor($tor)) {
  834. next;
  835. }
  836. # Skip files that contain a word from the series exclude list
  837. my $exclude = undef();
  838. foreach my $ex (keys(%{ $SERIES->{'exclude_hash'} })) {
  839. $ex = quotemeta($ex);
  840. if ($tor->{'title'} =~ m/${ex}/i) {
  841. $exclude = $ex;
  842. last;
  843. }
  844. }
  845. if (defined($exclude)) {
  846. if ($DEBUG) {
  847. print STDERR 'Skipping file: Title contains "' . $exclude . '": ' . $tor->{'title'} . "\n";
  848. }
  849. next;
  850. }
  851. # Enforce season and episode number matches for standard searches, or CUSTOM_SEARCH matching (if it's a regex)
  852. if (!$CUSTOM_SEARCH) {
  853. # Skip files that don't contain the right season number
  854. if (!defined($tor->{'season'}) || $tor->{'season'} != $SERIES->{'season'}) {
  855. if ($DEBUG) {
  856. print STDERR 'Skipping file: No match for season number (' . $SERIES->{'season'} . '): ' . $tor->{'title'} . "\n";
  857. }
  858. next;
  859. # Skip files that don't contain a needed episode number, unless there is no episode number and NO_QUALITY_CHECKS is set
  860. } elsif ((defined($tor->{'episode'}) && !$SERIES->{'need_hash'}->{ $tor->{'episode'} })
  861. || (!defined($tor->{'episode'}) && !$CONFIG{'NO_QUALITY_CHECKS'}))
  862. {
  863. if ($DEBUG) {
  864. print STDERR 'Skipping file: No match for episode number (' . $tor->{'episode'} . '): ' . $tor->{'title'} . "\n";
  865. }
  866. next;
  867. }
  868. } elsif (ref($CUSTOM_SEARCH) eq 'Regexp') {
  869. # Skip files that don't match the regex
  870. if (!($tor->{'title'} =~ $CUSTOM_SEARCH)) {
  871. if ($DEBUG) {
  872. print STDERR 'Skipping file: No match for CUSTOM_SEARCH regex (' . $CUSTOM_SEARCH . '): ' . $tor->{'title'} . "\n";
  873. }
  874. next;
  875. }
  876. }
  877. # Ensure the seed/leach count is usable
  878. seedCleanup($tor);
  879. # Only apply the quality rules if NO_QUALITY_CHECKS is not in effect
  880. if (!$CONFIG{'NO_QUALITY_CHECKS'} && lowQualityTor($tor)) {
  881. next;
  882. }
  883. # Save good torrents
  884. push(@{ $tors{ $tor->{'episode'} } }, $tor);
  885. if ($DEBUG) {
  886. print STDERR 'Possible URL ('
  887. . $tor->{'seeds'} . '/'
  888. . $tor->{'leaches'}
  889. . ' seeds/leaches, '
  890. . $tor->{'size'}
  891. . ' MiB): '
  892. . $tor->{'title'} . "\n";
  893. }
  894. }
  895. # Find the average torrent size for each episode
  896. my %size = ();
  897. {
  898. my %max = ();
  899. my %avg = ();
  900. foreach my $episode (keys(%tors)) {
  901. my $count = 0;
  902. $avg{$episode} = 0.001;
  903. $max{$episode} = 0.001;
  904. foreach my $tor (@{ $tors{$episode} }) {
  905. $count++;
  906. $avg{$episode} += $tor->{'size'};
  907. if ($tor->{'size'} > $max{$episode}) {
  908. $max{$episode} = $tor->{'size'};
  909. }
  910. }
  911. if ($count > 0) {
  912. $avg{$episode} /= $count;
  913. }
  914. if (!defined($max{$episode})) {
  915. warn("Invalid max episode\n");
  916. }
  917. if (!defined($avg{$episode})) {
  918. warn("Invalid avg episode\n");
  919. }
  920. $size{$episode} = ($max{$episode} + $avg{$episode}) / 2;
  921. if ($DEBUG) {
  922. print STDERR 'Episode '
  923. . $episode
  924. . ' max/avg/cmp size: '
  925. . int($max{$episode}) . '/'
  926. . int($avg{$episode}) . '/'
  927. . int($size{$episode})
  928. . " MiB\n";
  929. }
  930. }
  931. }
  932. # Calculate an adjusted count based on the peer count, relative file size, and title contents
  933. foreach my $episode (keys(%tors)) {
  934. foreach my $tor (@{ $tors{$episode} }) {
  935. my $count = $tor->{'seeds'} + $tor->{'leaches'};
  936. # Start with the peer count
  937. $tor->{'adj_count'} = $count;
  938. # Adjust based on file size
  939. {
  940. my $size_ratio = $tor->{'size'} / $size{$episode};
  941. if ($tor->{'size'} >= $size{$episode}) {
  942. $tor->{'adj_count'} *= $CONFIG{'SIZE_BONUS'} * $size_ratio;
  943. } else {
  944. $tor->{'adj_count'} *= (1 / $CONFIG{'SIZE_PENALTY'}) * $size_ratio;
  945. }
  946. }
  947. # Adjust based on title contents
  948. if ($tor->{'title'} =~ /Subtitulado/i) {
  949. $tor->{'adj_count'} *= 1 / $CONFIG{'TITLE_PENALTY'};
  950. }
  951. # Truncate to an integer
  952. $tor->{'adj_count'} = int($tor->{'adj_count'});
  953. if ($DEBUG) {
  954. print STDERR 'Possible URL (' . $tor->{'adj_count'} . ' size-adjusted sources): ' . $tor->{'url'} . "\n";
  955. }
  956. }
  957. }
  958. # Pick the best-adjusted-count torrent for each episode
  959. my %urls = ();
  960. foreach my $episode (keys(%tors)) {
  961. my @sorted = sort { $b->{'adj_count'} <=> $a->{'adj_count'} } @{ $tors{$episode} };
  962. my $max = undef();
  963. foreach my $tor (@sorted) {
  964. if (!defined($max) || $tor->{'adj_count'} > $max) {
  965. if ($urls{$episode} = resolveSecondary($tor)) {
  966. if (!$tor->{'hash'} || $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  967. print STDERR 'Double-skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  968. next;
  969. }
  970. $max = $tor->{'adj_count'};
  971. }
  972. if ($DEBUG) {
  973. print STDERR 'Semi-final URL (adjusted count: ' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  974. }
  975. } elsif ($DEBUG) {
  976. print STDERR 'Skipping for lesser adjusted count (' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
  977. }
  978. }
  979. }
  980. # Output
  981. foreach my $episode (keys(%tors)) {
  982. if (defined($urls{$episode})) {
  983. $urls{$episode} = resolveTrackers($urls{$episode});
  984. print $urls{$episode} . "\n";
  985. if ($CONFIG{'SYSLOG'}) {
  986. syslog(LOG_NOTICE, $SERIES->{'name'} . ': ' . getHash($urls{$episode}));
  987. }
  988. if ($DEBUG) {
  989. print STDERR 'Final URL: ' . $urls{$episode} . "\n";
  990. }
  991. } elsif ($DEBUG) {
  992. print STDERR 'No URL for: ' . $episode . "\n";
  993. }
  994. }
  995. # Cleanup
  996. if ($COOKIES) {
  997. unlink($COOKIES);
  998. }
  999. if ($SCRIPT) {
  1000. unlink($SCRIPT);
  1001. }
  1002. if ($CONFIG{'SYSLOG'}) {
  1003. closelog();
  1004. }
  1005. exit(0);
  1006. sub confDefault($;$) {
  1007. my ($name, $default) = @_;
  1008. if (!$name) {
  1009. return;
  1010. }
  1011. # Enviornment, config file, default, undef()
  1012. if (defined($ENV{$name})) {
  1013. $CONFIG{$name} = $ENV{$name};
  1014. } elsif (!defined($CONFIG{$name})) {
  1015. if ($default) {
  1016. $CONFIG{$name} = $default;
  1017. } else {
  1018. $CONFIG{$name} = undef();
  1019. }
  1020. }
  1021. }
  1022. sub readGlobalExcludes($) {
  1023. my ($file) = @_;
  1024. my %excludes = ();
  1025. if (!$file || !-r $file) {
  1026. warn('No excludes files available' . $file . "\n");
  1027. return \%excludes;
  1028. }
  1029. my $fh;
  1030. open($fh, '<', $file)
  1031. or die('Unable to open excludes file: ' . $file . ': ' . $! . "\n");
  1032. while (<$fh>) {
  1033. # Skip blank lines and comments
  1034. if (/^\s*$/ || /^\s*#/) {
  1035. next;
  1036. }
  1037. # Assume everything else is one BT hash/NZB GUID per line
  1038. if (/^\s*(\w+)\s*$/) {
  1039. my $hash = lc($1);
  1040. if ($DEBUG > 1) {
  1041. print STDERR 'Excluding hash: ' . $hash . "\n";
  1042. }
  1043. $excludes{$hash} = 1;
  1044. } else {
  1045. die('Invalid exclude line: ' . $_ . "\n");
  1046. }
  1047. }
  1048. close($fh);
  1049. return \%excludes;
  1050. }
  1051. # Find an input directory and read the local config
  1052. sub readInputDir($) {
  1053. my ($path) = @_;
  1054. my %series = (
  1055. 'path' => undef(),
  1056. 'name' => '',
  1057. 'path_name' => '',
  1058. 'season' => 0,
  1059. 'episodes' => {},
  1060. 'exclude' => '',
  1061. 'exclude_hash' => {},
  1062. 'need' => (),
  1063. 'need_hash' => {},
  1064. 'season_done' => 0,
  1065. 'search_name' => 0,
  1066. 'no_quality_checks' => 0,
  1067. 'more_number_formats' => 0,
  1068. );
  1069. # Allow use of the raw series name
  1070. if (!($path =~ /\//)) {
  1071. $series{'name'} = $path;
  1072. $series{'path'} = $CONFIG{'TV_DIR'} . '/' . $path;
  1073. } else {
  1074. $series{'path'} = $path;
  1075. }
  1076. # Allow use of relative paths
  1077. $series{'path'} = File::Spec->rel2abs($series{'path'});
  1078. # Isolate the season from the path, if provided
  1079. if ($series{'path'} =~ /\/Season\s+(\d+)\/?$/i) {
  1080. $series{'season'} = $1;
  1081. $series{'path'} = dirname($series{'path'});
  1082. }
  1083. # Sanity check
  1084. if ($DEBUG) {
  1085. print STDERR 'Checking directory: ' . $series{'path'} . "\n";
  1086. }
  1087. if (!-d $series{'path'}) {
  1088. die('Invalid input directory: ' . $series{'path'} . "\n");
  1089. }
  1090. # Allow the series name to be overriden
  1091. my $search_name = $series{'path'} . '/search_name';
  1092. if (-e $search_name) {
  1093. local ($/, *FH);
  1094. open(FH, $search_name)
  1095. or die('Unable to read search_name for series: ' . $series{'name'} . ': ' . $! . "\n");
  1096. my $text = <FH>;
  1097. close(FH);
  1098. if ($text =~ /^\s*(\S.*\S)\s*$/) {
  1099. $series{'name'} = $1;
  1100. } else {
  1101. warn('Skipping invalid search_name for series: ' . $series{'name'} . ': ' . $text . "\n");
  1102. }
  1103. }
  1104. if ($DEBUG) {
  1105. print STDERR 'Searching with series title: ' . $series{'name'} . "\n";
  1106. }
  1107. # Final series name
  1108. if (!$series{'name'}) {
  1109. $series{'name'} = basename($series{'path'});
  1110. }
  1111. $series{'name'} =~ s/[\'\"\.]//g;
  1112. $series{'path_name'} = $series{'name'};
  1113. # If no explicit season is provided find the latest
  1114. if (!$series{'season'}) {
  1115. opendir(SERIES, $series{'path'})
  1116. or die("Unable to open series directory: ${!}\n");
  1117. while (my $file = readdir(SERIES)) {
  1118. if ($file =~ /^Season\s+(\d+)$/i) {
  1119. if (!$series{'season'} || $series{'season'} < $1) {
  1120. $series{'season'} = $1;
  1121. }
  1122. }
  1123. }
  1124. closedir(SERIES);
  1125. }
  1126. # Validate the season number
  1127. if (!defined($series{'season'}) || $series{'season'} < 1 || $series{'season'} > 2000) {
  1128. die('Invalid season number: ' . $series{'name'} . ' => ' . $series{'season'} . "\n");
  1129. }
  1130. # Allow quality checks to be disabled
  1131. if (-e $series{'path'} . '/no_quality_checks') {
  1132. $series{'no_quality_checks'} = 1;
  1133. $CONFIG{'NO_QUALITY_CHECKS'} = 1;
  1134. if ($DEBUG) {
  1135. print STDERR 'Searching with no quality checks: ' . $series{'name'} . "\n";
  1136. }
  1137. }
  1138. # Allow use of more number formats
  1139. if (-e $series{'path'} . '/more_number_formats') {
  1140. $series{'more_number_formats'} = 1;
  1141. $CONFIG{'MORE_NUMBER_FORMATS'} = 1;
  1142. if ($DEBUG) {
  1143. print STDERR 'Searching with more number formats: ' . $series{'name'} . "\n";
  1144. }
  1145. }
  1146. # Read the search excludes file, if any
  1147. if (-e $series{'path'} . '/excludes') {
  1148. local $/ = undef;
  1149. open(EX, $series{'path'} . '/excludes')
  1150. or die("Unable to open series excludes file: ${!}\n");
  1151. my $ex = <EX>;
  1152. close(EX);
  1153. $ex =~ s/^\s+//;
  1154. $ex =~ s/\s+$//;
  1155. my @excludes = split(/\s*,\s*/, $ex);
  1156. foreach my $ex (@excludes) {
  1157. $series{'excludes_hash'}->{$ex} = 1;
  1158. if (length($series{'exclude'})) {
  1159. $series{'exclude'} .= ' ';
  1160. }
  1161. $series{'exclude'} .= '-"' . $ex . '"';
  1162. }
  1163. $series{'exclude'} = uri_encode(' ' . $series{'exclude'});
  1164. }
  1165. # Get the last episode number
  1166. opendir(SEASON, $series{'path'} . '/Season ' . $series{'season'})
  1167. or die("Unable to open season directory: ${!}\n");
  1168. while (my $file = readdir(SEASON)) {
  1169. # Skip support files
  1170. if ($file =~ /\.(?:png|xml|jpg|gif|tbn|txt|nfo|torrent)\s*$/i) {
  1171. next;
  1172. }
  1173. # Check for a season_done file
  1174. if ($file eq 'season_done') {
  1175. $series{'season_done'} = 1;
  1176. next;
  1177. }
  1178. # Extract the episode number
  1179. my ($num) = $file =~ /^\s*\.?(\d+)\s*\-\s*/i;
  1180. if (defined($num)) {
  1181. $num = int($num);
  1182. if ($DEBUG) {
  1183. print STDERR 'Found episode number: ' . $num . ' in file: ' . $file . "\n";
  1184. }
  1185. # Record it
  1186. $series{'episodes'}->{$num} = 1;
  1187. }
  1188. }
  1189. close(SEASON);
  1190. # Assume we need any missing episodes, and the next few (unless season_done is set)
  1191. my $highest = (sort { $b <=> $a } @{ $series{'need'} })[0];
  1192. for (my $i = 1 ; $i <= $highest ; $i++) {
  1193. if (!$series{'episodes'}->{$i}) {
  1194. push(@{ $series{'need'} }, $i);
  1195. }
  1196. }
  1197. if (!$series{'season_done'}) {
  1198. for (my $i = 1 ; $i <= $CONFIG{'NEXT_EPISODES'} ; $i++) {
  1199. push(@{ $series{'need'} }, $highest + $i);
  1200. }
  1201. }
  1202. if ($DEBUG) {
  1203. print STDERR 'Needed episodes: ' . join(', ', @{ $series{'need'} }) . "\n";
  1204. }
  1205. # Reverse the array for later matching
  1206. foreach my $episode (@{ $series{'need'} }) {
  1207. $series{'need_hash'}->{$episode} = 1;
  1208. }
  1209. # Log and return
  1210. if ($CONFIG{'SYSLOG'}) {
  1211. syslog(LOG_NOTICE, $series{'name'});
  1212. }
  1213. return \%series;
  1214. }
  1215. sub parseNZB($) {
  1216. my ($item) = @_;
  1217. if (!$item || ref($item) ne 'HASH' || !exists($item->{'guid'})) {
  1218. warn("Unable to parse NZB item\n");
  1219. return undef();
  1220. }
  1221. # Ensure the record is sensible
  1222. my $id = undef();
  1223. if (ref($item->{'guid'}) eq 'HASH' && exists($item->{'guid'}->{'text'})) {
  1224. $id = $item->{'guid'}->{'text'};
  1225. } elsif (!ref($item->{'guid'})) {
  1226. $id = $item->{'guid'};
  1227. }
  1228. if (!$id) {
  1229. warn("Unable to parse NZB item GUID\n");
  1230. return undef();
  1231. }
  1232. # NZB URL
  1233. my $url = undef();
  1234. if (exists($item->{'link'}) && $item->{'link'}) {
  1235. $url = $item->{'link'};
  1236. $url =~ s/^(https?)\:/${CONFIG{'PROTOCOL'}}\:/;
  1237. }
  1238. # Title
  1239. my $title = undef();
  1240. if (exists($item->{'title'}) && $item->{'title'}) {
  1241. $title = $item->{'title'};
  1242. } elsif (exists($item->{'description'}) && $item->{'description'}) {
  1243. $title = $item->{'description'};
  1244. }
  1245. if ($title) {
  1246. $url .= '#' . $title;
  1247. }
  1248. # Extract the season and episode numbers
  1249. my ($season, $episode) = findSE($title);
  1250. # Size
  1251. my $size = 0;
  1252. if (exists($item->{'newznab:attr'}) && $item->{'newznab:attr'} && ref($item->{'newznab:attr'}) eq 'ARRAY') {
  1253. foreach my $hash (@{ $item->{'newznab:attr'} }) {
  1254. if (ref($hash) eq 'HASH' && exists($hash->{'_name'}) && exists($hash->{'_value'}) && $hash->{'_name'} eq 'size') {
  1255. $size = int($hash->{'_value'}) / 1024;
  1256. last;
  1257. }
  1258. }
  1259. } elsif (exists($item->{'attr'}) && $item->{'attr'} && ref($item->{'attr'}) eq 'ARRAY') {
  1260. foreach my $hash (@{ $item->{'attr'} }) {
  1261. if (!exists($hash->{'@attributes'}) || ref($hash->{'@attributes'}) ne 'HASH') {
  1262. next;
  1263. }
  1264. $hash = $hash->{'@attributes'};
  1265. if (exists($hash->{'name'}) && $hash->{'name'} eq 'size' && exists($hash->{'value'})) {
  1266. $size = int($hash->{'value'}) / 1024;
  1267. last;
  1268. }
  1269. }
  1270. }
  1271. # Date
  1272. my $date = undef();
  1273. if (exists($item->{'pubDate'}) && $item->{'pubDate'}) {
  1274. $date = str2time($item->{'pubDate'});
  1275. }
  1276. # Sanity checks
  1277. if (!$url) {
  1278. warn('No URL in NZB item: ' . $id . "\n");
  1279. return undef();
  1280. }
  1281. if (!$title) {
  1282. warn('No title in NZB item: ' . $id . "\n");
  1283. return undef();
  1284. }
  1285. if (!$size) {
  1286. warn('No size in NZB item: ' . $id . "\n");
  1287. return undef();
  1288. }
  1289. if ($DEBUG) {
  1290. print STDERR 'Found file (' . $title . '): ' . $url . "\n";
  1291. }
  1292. # Save the extracted data
  1293. my %tor = (
  1294. 'title' => $title,
  1295. 'season' => $season,
  1296. 'episode' => $episode,
  1297. 'date' => $date,
  1298. 'size' => $size,
  1299. 'url' => $url,
  1300. );
  1301. return \%tor;
  1302. }
  1303. sub badTor($) {
  1304. my ($tor) = @_;
  1305. # Skip files that are in the torrent excludes list
  1306. if ($tor->{'hash'} && $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  1307. if ($DEBUG) {
  1308. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1309. }
  1310. return 1;
  1311. }
  1312. # Skip files that don't start with our show title
  1313. if (!($tor->{'title'} =~ $showRegex)) {
  1314. if ($DEBUG) {
  1315. print STDERR 'Skipping file: Title does not match (' . $showRegex . '): ' . $tor->{'title'} . "\n";
  1316. }
  1317. return 1;
  1318. # Skip leaked files
  1319. } elsif ($tor->{'title'} =~ /leaked/i) {
  1320. if ($DEBUG) {
  1321. print STDERR 'Skipping file: Title contains "leaked": ' . $tor->{'title'} . "\n";
  1322. }
  1323. return 1;
  1324. # Skip pre-air files
  1325. } elsif ($tor->{'title'} =~ /preair/i) {
  1326. if ($DEBUG) {
  1327. print STDERR 'Skipping file: Title contains "preair": ' . $tor->{'title'} . "\n";
  1328. }
  1329. return 1;
  1330. # Skip SWESUB files
  1331. } elsif ($tor->{'title'} =~ /swesub/i) {
  1332. if ($DEBUG) {
  1333. print STDERR 'Skipping file: Title contains "SWESUB": ' . $tor->{'title'} . "\n";
  1334. }
  1335. return 1;
  1336. # Skip .rus. files
  1337. } elsif ($tor->{'title'} =~ /\.rus\./i) {
  1338. if ($DEBUG) {
  1339. print STDERR 'Skipping file: Title contains ".rus.": ' . $tor->{'title'} . "\n";
  1340. }
  1341. return 1;
  1342. # Skip German files
  1343. } elsif ($tor->{'title'} =~ /german/i) {
  1344. if ($DEBUG) {
  1345. print STDERR 'Skipping file: Title contains "german": ' . $tor->{'title'} . "\n";
  1346. }
  1347. return 1;
  1348. # Skip French files
  1349. } elsif ($tor->{'title'} =~ /french/i) {
  1350. if ($DEBUG) {
  1351. print STDERR 'Skipping file: Title contains "french": ' . $tor->{'title'} . "\n";
  1352. }
  1353. return 1;
  1354. # Skip Spanish files
  1355. } elsif ($tor->{'title'} =~ /Español/i) {
  1356. if ($DEBUG) {
  1357. print STDERR 'Skipping file: Title contains "Español": ' . $tor->{'title'} . "\n";
  1358. }
  1359. return 1;
  1360. # Skip Castellano files
  1361. } elsif ($tor->{'title'} =~ /Castellano/i) {
  1362. if ($DEBUG) {
  1363. print STDERR 'Skipping file: Title contains "Castellano": ' . $tor->{'title'} . "\n";
  1364. }
  1365. return 1;
  1366. # Skip unedited files
  1367. } elsif ($tor->{'title'} =~ /unedited/i) {
  1368. if ($DEBUG) {
  1369. print STDERR 'Skipping file: Title contains "unedited": ' . $tor->{'title'} . "\n";
  1370. }
  1371. return 1;
  1372. }
  1373. # If we're still around the TOR is good
  1374. return 0;
  1375. }
  1376. sub seedCleanup($) {
  1377. my ($tor) = @_;
  1378. # Cap seeds/leaches/date and ensure they are always defined
  1379. if (!exists($tor->{'seeds'}) || !$tor->{'seeds'}) {
  1380. $tor->{'seeds'} = 0;
  1381. }
  1382. if ($tor->{'seeds'} > $CONFIG{'MAX_SEEDS'}) {
  1383. $tor->{'seeds'} = $CONFIG{'MAX_SEEDS'};
  1384. }
  1385. if (!exists($tor->{'leaches'}) || !$tor->{'leaches'}) {
  1386. $tor->{'leaches'} = 0;
  1387. }
  1388. if ($tor->{'leaches'} > $CONFIG{'MAX_SEEDS'}) {
  1389. $tor->{'leaches'} = $CONFIG{'MAX_SEEDS'};
  1390. }
  1391. # Proxy publication date to seeder/leacher quality
  1392. if ($tor->{'date'} && !$tor->{'seeds'}) {
  1393. my $age = time() - $tor->{'date'};
  1394. if ($age < 0) {
  1395. warn('Invalid date (' . $tor->{'date'} . '): ' . $tor->{'name'} . "\n");
  1396. $age = 0;
  1397. }
  1398. $age /= 86400;
  1399. if ($age > $CONFIG{'NZB_AGE_GOOD'}) {
  1400. $tor->{'seeds'} = $CONFIG{'MIN_COUNT'};
  1401. if ($age > $CONFIG{'NZB_AGE_MAX'}) {
  1402. $tor->{'seeds'} = ($CONFIG{'MIN_COUNT'} / 2) - 1;
  1403. }
  1404. } else {
  1405. my $penalty = ($CONFIG{'MAX_SEEDS'} / 2) / $CONFIG{'NZB_AGE_GOOD'};
  1406. $tor->{'seeds'} = $CONFIG{'MAX_SEEDS'} - ($age * $penalty);
  1407. }
  1408. $tor->{'seeds'} = int($tor->{'seeds'});
  1409. $tor->{'leaches'} = $tor->{'seeds'};
  1410. }
  1411. }
  1412. sub lowQualityTor($) {
  1413. my ($tor) = @_;
  1414. # Skip torrents with too few seeders/leachers
  1415. if (($tor->{'seeds'} + $tor->{'leaches'}) * $SOURCES->{ $tor->{'source'} }->{'weight'} < $CONFIG{'MIN_COUNT'}) {
  1416. if ($DEBUG) {
  1417. print STDERR 'Skipping file: Insufficient seeder/leacher count ('
  1418. . $tor->{'seeds'} . '/'
  1419. . $tor->{'leaches'} . '): '
  1420. . $tor->{'title'} . "\n";
  1421. }
  1422. return 1;
  1423. # Skip torrents with unusual seed/leach ratios
  1424. } elsif ($tor->{'seeds'} > 1
  1425. && $tor->{'seeds'} < $CONFIG{'SEED_RATIO_COUNT'}
  1426. && $tor->{'seeds'} > $tor->{'leaches'} * $CONFIG{'MAX_SEED_RATIO'})
  1427. {
  1428. if ($DEBUG) {
  1429. print STDERR 'Skipping file: Unusual seeder/leacher ratio ('
  1430. . $tor->{'seeds'} . '/'
  1431. . $tor->{'leaches'} . '): '
  1432. . $tor->{'title'} . "\n";
  1433. }
  1434. return 1;
  1435. # Skip torrents that are too small
  1436. } elsif ($tor->{'size'} < $CONFIG{'MIN_SIZE'}) {
  1437. if ($DEBUG) {
  1438. print STDERR 'Skipping file: Insufficient size (' . $tor->{'size'} . ' MiB): ' . $tor->{'title'} . "\n";
  1439. }
  1440. return 1;
  1441. }
  1442. # If we're still around the TOR is good
  1443. return 0;
  1444. }
  1445. sub getHash($) {
  1446. my ($tor) = @_;
  1447. my $url;
  1448. # Allow either a URL or a tor hashref
  1449. if (ref($tor) eq 'HASH') {
  1450. $url = $tor->{'url'};
  1451. } else {
  1452. $url = $tor;
  1453. undef($tor);
  1454. }
  1455. # Extract the BTIH hash or NZB ID, if available
  1456. my $hash = undef();
  1457. if ($url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1458. $hash = lc($1);
  1459. } elsif ($url =~ /\/getnzb\/(\w+)\.nzb\&/i) {
  1460. $hash = lc($1);
  1461. }
  1462. # Save back to the tor, if provided
  1463. if (defined($tor) && defined($hash)) {
  1464. $tor->{'hash'} = lc($hash);
  1465. }
  1466. # Always return the hash, or undef() if none can be found
  1467. return $hash;
  1468. }
  1469. # Resolve secondary URLs
  1470. sub resolveSecondary($) {
  1471. my ($tor) = @_;
  1472. # Fetch the torrent-specific page and extract the magent link
  1473. if ($tor->{'source'} eq 'ISO') {
  1474. my ($content, $code) = fetch($tor->{'url'}, 'secondary.html');
  1475. if ($code != 200) {
  1476. print STDERR 'Error fetching secondary URL: ' . $tor->{'url'} . "\n";
  1477. goto OUT;
  1478. }
  1479. my ($magnet) = $content =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
  1480. if (!defined($magnet) || length($magnet) < 1) {
  1481. if ($DEBUG) {
  1482. print STDERR 'No secondary URL available from: ' . $tor->{'url'} . "\n";
  1483. }
  1484. goto OUT;
  1485. }
  1486. $tor->{'url'} = decode_entities($magnet);
  1487. }
  1488. # Extract the hash
  1489. getHash($tor);
  1490. if (!$tor->{'hash'} || $CONFIG{'EXCLUDES'}->{ $tor->{'hash'} }) {
  1491. if ($DEBUG) {
  1492. print STDERR 'Skipping file: Excluded hash (' . $tor->{'hash'} . '): ' . $tor->{'title'} . "\n";
  1493. }
  1494. goto OUT;
  1495. }
  1496. OUT:
  1497. return $tor->{'url'};
  1498. }
  1499. sub resolveTrackers($) {
  1500. my ($url) = @_;
  1501. # Always append a few static trackers to the URI
  1502. if ($url =~ /^magnet\:/i) {
  1503. foreach my $tracker (@{ $CONFIG{'TRACKERS'} }) {
  1504. my $arg = '&tr=' . uri_encode($tracker, { 'encode_reserved' => 1 });
  1505. my $match = quotemeta($arg);
  1506. if (!($url =~ /${match}/)) {
  1507. $url .= $arg;
  1508. }
  1509. }
  1510. }
  1511. # Fetch a dynamic tracker list for any hashinfo from TorrentZ
  1512. if ($CONFIG{'TRACKER_LOOKUP'} && defined($SOURCES->{'Z'}) && $url =~ /^magnet\:/ && $url =~ /\bxt\=urn\:btih\:(\w+)/i) {
  1513. my $hash = lc($1);
  1514. my $baseURL = $SOURCES->{'Z'}->{'protocol'} . '://' . $SOURCES->{'Z'}->{'host'};
  1515. my $hashURL = $baseURL . '/' . $hash;
  1516. my ($content, $code) = fetch($hashURL, 'hashinfo.html');
  1517. if ($code != 200) {
  1518. print STDERR 'Error fetching hashinfo URL: ' . $hashURL . "\n";
  1519. return $url;
  1520. }
  1521. # Parse out and fetch the tracker list URL
  1522. my ($list) = $content =~ /\<a rel\=\"nofollow\" href\=\"(\/announcelist_\d+)\"\>/;
  1523. if (!defined($list)) {
  1524. if ($DEBUG) {
  1525. print STDERR 'Unable to find tracker list link at: ' . $hashURL . "\n";
  1526. }
  1527. return $url;
  1528. }
  1529. my $listURL = $baseURL . $list;
  1530. ($content, $code) = fetch($listURL, 'tracker.html');
  1531. if ($code != 200) {
  1532. print STDERR 'Error fetching tracker list URL: ' . $listURL . "\n";
  1533. return $url;
  1534. }
  1535. # Parse the tracker list and append our URL
  1536. foreach my $line ($content) {
  1537. if ($line =~ /^\s*$/) {
  1538. next;
  1539. }
  1540. $line =~ s/^\s+//;
  1541. $line =~ s/\s+$//;
  1542. if ($line =~ /^(?:http|udp)\:/i) {
  1543. my $arg = '&tr=' . uri_encode($line, { 'encode_reserved' => 1 });
  1544. my $match = quotemeta($arg);
  1545. if (!($url =~ /${match}/)) {
  1546. if ($DEBUG) {
  1547. print STDERR 'Adding tracker: ' . $line . "\n";
  1548. }
  1549. $url .= $arg;
  1550. }
  1551. } else {
  1552. print STDERR 'Unknown tracker type: ' . $line . "\n";
  1553. }
  1554. }
  1555. }
  1556. return $url;
  1557. }
  1558. sub splitTags($$$$) {
  1559. my ($content, $tag, $header, $match) = @_;
  1560. # Find each tag element
  1561. my @out = ();
  1562. my @parts = split(/\<${tag}(?:\s+[^\>]*)?\>/i, $content);
  1563. foreach my $part (@parts) {
  1564. # Trim trailing tags and skip things that aren't complete tags
  1565. if (!($part =~ s/\<\/${tag}\>.*$//is)) {
  1566. if ($DEBUG > 2) {
  1567. print STDERR 'Skipping non-tag line: ' . $part . "\n\n";
  1568. }
  1569. next;
  1570. }
  1571. # Skip headers
  1572. if (defined($header) && $part =~ /<${header}(?:\s+[^\>]*)?\>/i) {
  1573. if ($DEBUG > 2) {
  1574. print STDERR 'Skipping header line: ' . $part . "\n\n";
  1575. }
  1576. next;
  1577. }
  1578. # Skip non-matches
  1579. if (defined($match) && !($part =~ m/${match}/i)) {
  1580. if ($DEBUG > 2) {
  1581. print STDERR 'Skipping non-match line: ' . $part . "\n\n";
  1582. }
  1583. next;
  1584. }
  1585. # Save good tags
  1586. push(@out, $part);
  1587. }
  1588. return @out;
  1589. }
  1590. # Extract season
  1591. sub findSE($) {
  1592. my ($name) = @_;
  1593. my $season = 0;
  1594. my $episode = 0;
  1595. my $seasonBlock = '';
  1596. if ($name =~ /(?:\b|_)(20\d\d(?:\.|\-)[01]?\d(?:\.|\-)[0-3]?\d)(?:\b|_)/) {
  1597. $seasonBlock = $1;
  1598. my ($month, $day);
  1599. ($season, $month, $day) = $seasonBlock =~ /(20\d\d)(?:\.|\-)([01]?\d)(?:\.|\-)([0-3]?\d)/;
  1600. $episode = sprintf('%04d-%02d-%02d', $season, $month, $day);
  1601. } elsif ($name =~ /(?:\b|_)(S(?:eason)?[_\s\.\-]*\d{1,2}[_\s\.\-]*E(?:pisode)?[_\s\.]*\d{1,3}(?:E\d{1,3})?)(?:\b|_)/i) {
  1602. $seasonBlock = $1;
  1603. ($season, $episode) = $seasonBlock =~ /S(?:eason)?[_\s\.\-]*(\d{1,2})[_\s\.\-]*E(?:pisode)?[_\s\.\-]*(\d{1,3})/i;
  1604. } elsif ($name =~ /[\[\_\.](\d{1,2}x\d{2,3})[\]\_\.]/i) {
  1605. $seasonBlock = $1;
  1606. ($season, $episode) = $seasonBlock =~ /(\d+)x(\d+)/i;
  1607. } elsif ($name =~ /(?:\b|_)([01]?\d[_\s\.]?[0-3]\d)(?:\b|_)/i) {
  1608. $seasonBlock = $1;
  1609. ($season, $episode) = $seasonBlock =~ /(?:\b|_)([01]?\d)[_\s\.]?([0-3]\d)/i;
  1610. }
  1611. # Return something valid and INTy or UNDEF
  1612. if (!defined($seasonBlock) || $season < 1 || length($episode) < 1 || $episode eq '0') {
  1613. if ($DEBUG) {
  1614. print STDERR 'Could not find seasonBlock in: ' . $name . "\n";
  1615. }
  1616. $season = undef();
  1617. $episode = undef();
  1618. } elsif ($episode =~ /^20\d{2}\-\d{2}\-\d{2}$/) {
  1619. # Episode-by-date
  1620. $season = int($season);
  1621. } else {
  1622. # Catchall
  1623. # This will warn and return undef() (which is valid) on error
  1624. $season = int($season);
  1625. $episode = int($episode);
  1626. }
  1627. return ($season, $episode);
  1628. }
  1629. sub initSources() {
  1630. my %sources = ();
  1631. # NZBplanet.net
  1632. if ($CONFIG{'SOURCES'}->{'PLANET'}) {
  1633. my @proxies = ('api.nzbplanet.net/api');
  1634. my $source = findProxy(\@proxies, '\bNzbplanet\b');
  1635. if ($source && exists($CONFIG{'PLANET_APIKEY'}) && $CONFIG{'PLANET_APIKEY'}) {
  1636. $source->{'weight'} = 1.00;
  1637. $source->{'quote'} = 0;
  1638. $source->{'search_exclude'} = 0;
  1639. $source->{'search_suffix'} = '';
  1640. $source->{'custom_search'} = sub ($$$$) {
  1641. my ($urls, $series, $season, $episode) = @_;
  1642. my $url =
  1643. $source->{'search_url'}
  1644. . '?o=json&t=tvsearch&q='
  1645. . $series
  1646. . '&season='
  1647. . $season . '&ep='
  1648. . $episode
  1649. . '&apikey='
  1650. . $CONFIG{'PLANET_APIKEY'};
  1651. push(@{$urls}, $url);
  1652. };
  1653. $sources{'PLANET'} = $source;
  1654. }
  1655. }
  1656. # NZB.is
  1657. if ($CONFIG{'SOURCES'}->{'IS'}) {
  1658. my @proxies = ('nzb.is/api');
  1659. my $source = findProxy(\@proxies, '\bnzb\.is\b');
  1660. if ($source && exists($CONFIG{'IS_APIKEY'}) && $CONFIG{'IS_APIKEY'}) {
  1661. $source->{'weight'} = 1.00;
  1662. $source->{'quote'} = 0;
  1663. $source->{'search_exclude'} = 0;
  1664. $source->{'search_suffix'} = '';
  1665. $source->{'custom_search'} = sub ($$$$) {
  1666. my ($urls, $series, $season, $episode) = @_;
  1667. my $url =
  1668. $source->{'search_url'}
  1669. . '?o=json&t=tvsearch&q='
  1670. . $series
  1671. . '&season='
  1672. . $season . '&ep='
  1673. . $episode
  1674. . '&apikey='
  1675. . $CONFIG{'IS_APIKEY'};
  1676. push(@{$urls}, $url);
  1677. };
  1678. $sources{'IS'} = $source;
  1679. }
  1680. }
  1681. # Usenet Crawler
  1682. if ($CONFIG{'SOURCES'}->{'UC'}) {
  1683. my @proxies = ('www.usenet-crawler.com/api');
  1684. my $source = findProxy(\@proxies, '\busenet-crawler\b');
  1685. if ($source && exists($CONFIG{'UC_APIKEY'}) && $CONFIG{'UC_APIKEY'}) {
  1686. $source->{'weight'} = 1.00;
  1687. $source->{'quote'} = 0;
  1688. $source->{'search_exclude'} = 0;
  1689. $source->{'search_suffix'} = '';
  1690. $source->{'custom_search'} = sub ($$$$) {
  1691. my ($urls, $series, $season, $episode) = @_;
  1692. my $url =
  1693. $source->{'search_url'}
  1694. . '?o=json&t=tvsearch&q='
  1695. . $series
  1696. . '&season='
  1697. . $season . '&ep='
  1698. . $episode
  1699. . '&apikey='
  1700. . $CONFIG{'UC_APIKEY'};
  1701. push(@{$urls}, $url);
  1702. };
  1703. $sources{'UC'} = $source;
  1704. }
  1705. }
  1706. # NZB Cat
  1707. if ($CONFIG{'SOURCES'}->{'NCAT'}) {
  1708. my @proxies = ('nzb.cat/api');
  1709. my $source = findProxy(\@proxies, '\bforgottenpassword\b');
  1710. if ($source && exists($CONFIG{'NCAT_APIKEY'}) && $CONFIG{'NCAT_APIKEY'}) {
  1711. $source->{'weight'} = 1.00;
  1712. $source->{'quote'} = 0;
  1713. $source->{'search_exclude'} = 0;
  1714. $source->{'search_suffix'} = '';
  1715. $source->{'custom_search'} = sub ($$$$) {
  1716. my ($urls, $series, $season, $episode) = @_;
  1717. my $url =
  1718. $source->{'search_url'}
  1719. . '?o=json&t=tvsearch&q='
  1720. . $series
  1721. . '&season='
  1722. . $season . '&ep='
  1723. . $episode
  1724. . '&apikey='
  1725. . $CONFIG{'NCAT_APIKEY'};
  1726. push(@{$urls}, $url);
  1727. };
  1728. $sources{'NCAT'} = $source;
  1729. }
  1730. }
  1731. # The Pirate Bay
  1732. if ($CONFIG{'SOURCES'}->{'TPB'}) {
  1733. my @proxies = ('thepiratebay.org/search/', 'tpb.unblocked.co/search/');
  1734. my $source = findProxy(\@proxies, '\bPirate Search\b');
  1735. if ($source) {
  1736. $source->{'weight'} = 1.00;
  1737. $source->{'quote'} = 0;
  1738. $source->{'search_exclude'} = 1;
  1739. $source->{'search_suffix'} = '/0/7/0';
  1740. $sources{'TPB'} = $source;
  1741. }
  1742. }
  1743. # ISOhunt
  1744. if ($CONFIG{'SOURCES'}->{'ISO'}) {
  1745. my @proxies = ('isohunt.to/torrents/?ihq=', 'isohunters.net/torrents/?ihq=');
  1746. my $source = findProxy(\@proxies, 'Last\s+\d+\s+files\s+indexed');
  1747. if ($source) {
  1748. $source->{'weight'} = 0.75;
  1749. $source->{'quote'} = 1;
  1750. $source->{'search_exclude'} = 1;
  1751. $source->{'search_suffix'} = '';
  1752. $sources{'ISO'} = $source;
  1753. }
  1754. }
  1755. # Kickass
  1756. if ($CONFIG{'SOURCES'}->{'KICK'}) {
  1757. my @proxies = ('kickass.cd/search.php?q=', 'kickass.mx/search.php?q=');
  1758. my $source = findProxy(\@proxies, '/search.php');
  1759. if ($source) {
  1760. $source->{'weight'} = 1.00;
  1761. $source->{'quote'} = 1;
  1762. $source->{'search_exclude'} = 1;
  1763. $source->{'search_suffix'} = '/';
  1764. $sources{'KICK'} = $source;
  1765. }
  1766. }
  1767. # Torrentz
  1768. if ($CONFIG{'SOURCES'}->{'Z'}) {
  1769. my @proxies = ('torrentz.eu/search?q=', 'torrentz.me/search?q=', 'torrentz.ch/search?q=', 'torrentz.in/search?q=');
  1770. my $source = findProxy(\@proxies, 'Indexing [\d\,]+ active torrents');
  1771. if ($source) {
  1772. $source->{'weight'} = 1.00;
  1773. $source->{'quote'} = 1;
  1774. $source->{'search_exclude'} = 1;
  1775. $source->{'search_suffix'} = '';
  1776. if (!$CONFIG{'NO_QUALITY_CHECKS'}) {
  1777. $source->{'search_suffix'} = '+peer+%3E+' . $CONFIG{'MIN_COUNT'};
  1778. }
  1779. $sources{'Z'} = $source;
  1780. }
  1781. }
  1782. # ExtraTorrent
  1783. if ($CONFIG{'SOURCES'}->{'ET'}) {
  1784. my @proxies = (
  1785. 'extra.to/search/?search=', 'etmirror.com/search/?search=',
  1786. 'etproxy.com/search/?search=', 'extratorrentlive.com/search/?search='
  1787. );
  1788. my $source = findProxy(\@proxies, '/search/');
  1789. if ($source) {
  1790. $source->{'weight'} = 1.00;
  1791. $source->{'quote'} = 1;
  1792. $source->{'search_exclude'} = 0;
  1793. $source->{'search_suffix'} = '&new=1&x=0&y=0&srt=seeds&order=desc';
  1794. $source->{'search_url'} = 'phantomjs:' . $source->{'search_url'};
  1795. $sources{'ET'} = $source;
  1796. }
  1797. }
  1798. # Sanity check
  1799. if (scalar(keys(%sources)) < 1) {
  1800. die("No sources available\n");
  1801. }
  1802. return \%sources;
  1803. }
  1804. sub findProxy($$) {
  1805. my ($proxies, $match) = @_;
  1806. # Automatically select a proxy
  1807. my $host = '';
  1808. my $protocol = '';
  1809. my $search_url = '';
  1810. foreach my $url (@{$proxies}) {
  1811. my $path;
  1812. ($protocol, $host, $path) = $url =~ /^(?:(https?)\:\/\/)?([^\/]+)(\/.*)?$/i;
  1813. if (!$protocol) {
  1814. $protocol = $CONFIG{'PROTOCOL'};
  1815. }
  1816. if (!$host) {
  1817. print STDERR 'Could not parse proxy URL: ' . $url . "\n";
  1818. next;
  1819. }
  1820. my ($content, $code) = fetch($protocol . '://' . $host, undef(), 1);
  1821. if ($code == 200 && $content =~ m/${match}/i) {
  1822. $search_url = $protocol . '://' . $host . $path;
  1823. last;
  1824. } elsif ($DEBUG) {
  1825. print STDERR 'Proxy not available: ' . $host . "\n";
  1826. if ($DEBUG > 1) {
  1827. print STDERR $content . "\n";
  1828. }
  1829. }
  1830. }
  1831. # Return the search data if at least one proxy is up
  1832. if ($search_url) {
  1833. my %tmp = (
  1834. 'protocol' => $protocol,
  1835. 'host' => $host,
  1836. 'search_url' => $search_url,
  1837. );
  1838. return \%tmp;
  1839. }
  1840. # Return undef if we didn't find any working proxies
  1841. return undef();
  1842. }
  1843. sub fetch($;$$) {
  1844. my ($url, $file, $nocheck) = @_;
  1845. if ($DEBUG > 1) {
  1846. print STDERR 'Fetch: ' . $url . "\n";
  1847. }
  1848. # Allow usage with short names
  1849. if ($file && !($file =~ /^\//)) {
  1850. $file = '/tmp/findTorrent-' . $file;
  1851. }
  1852. # Random delay
  1853. if ($DEBUG > 1) {
  1854. sleep($CONFIG{'SLEEP'} * 0.25);
  1855. } else {
  1856. sleep($CONFIG{'DELAY'} * (rand($CONFIG{'SLEEP'}) + ($CONFIG{'SLEEP'} / 2)));
  1857. }
  1858. # Allow JS processing
  1859. if ($url =~ /^phantomjs\:\s*(\S.*)$/i) {
  1860. return phantomFetch($1, $file);
  1861. }
  1862. # Fetch object
  1863. if (!$FETCH) {
  1864. if (!$COOKIES) {
  1865. $COOKIES = mktemp('/tmp/findTorrent.cookies.XXXXXXXX');
  1866. }
  1867. $FETCH = Fetch->new(
  1868. 'cookiefile' => $COOKIES,
  1869. 'timeout' => $CONFIG{'TIMEOUT'},
  1870. 'uas' => $CONFIG{'UA'}
  1871. );
  1872. }
  1873. # Debug as requested
  1874. if ($DEBUG > 1 && $file) {
  1875. $FETCH->file($file);
  1876. } else {
  1877. $FETCH->file('');
  1878. }
  1879. # Standard fetch
  1880. $FETCH->url($url);
  1881. if ($nocheck) {
  1882. $FETCH->fetch('nocheck' => 1);
  1883. } else {
  1884. $FETCH->fetch();
  1885. }
  1886. return (scalar($FETCH->content()), $FETCH->status_code());
  1887. }
  1888. sub phantomFetch($$) {
  1889. my ($url, $file) = @_;
  1890. # Init
  1891. if (!$SCRIPT) {
  1892. $SCRIPT = mktemp('/tmp/findTorrent.phantom.XXXXXXXX');
  1893. open(my $fh, '>', $SCRIPT)
  1894. or die('Unable to open PhantomJS script file: ' . $! . "\n");
  1895. print $fh $CONFIG{'PHANTOM_CONFIG'};
  1896. close($fh);
  1897. }
  1898. # Capture STDOUT, drop STDERR
  1899. my @cmd = ($CONFIG{'PHANTOM_BIN'}, $SCRIPT, $url);
  1900. my ($content, undef(), undef()) = capture { system(@cmd); };
  1901. # Debug as requested
  1902. if ($DEBUG > 1 && $file) {
  1903. open(my $fh, '>', $file)
  1904. or warn('Unable to open phantom debug file: ' . $! . "\n");
  1905. if ($fh) {
  1906. print $fh $content;
  1907. close($fh);
  1908. }
  1909. }
  1910. # Try to get the response status code, fake one if we cannot
  1911. my ($code) = $content =~ /^(\d\d\d)$/m;
  1912. if (!defined($code)) {
  1913. warn("Unable to retrive HTTP status code. Faking success...\n");
  1914. $code = 200;
  1915. }
  1916. return ($content, $code);
  1917. }