Merge branch 'master' of github.com:profplump/UberZach-Video

Fetch.pm

@@ -655,6 +655,7 @@ sub save() {
 	if (!open(OUT, '>', $self->Fetch::file())) {
 		die('Fetch::save: Unable to open output file: ' . $self->Fetch::file() . ': ' . $! . "\n");
 	}
+	binmode OUT, ':encoding(UTF-8)';
 
 	# Print out the URL, the post content (if any)
 	if (defined($self->{'print_comments'}) && $self->{'print_comments'}) {

findTorrent.pl

@@ -16,7 +16,7 @@ my $TV_DIR = `~/bin/video/mediaPath` . '/TV';
 
 # Search parameters
 my $PROTOCOL = 'http';
-my %ENABLE_SOURCE = ('TPB' => 1, 'ISO' => 0);
+my %ENABLE_SOURCE = ('TPB' => 1, 'ISO' => 1);
 
 # Selection parameters
 my $MIN_COUNT        = 10;
@@ -42,6 +42,8 @@ use lib $Bin;
 use Fetch;
 
 # Prototypes
+sub resolveSecondary($);
+sub splitTRs($);
 sub findSE($);
 
 # Command line
@@ -85,10 +87,11 @@ if ($ENABLE_SOURCE{'TPB'}) {
 	# Available TPB proxies, in order of preference
 	my @TPBs = ('thepiratebay.se/search/', 'pirateproxy.se/search/', 'tpb.unblocked.co/search/');
 
-	# Automatically select a TPB proxy that returns a search page
+	# Automatically select a proxy that returns a search page
+	my $host       = '';
 	my $search_url = '';
 	foreach my $url (@TPBs) {
-		my ($host) = $url =~ /^([^\/]+)/;
+		($host) = $url =~ /^([^\/]+)/;
 		$fetch->url($PROTOCOL . '://' . $host);
 		if ($fetch->fetch('nocheck' => 1) == 200 && $fetch->content() =~ /\bSearch\b/i) {
 			$search_url = $url;
@@ -101,6 +104,7 @@ if ($ENABLE_SOURCE{'TPB'}) {
 	# Only add TPB if one of the proxies is up
 	if ($search_url) {
 		my %tmp = (
+			'host'          => $host,
 			'search_url'    => $search_url,
 			'search_suffix' => '/0/7/0',
 			'weight'        => 1.5,
@@ -114,12 +118,13 @@ if ($ENABLE_SOURCE{'TPB'}) {
 if ($ENABLE_SOURCE{'ISO'}) {
 
 	# Available ISOhunt proxies, in order of preference
-	my @ISOs = ('isohunt.com/js/json.php?rows=10&start=1&ihq=');
+	my @ISOs = ('isohunt.to/torrents/?ihq=', 'isohunters.net/torrents/?ihq=');
 
-	# Automatically select a TPB proxy that returns the non-US page
+	# Automatically select a proxy that returns the non-US page
+	my $host       = '';
 	my $search_url = '';
 	foreach my $url (@ISOs) {
-		my ($host) = $url =~ /^([^\/]+)/;
+		($host) = $url =~ /^([^\/]+)/;
 		$fetch->url($PROTOCOL . '://' . $host);
 		if ($fetch->fetch('nocheck' => 1) == 200 && $fetch->content() =~ /Last\s+\d+\s+files\s+indexed/i) {
 			$search_url = $url;
@@ -132,7 +137,8 @@ if ($ENABLE_SOURCE{'ISO'}) {
 	# Only add ISOhunt if one of the proxies is up
 	if ($search_url) {
 		my %tmp = (
-			'search_url'    => 'isohunt.com/js/json.php?rows=10&start=1&ihq=',
+			'host'          => $host,
+			'search_url'    => $search_url,
 			'search_suffix' => '',
 			'weight'        => 0.30,
 			'quote'         => 1
@@ -427,10 +433,12 @@ if (scalar(@urls) < 1) {
 	my $safeShow = $show;
 	$safeShow =~ s/\s+\&\s+/ and /i;
 	$safeShow =~ s/^\s*The\b//i;
-	$safeShow =~ s/\'//g;
+	$safeShow =~ s/\s+\-\s+/ /g;
+	$safeShow =~ s/[\'\:]//g;
 	$safeShow =~ s/[^\w\"\-]+/ /g;
 	$safeShow =~ s/^\s+//;
 	$safeShow =~ s/\s+$//;
+	$safeShow =~ s/\s\s+/ /g;
 	$safeShow =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
 	$safeShow =~ s/\%20/\+/g;
 
@@ -504,7 +512,7 @@ if (scalar(@urls) < 1) {
 	}
 }
 
-my $content      = '';
+my @html_content = ();
 my @json_content = ();
 foreach my $url (@urls) {
 
@@ -522,7 +530,7 @@ foreach my $url (@urls) {
 		print STDERR 'Error fetching URL: ' . $url . "\n";
 	}
 
-	# Save and compact the content
+	# Save the content, discriminating by data type
 	if ($DEBUG) {
 		print STDERR 'Fetched ' . length($fetch->content()) . " bytes\n";
 	}
@@ -534,139 +542,171 @@ foreach my $url (@urls) {
 			print STDERR 'JSON parsing failure on: ' . $fetch->content() . "\n";
 		}
 	} else {
-		$content .= ' ' . $fetch->content();
-		$content =~ s/\s+/ /g;
+		push(@html_content, scalar($fetch->content()));
 	}
 }
 
-# Find each TR element in TPB content
+# We need someplace to store parsed torrent data from the various sources
 my @tors = ();
-my @trs = split(/\<tr(?:\s+[^\>]*)?\>/i, $content);
-if ($DEBUG) {
-	print STDERR 'TR count: ' . scalar(@trs) . "\n";
-}
-foreach my $tr (@trs) {
 
-	# Trim trailing tags
-	if (!($tr =~ s/\<\/tr\>.*$//is)) {
+# Handle HTML content
+foreach my $content (@html_content) {
+	if ($content =~ /\<title\>The Pirate Bay/i) {
 
-		# Skip things that aren't compelete TRs
-		if ($DEBUG > 1) {
-			print STDERR 'Skipping non-TR line: ' . $tr . "\n\n";
-		}
-		next;
-	}
+		# Find TR elements from TPB
+		my @trs = splitTRs($content);
+		foreach my $tr (@trs) {
 
-	# Find the show title
-	my ($title) = $tr =~ /title\=\"Details\s+for\s+([^\"]*)\"/i;
-	if (!defined($title) || length($title) < 1) {
-		if ($DEBUG) {
-			print STDERR "Unable to find show title in TR\n";
-		}
-		next;
-	}
-	$title =~ s/^\s+//;
+			# Find the show title
+			my ($title) = $tr =~ /title\=\"Details\s+for\s+([^\"]*)\"/i;
+			if (!defined($title) || length($title) < 1) {
+				if ($DEBUG) {
+					print STDERR "Unable to find show title in TR\n\t" . $tr . "\n";
+				}
+				next;
+			}
+			$title =~ s/^\s+//;
 
-	# Extract the season and episode numbers
-	my ($fileSeason, $episode) = findSE($title);
+			# Extract the season and episode numbers
+			my ($fileSeason, $episode) = findSE($title);
 
-	# Extract the URL
-	my ($url) = $tr =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
-	if (!defined($url) || length($url) < 1) {
-		if ($DEBUG) {
-			print STDERR "Skipping TR with no magnet URL\n";
-		}
-		next;
-	}
+			# Extract the URL
+			my ($url) = $tr =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
+			if (!defined($url) || length($url) < 1) {
+				if ($DEBUG) {
+					print STDERR "Skipping TR with no magnet URL\n";
+				}
+				next;
+			}
 
-	# Count the sum of seeders and leachers
-	my $seeds   = 0;
-	my $leaches = 0;
-	if ($tr =~ /\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*$/i) {
-		$seeds   = $1;
-		$leaches = $2;
-	}
+			# Count the sum of seeders and leachers
+			my $seeds   = 0;
+			my $leaches = 0;
+			if ($tr =~ /\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*\<td(?:\s+[^\>]*)?\>(\d+)\<\/td\>\s*$/i) {
+				$seeds   = $1;
+				$leaches = $2;
+			}
 
-	# Extract the size (from separate column or inline)
-	my $size = 0;
-	my $unit = 'M';
-	if ($tr =~ m/Size (\d+(?:\.\d+)?)\&nbsp\;(G|M)iB/) {
-		$size = $1;
-		$unit = $2;
-	} elsif ($tr =~ m/(\d+(?:\.\d+)?)\&nbsp\;(G|M)iB\<\/[tT][dD]\>/) {
-		$size = $1;
-		$unit = $2;
-	}
-	if ($unit eq 'G') {
-		$size *= 1024;
-	}
-	$size = int($size);
+			# Extract the size (from separate column or inline)
+			my $size = 0;
+			my $unit = 'M';
+			if ($tr =~ m/Size (\d+(?:\.\d+)?)\&nbsp\;(G|M)iB/) {
+				$size = $1;
+				$unit = $2;
+			} elsif ($tr =~ m/(\d+(?:\.\d+)?)\&nbsp\;(G|M)iB\<\/[tT][dD]\>/) {
+				$size = $1;
+				$unit = $2;
+			}
+			if ($unit eq 'G') {
+				$size *= 1024;
+			}
+			$size = int($size);
 
-	if ($DEBUG) {
-		print STDERR 'Found file (' . $title . '): ' . $url . "\n";
-	}
-
-	# Save the extracted data
-	my %tor = (
-		'title'   => $title,
-		'season'  => $fileSeason,
-		'episode' => $episode,
-		'seeds'   => $seeds,
-		'leaches' => $leaches,
-		'size'    => $size,
-		'url'     => $url,
-		'source'  => 'TPB'
-	);
-	push(@tors, \%tor);
-}
+			if ($DEBUG) {
+				print STDERR 'Found file (' . $title . '): ' . $url . "\n";
+			}
 
-# Find each discrete torrent from ISOhunt
-foreach my $json (@json_content) {
-	foreach my $tor (@{$json}) {
+			# Save the extracted data
+			my %tor = (
+				'title'   => $title,
+				'season'  => $fileSeason,
+				'episode' => $episode,
+				'seeds'   => $seeds,
+				'leaches' => $leaches,
+				'size'    => $size,
+				'url'     => $url,
+				'source'  => 'TPB'
+			);
+			push(@tors, \%tor);
+		}
+
+	} elsif ($content =~ /Isohunt Torrent Search Engine\<\/title\>/i) {
+
+		# Find each TR element from ISOHunt
+		my @trs = splitTRs($content);
+		foreach my $tr (@trs) {
+
+			# Split each TD element from the row
+			my @tds = split(/\<td(?:\s+[^\>]*)?\>/i, $tr);
+			if (scalar(@tds) != 10) {
+				print STDERR 'Invalid ISO TR: ' . $tr . "\n";
+				next;
+			}
+			
+			# Skip sponsored results
+			if ($tds[1] =~ /Sponsored\<\/td\>/i) {
+				if ($DEBUG > 1) {
+					print STDERR "Skipping sponsored result\n";
+				}
+				next;
+			}
 
-		# Extract the title
-		my $title = $tor->{'title'};
-		$title =~ s/\<\/?b\>//gi;
-		$title =~ s/^\s+//;
+			# Find the torrent ID and show title
+			my ($id, $title) = $tds[2] =~ /\<a(?:\s+[^\>]*)?href=\"\/torrent_details\/(\d+)\/[^\"]*\"\>\<span\>([^\>]+)\<\/span\>/i;
+			if (!defined($id) || length($id) < 1 || !defined($title) || length($title) < 1) {
+				if ($DEBUG) {
+					print STDERR "Unable to find show ID or title in TD:\n\t" . $tds[2] . "\n";
+				}
+				next;
+			}
+			$title =~ s/^\s+//;
 
-		# Extract the season and episode numbers
-		my ($fileSeason, $episode) = findSE($tor->{'title'});
+			# Extract the season and episode numbers
+			my ($fileSeason, $episode) = findSE($title);
 
-		# Count the sum of seeders and leachers
-		my $seeds   = 0;
-		my $leaches = 0;
-		if (defined($tor->{'Seeds'}) && length($tor->{'Seeds'}) > 0) {
-			$seeds = $tor->{'Seeds'};
-		}
-		if (defined($tor->{'leechers'}) && length($tor->{'leechers'}) > 0) {
-			$leaches = $tor->{'leechers'};
-		}
+			# Extract the size
+			my $size = 0;
+			my $unit = 'M';
+			if ($tds[6] =~ m/(\d+(?:\.\d+)?) (G|M)B<\/td\>/i) {
+				$size = $1;
+				$unit = $2;
+			}
+			if ($unit eq 'G') {
+				$size *= 1024;
+			}
+			$size = int($size);
 
-		# Extract the size
-		my $size = 0;
-		if (defined($tor->{'length'}) && length($tor->{'length'}) > 0) {
-			$size = int($tor->{'length'} / 1048576);
-		}
+			# Count the sum of seeders and leachers
+			my $seeds = 0;
+			if ($tds[7] =~ /(\d+)\<\/td\>/i) {
+				$seeds = $1;
+			}
+			my $leaches = 0;
+			if ($tds[8] =~ /(\d+)\<\/td\>/i) {
+				$leaches = $1;
+			}
 
-		if ($DEBUG) {
-			print STDERR 'Found file (' . $title . '): ' . $tor->{'enclosure_url'} . "\n";
+			# Fetch the detail page for the URL
+			my $url = $PROTOCOL . '://' . $SOURCES{'ISO'}->{'host'} . '/torrent_details/' . $id . '/';
+
+			if ($DEBUG) {
+				print STDERR 'Found file (' . $title . '): ' . $url . "\n";
+			}
+
+			# Save the extracted data
+			my %tor = (
+				'title'   => $title,
+				'season'  => $fileSeason,
+				'episode' => $episode,
+				'seeds'   => $seeds,
+				'leaches' => $leaches,
+				'size'    => $size,
+				'url'     => $url,
+				'source'  => 'ISO'
+			);
+			push(@tors, \%tor);
 		}
 
-		# Save the extracted data
-		my %tmp = (
-			'title'   => $title,
-			'season'  => $fileSeason,
-			'episode' => $episode,
-			'seeds'   => $seeds,
-			'leaches' => $leaches,
-			'size'    => $size,
-			'url'     => $tor->{'enclosure_url'},
-			'source'  => 'ISO'
-		);
-		push(@tors, \%tmp);
+	} else {
+		print STDERR "Unknown HTML content:\n" . $content . "\n\n";
 	}
 }
 
+# Handle JSON content
+foreach my $content (@json_content) {
+	print STDERR "Unknown JSON content:\n" . $content . "\n\n";
+}
+
 # Filter for size/count/etc.
 my %tors      = ();
 my $showRegex = undef();
@@ -837,16 +877,18 @@ foreach my $episode (keys(%tors)) {
 # Pick the best-adjusted-count torrent for each episode
 my %urls = ();
 foreach my $episode (keys(%tors)) {
+	my @sorted = sort { $b->{'adj_count'} <=> $a->{'adj_count'} } @{ $tors{$episode} };
 	my $max = undef();
-	foreach my $tor (@{ $tors{$episode} }) {
+	foreach my $tor (@sorted) {
 		if (!defined($max) || $tor->{'adj_count'} > $max) {
-			$urls{$episode} = $tor->{'url'};
-			$max = $tor->{'adj_count'};
+			if ($urls{$episode} = resolveSecondary($tor)) {
+				$max = $tor->{'adj_count'};
+			}
 			if ($DEBUG) {
-				print STDERR 'New semi-final URL (adjusted count: ' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
+				print STDERR 'Semi-final URL (adjusted count: ' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
 			}
 		} elsif ($DEBUG) {
-			print STDERR 'Skipping for non-max adjusted count (' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
+			print STDERR 'Skipping for lesser adjusted count (' . $tor->{'adj_count'} . '): ' . $tor->{'url'} . "\n";
 		}
 	}
 }
@@ -867,6 +909,66 @@ foreach my $episode (keys(%tors)) {
 unlink($cookies);
 exit(0);
 
+# Resolve secondary URLs
+sub resolveSecondary($) {
+	my ($tor) = @_;
+	my $url = $tor->{'url'};
+
+	if ($tor->{'source'} eq 'ISO') {
+		if ($DEBUG) {
+			print STDERR 'Secondary fetch with URL: ' . $tor->{'url'} . "\n";
+			$fetch->file('/tmp/findTorrent-secondary.html');
+		}
+		sleep($DELAY * (rand(2) + 0.5));
+		$fetch->url($tor->{'url'});
+		$fetch->fetch();
+		if ($fetch->status_code() != 200) {
+			print STDERR 'Error fetching secondary URL: ' . $tor->{'url'} . "\n";
+			return undef();
+		}
+		($url) = $fetch->content() =~ /\<a\s+href\=\"(magnet\:\?[^\"]+)\"/i;
+		if (!defined($url) || length($url) < 1) {
+			if ($DEBUG) {
+				print STDERR 'No secondary URL available from: ' . $tor->{'url'} . "\n";
+			}
+			return undef();
+		}
+	}
+
+	return $url;
+}
+
+sub splitTRs($) {
+	my ($content) = @_;
+
+	# Find each TR element
+	my @trs = ();
+	my @rawTRs = split(/\<tr(?:\s+[^\>]*)?\>/i, $content);
+	foreach my $tr (@rawTRs) {
+
+		# Trim trailing tags and skip things that aren't complete TRs
+		if (!($tr =~ s/\<\/tr\>.*$//is)) {
+			if ($DEBUG > 1) {
+				print STDERR 'Skipping non-TR line: ' . $tr . "\n\n";
+			}
+			next;
+		}
+
+		# Skip header rows
+		if ($tr =~ /<th(?:\s+[^\>]*)?\>/i) {
+			if ($DEBUG > 2) {
+				print STDERR 'Skipping header TR line: ' . $tr . "\n\n";
+			}
+			next;
+		}
+
+		# Save good TRs
+		push(@trs, $tr);
+	}
+
+	return @trs;
+}
+
 # Extract season
 sub findSE($) {
 	my ($title) = @_;

sync/recentlyUnwatched.sh

@@ -52,7 +52,7 @@ SERIES_COUNT=0
 IFS=$'\n'
 for i in $SERIES; do
 	SERIES_COUNT=$(( $SERIES_COUNT + 1 ))
-	FILES="`curl ${CURL_OPTS[@]} "${HOST}/library/metadata/${i}/${URL2_POST}" | \
+	FILES="`curl ${CURL_OPTS[@]} "${HOST}/library/metadata/${i}/${URL2_POST}" 2>/dev/null | \
 		grep '<Part ' | \
 		head -n "${MAX_RESULTS}" | \
 		sed 's%^.*file="\([^\"]*\)".*$%\1%' | \

transmission.pl

@@ -186,7 +186,9 @@ foreach my $tor (@{$torrents}) {
 				print STDERR 'Torrent stored successfully: ' . $tor->{'name'} . "\n";
 			}
 		} elsif (defined($result) && $result == -1) {
-			print STDERR 'Deleting bad torrent: ' . $tor->{'name'} . "\n";
+			if ($DEBUG) {
+				print STDERR 'Deleting bad torrent: ' . $tor->{'name'} . "\n";
+			}
 			last;
 		} else {
 			print STDERR 'Error storing file "' . basename($file) . '" from torrent: ' . $tor->{'name'} . "\n";
@@ -465,10 +467,10 @@ sub delTor($) {
 
 sub guessExt($) {
 	my ($file) = @_;
+	my $ext = '';
 
 	# Believe most non-avi file extensions without checking
 	# It's mostly "avi" that lies, and the checks are expensive
-	my $ext = '';
 	{
 		my $orig_ext = basename($file);
 		$orig_ext =~ s/^.*\.(\w{2,3})$/$1/;
@@ -478,6 +480,11 @@ sub guessExt($) {
 		}
 	}
 
+	# Deal with the weird case of text files labeled as .txt.mp4
+	if ($file =~ /\.txt\.mp4$/i) {
+		$ext = 'txt';
+	}
+
 	# Ask movInfo.pl about the demuxer
 	if (!$ext) {
 		my $demux = '';
@@ -529,22 +536,26 @@ sub processFile($$) {
 	my $ext = &guessExt($file);
 	my $filename = basename($file);
 
-	# Delete WMV files -- mostly viruses
+	# Delete WMV files -- mostly viruses/fake
 	if ($ext =~ /wmv/i) {
-		print STDERR 'Declining to save WMV file: ' . $filename . "\n";
+		if ($DEBUG) {
+			print STDERR 'Declining to save WMV file: ' . $filename . "\n";
+		}
 		return -1;
 	}
 
 	# Delete ZIP files -- mostly fake
 	if ($ext =~ /zip/i) {
-		print STDERR 'Declining to save ZIP file: ' . $filename . "\n";
+		if ($DEBUG) {
+			print STDERR 'Declining to save ZIP file: ' . $filename . "\n";
+		}
 		return -1;
 	}
 
-	# Skip RARBG.com files with success code -- always extra but not a problem
-	if ($filename =~ /RARBG\.com/i) {
+	# Skip known packaging files files with success code
+	if ($ext =~ /txt/i || $filename =~ /RARBG\.com/i) {
 		if ($DEBUG) {
-			print STDERR 'Declining to save RARBG.com file: ' . $filename . "\n";
+			print STDERR 'Declining to save extra file: ' . $filename . "\n";
 		}
 		return 1;
 	}