critrole.pl 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use LWP::UserAgent;
  5. use File::Basename;
  6. use IPC::Cmd qw( can_run );
  7. # Parameters
  8. my $TOP_URL = 'http://geekandsundry.com/shows/critical-role/';
  9. my $PG_MATCH = qr/\<link rel=\'next\' href=\'([^\']+\/page\/\d+\/)\'/;
  10. my $EP_MATCH = qr/^https?\:\/\/[^\/]+\/(?:critical\-role\-episode\-\d+|\d{4,}\-\d\/)/i;
  11. my $INI_PATH = `~/bin/video/mediaPath` . '/YouTube/Critical Role (None)/extra_videos.ini';
  12. my $UA_STR = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A';
  13. my $TIMEOUT = 10;
  14. # Debug
  15. our $DEBUG = 0;
  16. if (defined($ENV{'DEBUG'}) && $ENV{'DEBUG'} =~ /(\d+)/) {
  17. $DEBUG = $1;
  18. }
  19. # ENV Parameters
  20. if (exists($ENV{'TOP_URL'})) {
  21. $TOP_URL = $ENV{'TOP_URL'};
  22. if ($DEBUG) {
  23. print STDERR 'Using ENV TOP_URL: ' . $TOP_URL . "\n";
  24. }
  25. }
  26. if (exists($ENV{'EP_MATCH'})) {
  27. $EP_MATCH = qr/${ENV{'EP_MATCH'}}/i;
  28. if ($DEBUG) {
  29. print STDERR 'Using ENV EP_MATCH: qr/' . $ENV{'EP_MATCH'} . "/i\n";
  30. }
  31. }
  32. if (exists($ENV{'INI_PATH'})) {
  33. $INI_PATH = $ENV{'INI_PATH'};
  34. if ($DEBUG) {
  35. print STDERR 'Using ENV INI_PATH: ' . $INI_PATH . "\n";
  36. }
  37. }
  38. # Setup LWP
  39. my $ua = LWP::UserAgent->new();
  40. if (!$ua) {
  41. die('Unable to init UA: ' . $! . "\n");
  42. }
  43. $ua->agent($UA_STR);
  44. $ua->timeout($TIMEOUT);
  45. $ua->env_proxy();
  46. # Read the existing INI file
  47. my %old_ytids = ();
  48. {
  49. my $fh = undef();
  50. open($fh, '<', $INI_PATH)
  51. or die('Unable to read ini file: ' . $INI_PATH . ': ' . $! . "\n");
  52. my $url = undef();
  53. while (<$fh>) {
  54. if (/^\s*#\s*(https?\:\/\/[^\s]+)/i) {
  55. $url = $1;
  56. } elsif (/([\w\-]+)/) {
  57. if (!$url) {
  58. $url = 'Unknown';
  59. }
  60. $old_ytids{$1} = $url;
  61. $url = undef();
  62. if ($DEBUG > 1) {
  63. print STDERR 'Found existing YTID: ' . $1 . ' # ' . $old_ytids{$1} . "\n";
  64. }
  65. }
  66. }
  67. close($fh);
  68. }
  69. # Use the top-level page to find episode-specific pages
  70. my @episodes = ();
  71. {
  72. my $url = $TOP_URL;
  73. NEXT_PAGE: {
  74. if ($DEBUG) {
  75. print STDERR 'Loading top page: ' . $url . "\n";
  76. }
  77. my $response = $ua->get($url);
  78. if (!$response->is_success()) {
  79. die(basename($0) . ": Unable to fetch top-level page\n");
  80. }
  81. foreach my $link (split(/\<a\s+/, $response->decoded_content())) {
  82. if ($link =~ /\s+href=\"([^\"]+)\"/) {
  83. my $href = $1;
  84. if ($href =~ $EP_MATCH) {
  85. push(@episodes, $href);
  86. }
  87. }
  88. }
  89. if ($response->decoded_content() =~ $PG_MATCH) {
  90. $url = $1;
  91. redo NEXT_PAGE;
  92. }
  93. }
  94. }
  95. if ($DEBUG) {
  96. print STDERR 'Episode URLs (' . scalar(@episodes) . "):\n\t" . join("\n\t", @episodes) . "\n";
  97. }
  98. # Find each episode's YTID
  99. my %ytids = ();
  100. {
  101. foreach my $url (@episodes) {
  102. my $response = $ua->get($url);
  103. if (!$response->is_success()) {
  104. die(basename($0) . ': Unable to fetch episode page: ' . $url . "\n");
  105. }
  106. if ($response->decoded_content() =~ /data-ytid=\"([^\"]+)\"/) {
  107. $ytids{$1} = $url;
  108. } elsif ($DEBUG > 0) {
  109. print STDERR 'No YTID found on: ' .$url . "\n";
  110. if ($DEBUG > 1) {
  111. print STDERR "\tPage content:\n" . $response->decoded_content() . "\n\n";
  112. }
  113. }
  114. }
  115. }
  116. if ($DEBUG) {
  117. print STDERR 'YouTube IDs (' . scalar(keys(%ytids)) . "):\n\t" . join("\n\t", keys(%ytids)) . "\n";
  118. }
  119. # Drop known episodes
  120. {
  121. foreach my $id (keys(%ytids)) {
  122. if (exists($old_ytids{$id})) {
  123. if ($DEBUG > 1) {
  124. print 'Ignoring known episode: ' . $id . ' # ' . $old_ytids{$id} . "\n";
  125. }
  126. delete($ytids{$id});
  127. }
  128. }
  129. }
  130. # Update the INI file
  131. {
  132. my $fh = undef();
  133. open($fh, '>>', $INI_PATH)
  134. or die('Unable to write ini file: ' . $INI_PATH . ': ' . $! . "\n");
  135. foreach my $id (keys(%ytids)) {
  136. if ($DEBUG) {
  137. print 'Adding episode: ' . $id . ' # ' . $ytids{$id} . "\n";
  138. }
  139. print $fh "\n# " . $ytids{$id} . "\n" . $id . "\n";
  140. }
  141. close($fh);
  142. }
  143. # Autodate
  144. my @autodate = ($ENV{'HOME'} . '/bin/video/yt/autodate.sh');
  145. if (can_run($autodate[0])) {
  146. if ($DEBUG) {
  147. print STDERR "Executing autodate script...\n";
  148. }
  149. system(@autodate);
  150. }
  151. # Cleanup
  152. exit(0);