rss.pl 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. #!/usr/bin/perl
  2. use strict;
  3. use warnings;
  4. use Cwd qw(abs_path);
  5. use MP3::Tag;
  6. use XML::Feed;
  7. use LWP::Simple;
  8. use HTML::Strip;
  9. use Date::Parse;
  10. use Date::Format;
  11. use File::Basename;
  12. # Command-line
  13. my $OUT_DIR = abs_path($ARGV[0]);
  14. if (!$OUT_DIR || !-d $OUT_DIR) {
  15. die('Usage: ' . basename($0) . " output_directory\n");
  16. }
  17. # Parameters
  18. my $GENRE = 'Podcast';
  19. my $SERIES = basename($OUT_DIR);
  20. my $URL_FILE = $OUT_DIR . '/url';
  21. my $RULE_FILE = $OUT_DIR . '/rules.pm';
  22. my $ERR_RATIO = 0.05;
  23. my $DL_LIMIT = 50;
  24. # Init
  25. our $DEBUG = 0;
  26. if (defined($ENV{'DEBUG'}) && $ENV{'DEBUG'} =~ /(\d+)/) {
  27. $DEBUG = $1;
  28. }
  29. my %episodes = ();
  30. my %have = ();
  31. my @need = ();
  32. my $hs = HTML::Strip->new();
  33. # Skip if there is a season_done file
  34. if (-e $OUT_DIR . '/season_done') {
  35. if ($DEBUG) {
  36. print STDERR 'Podcast disabled: ' . $SERIES . "\n";
  37. }
  38. exit(0);
  39. }
  40. # Fetch and analyize the feed
  41. my $feed = undef();
  42. {
  43. open(my $fh, '<', $URL_FILE)
  44. or die('Unable to open feed file: ' . $URL_FILE . "\n");
  45. my $url = <$fh>;
  46. close($fh);
  47. if (!$url || length($url) < 5) {
  48. die('Invalid feed URL: ' . $url . "\n");
  49. }
  50. $feed = XML::Feed->parse(URI->new($url));
  51. if (!$feed) {
  52. die('Unable to parse feed (' . XML::Feed->errstr() . '): ' . $url . "\n");
  53. }
  54. }
  55. # Parse each episode from the feed
  56. for my $entry ($feed->entries()) {
  57. my $ep = $entry->unwrap();
  58. # Title
  59. my $title = '';
  60. if (defined($ep->{'title'})) {
  61. $title = $ep->{'title'};
  62. }
  63. $title = $hs->parse($title);
  64. $hs->eof();
  65. $title =~ s/^\s+//;
  66. $title =~ s/\s+$//;
  67. # Description
  68. my $desc = '';
  69. if (defined($ep->{'description'})) {
  70. $desc = $ep->{'description'};
  71. }
  72. $desc = $hs->parse($desc);
  73. $hs->eof();
  74. $desc =~ s/^\s+//;
  75. $desc =~ s/\s+$//;
  76. # Timestamp
  77. my $time = str2time($ep->{'pubDate'});
  78. # Author
  79. my $author = undef();
  80. if (exists($ep->{'author'})) {
  81. $author = $ep->{'author'};
  82. $author =~ s/^\s+//;
  83. $author =~ s/\s+$//;
  84. }
  85. # Enclosure (i.e. attachment/file) data
  86. my ($url, $ext, $size) = undef();
  87. {
  88. my $enc = $ep->{'enclosure'};
  89. if ($enc && ref($enc) eq 'HASH' && exists($enc->{'url'})) {
  90. $url = $enc->{'url'};
  91. if (exists($enc->{'type'})) {
  92. if ($enc->{'type'} =~ /audio\/mp4/i) {
  93. $ext = 'm4a';
  94. } elsif ($enc->{'type'} =~ /audio\/mpeg/i) {
  95. $ext = 'mp3';
  96. } elsif ($enc->{'type'} =~ /video\//i) {
  97. if ($DEBUG > 1) {
  98. print STDERR 'Skipping video file: ' . $time . "\n";
  99. }
  100. } else {
  101. print STDERR 'Unknown MIME type: ' . $enc->{'type'} . "\n";
  102. }
  103. }
  104. if (exists($enc->{'length'})) {
  105. $size = $enc->{'length'};
  106. }
  107. }
  108. }
  109. # iTunes data, if available (and useful)
  110. my $duration = undef();
  111. if ($ep->{'http://www.itunes.com/dtds/podcast-1.0.dtd'}) {
  112. my $itunes = $ep->{'http://www.itunes.com/dtds/podcast-1.0.dtd'};
  113. if (exists($itunes->{'duration'})) {
  114. if ($itunes->{'duration'} =~ /(\d{1,2})\:(\d{1,2})\:(\d{1,2})/) {
  115. $duration = (3600 * int($1)) + (60 * int($2)) + int($3);
  116. } else {
  117. $duration = int($itunes->{'duration'});
  118. }
  119. }
  120. if (!$desc && exists($itunes->{'summary'})) {
  121. $desc = $itunes->{'summary'};
  122. $desc =~ s/^\s+//;
  123. $desc =~ s/\s+$//;
  124. }
  125. if (!$author && exists($itunes->{'author'})) {
  126. $author = $itunes->{'author'};
  127. $author =~ s/^\s+//;
  128. $author =~ s/\s+$//;
  129. }
  130. }
  131. # Collect extracted, cleaned data
  132. if ($title && $desc && $time && $url && $ext) {
  133. my %tmp = (
  134. 'title' => $title,
  135. 'description' => $desc,
  136. 'time' => $time,
  137. 'url' => $url,
  138. 'ext' => $ext,
  139. 'size' => $size,
  140. 'duration' => $duration,
  141. 'author' => $author,
  142. );
  143. $episodes{$time} = \%tmp;
  144. } elsif ($DEBUG) {
  145. print STDERR 'Skipping incomplete entry: ' . $title . ': ' . $time . "\n";
  146. }
  147. }
  148. # Check for existing files
  149. opendir(my $fh, $OUT_DIR)
  150. or die('Unable to open output directory (' . $! . '): ' . $OUT_DIR . "\n");
  151. while (my $file = readdir($fh)) {
  152. my $path = $OUT_DIR . '/' . $file;
  153. if (-d $path) {
  154. next;
  155. }
  156. if ($file =~ /^(.*)\s+\((\d{7,})\)\.\w{3}$/) {
  157. my $title = $1;
  158. my $time = $2;
  159. if (exists($episodes{$time})) {
  160. if (exists($episodes{$time}->{'size'}) && $episodes{$time}->{'size'}) {
  161. (undef(), undef(), undef(), undef(), undef(), undef(), undef(), my $size) = stat($path);
  162. my $ratio = ($size / $episodes{$time}->{'size'});
  163. if ($ratio < (1 - $ERR_RATIO) || $ratio > (1 + $ERR_RATIO)) {
  164. print STDERR 'Invalid output file size (' . $size . '/' . $episodes{$time}->{'size'} . '): ' . $file . "\n";
  165. }
  166. }
  167. $have{$time} = 1;
  168. $episodes{$time}->{'title'} = $title;
  169. }
  170. if ($DEBUG) {
  171. print STDERR 'Found file: ' . $file . "\n";
  172. }
  173. } elsif ($DEBUG > 1) {
  174. print STDERR 'Skipping file: ' . $file . "\n";
  175. }
  176. }
  177. closedir($fh);
  178. # Decide what we want to download
  179. foreach my $time (keys(%episodes)) {
  180. if (!exists($have{$time})) {
  181. push(@need, $time);
  182. if ($DEBUG > 1) {
  183. print STDERR 'Will fetch: ' . $time . "\n";
  184. }
  185. }
  186. }
  187. # Apply naming rules
  188. if (-e $RULE_FILE) {
  189. if ($DEBUG > 1) {
  190. print STDERR 'Importing local rules from: ' . $RULE_FILE . "\n";
  191. }
  192. require $RULE_FILE;
  193. localRules(\%episodes);
  194. } else {
  195. foreach my $time (sort(keys(%episodes))) {
  196. $episodes{$time}->{'title'} = time2str('%Y-%m-%d', $time) . ' - 1 - ' . $episodes{$time}->{'title'};
  197. }
  198. }
  199. foreach my $time (keys(%episodes)) {
  200. $episodes{$time}->{'title'} =~ s/\s*\:\s*/ - /g;
  201. $episodes{$time}->{'title'} =~ s/\s+/ /g;
  202. $episodes{$time}->{'title'} =~ s/\"/\'/g;
  203. $episodes{$time}->{'title'} =~ s/[^\w\s\,\-\.\!\'\(\)\#\&\@]//g;
  204. }
  205. # Fetch needed files, with basic validation
  206. my $dlCount = 0;
  207. foreach my $time (sort(@need)) {
  208. if ($dlCount >= $DL_LIMIT) {
  209. die('Download limit (' . $DL_LIMIT . ') reached: ' . $SERIES . "\n");
  210. }
  211. # Build a file path
  212. my $ep = $episodes{$time};
  213. my $file = $OUT_DIR . '/' . $ep->{'title'} . ' (' . int($time) . ').' . $ep->{'ext'};
  214. if (-e $file) {
  215. die('Unexpected output file: ' . $file . "\n");
  216. }
  217. # Debug
  218. if ($DEBUG) {
  219. print STDERR 'Fetching (' . $time . '): ' . $ep->{'title'} . ' => ' . $ep->{'url'} . "\n";
  220. }
  221. # Fetch
  222. my $code = getstore($ep->{'url'}, $file);
  223. $dlCount++;
  224. # Validate
  225. my $err = undef();
  226. if ($code != 200) {
  227. $err = 'Invalid HTTP response: ' . $code;
  228. goto OUT;
  229. }
  230. if (!-e $file) {
  231. $err = 'No output file: ' . $file;
  232. goto OUT;
  233. }
  234. if ($ep->{'size'}) {
  235. (undef(), undef(), undef(), undef(), undef(), undef(), undef(), my $size) = stat($file);
  236. my $ratio = $size / $ep->{'size'};
  237. if ($ratio < (1 - $ERR_RATIO) || $ratio > (1 + $ERR_RATIO)) {
  238. $err = 'Invalid download size (' . $size . '/' . $ep->{'size'} . '): ' . $file;
  239. goto OUT;
  240. }
  241. }
  242. # Set MP3/M4A tags
  243. if ($ep->{'ext'} eq 'mp3') {
  244. # Parse the MP3
  245. my $mp3 = MP3::Tag->new($file);
  246. if (!$mp3) {
  247. $err = 'Unable to parse MP3 tags in: ' . $file . "\n";
  248. goto OUT;
  249. }
  250. # Delete any ID3v1 tags
  251. if (exists($mp3->{'ID3v1'})) {
  252. print STDERR "Removing ID3v1 tag\n";
  253. $mp3->{'ID3v1'}->remove_tag();
  254. print STDERR "Removed?\n";
  255. }
  256. # Find or create the ID3v2 tag
  257. my $tags = undef();
  258. if (exists($mp3->{'ID3v2'})) {
  259. $tags = $mp3->{'ID3v2'};
  260. } else {
  261. $tags = $mp3->new_tag('ID3v2');
  262. }
  263. # Update tags
  264. $tags->title($ep->{'title'});
  265. $tags->year(time2str('%Y', $ep->{'time'}));
  266. $tags->comment($ep->{'description'});
  267. $tags->album($SERIES);
  268. $tags->artist($SERIES);
  269. $tags->genre($GENRE);
  270. # Save
  271. if (!$tags->write_tag()) {
  272. $err = 'Unable to update MP3 tags in: ' . $file . "\n";
  273. goto OUT;
  274. }
  275. $mp3->close();
  276. } else {
  277. print STDERR 'Tag munging not available for ' . $ep->{'ext'} . ' files: ' . $file . "\n";
  278. }
  279. OUT:
  280. # Cleanup
  281. if (defined($err)) {
  282. print STDERR 'Unable to download ' . $ep->{'url'} . ': ' . $err . "\n";
  283. if (-e $file) {
  284. unlink($file);
  285. }
  286. }
  287. }