scripts/import_gmane_spool

   1 #!/usr/bin/perl -w
   2 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
   3 # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
   4 #
   5 # One-off script to convert an slrnpull news spool from gmane, usage:
   6 =begin usage
   7         mkdir -p $HOME/.public-inbox
   8         MAINREPO=/path/to/your/repo.git
   9         export ORIGINAL_RECIPIENT='list@example.com'
  10         git init --bare $MAINREPO
  11         export GIT_CONFIG=$HOME/.public-inbox/config
  12         git config publicinbox.$LISTNAME.address $ORIGINAL_RECIPIENT
  13         git config publicinbox.$LISTNAME.mainrepo $MAINREPO
  14         unset GIT_CONFIG
  15         ./import_gmane_spool SLRNPULL_ROOT/news/foo/bar
  16 =cut
  17 use strict;
  18 use warnings;
  19 use Email::Simple;
  20 use PublicInbox::Filter;
  21 use PublicInbox::Config;
  22 use IPC::Run qw(run);
  23 sub usage { "Usage:\n".join("",grep(/\t/, `head -n 24 $0`)) }
  24 my $spool = shift @ARGV or die usage();
  25 defined $ENV{ORIGINAL_RECIPIENT} or die usage();
  26 my @args = ('public-inbox-mda');
  27
  28 chdir $spool or die "chdir $spool failed: $!\n";
  29
  30 foreach my $n (sort { $a <=> $b } grep(/\d+\z/, glob("*"))) {
  31         if (open my $fh, '<', $n) {
  32                 my $s = eval {
  33                         local $/;
  34                         Email::Simple->new(<$fh>);
  35                 };
  36
  37                 # gmane rewrites Received headers, which increases spamminess
  38                 my @h = $s->header("Original-Received");
  39                 if (@h) {
  40                         $s->header_set("Received", @h);
  41                         $s->header_set("Original-Received");
  42                 }
  43
  44                 # this is needed for "git rev-list --since=..." to work
  45                 local $ENV{GIT_COMMITTER_DATE} = $s->header('Date');
  46
  47                 # triggers for the SA HEADER_SPAM rule
  48                 foreach my $drop (qw(Approved)) { $s->header_set($drop) }
  49
  50                 # appears to be an old gmane bug:
  51                 $s->header_set("connect()");
  52
  53                 my $orig = $s->as_string;
  54                 close $fh or die "close failed: $!\n";
  55                 eval { run(\@args, \$orig) };
  56                 die "fail $n: $?\n" if $?;
  57                 die "fail $n: $@\n" if $@;
  58         } else {
  59                 warn "Failed to open $n: $!\n";
  60         }
  61 }