init: add --skip-artnum parameter

For archivists with only newer mail archives, this option allows
reserving reserve NNTP article numbers for yet-to-be-archived
old messages.  Indexers will need to be updated to support this
feature in future commits.

-V1 inboxes will now be initialized with SQLite and Xapian
support if this option is used, or if --indexlevel= is
specified.

diff --git a/Documentation/public-inbox-init.pod b/Documentation/public-inbox-init.pod
index 495a258fdb696c3037d9adc7cf262f3cd39f68cc..5714828d9a1da509b1cf1d79d6753419bf186e5a 100644 (file)
--- a/Documentation/public-inbox-init.pod
+++ b/Documentation/public-inbox-init.pod
@@ -39,6 +39,20 @@ See L<public-inbox-config(5)> for more information.
 
 Default: C<full>
 
+=item -N, --skip-artnum
+
+This option allows archivists to publish incomplete archives
+with only new mail while allowing NNTP article numbers
+to be reserved for yet-to-be-archived old mail.
+
+This is mainly intended for users of C<--skip-epoch> (documented below)
+but may be of use to L<public-inbox-v1-format(5)> users.
+
+There is no automatic way to use reserved NNTP article numbers
+when old mail is found, yet.
+
+Default: unset, no NNTP article numbers are skipped
+
 =item -S, --skip-epoch
 
 For C<-V2> (L<public-inbox-v2-format(5)>) inboxes only, this option

diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index c54be046f9557dcc5eb92312f9c6e3bf32707849..f9e28502001499b72d7f6e7766f4e824920862db 100644 (file)
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -39,10 +39,21 @@ sub assert_usable_dir {
 
 sub init_inbox {
        my ($self, $shards, $skip_epoch, $skip_artnum) = @_;
-       # TODO: honor skip_artnum
        if ($self->version == 1) {
                my $dir = assert_usable_dir($self);
                PublicInbox::Import::init_bare($dir);
+               if (defined($self->{indexlevel}) || defined($skip_artnum)) {
+                       require PublicInbox::SearchIdx;
+                       my $sidx = PublicInbox::SearchIdx->new($self, 1); # just create
+                       $sidx->begin_txn_lazy;
+                       $self->with_umask(sub {
+                               my $mm = PublicInbox::Msgmap->new($dir, 1);
+                               $mm->{dbh}->begin_work;
+                               $mm->skip_artnum($skip_artnum);
+                               $mm->{dbh}->commit;
+                       }) if defined($skip_artnum);
+                       $sidx->commit_txn_lazy;
+               }
        } else {
                my $v2w = importer($self);
                $v2w->init_inbox($shards, $skip_epoch, $skip_artnum);

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index d115cbce352cd3551b406d5b5c9d90782e9feab1..aa07e344c6a9d75a9b376af5a6fbbd5911481198 100644 (file)
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -270,4 +270,30 @@ sub atfork_prepare {
        %$self = (tmp_name => $f, pid => $$);
 }
 
+sub skip_artnum {
+       my ($self, $skip_artnum) = @_;
+       return meta_accessor($self, 'skip_artnum') if !defined($skip_artnum);
+
+       my $cur = num_highwater($self) // 0;
+       if ($skip_artnum < $cur) {
+               die "E: current article number $cur ",
+                       "exceeds --skip-artnum=$skip_artnum\n";
+       } else {
+               my $ok;
+               for (1..10) {
+                       my $mid = 'skip'.rand.'@'.rand.'.example.com';
+                       $ok = mid_set($self, $skip_artnum, $mid);
+                       if ($ok) {
+                               mid_delete($self, $mid);
+                               last;
+                       }
+               }
+               $ok or die '--skip-artnum failed';
+
+               # in the future, the indexer may use this value for
+               # new messages in old epochs
+               meta_accessor($self, 'skip_artnum', $skip_artnum);
+       }
+}
+
 1;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 85821ea706a41a66a9a0e1aee3c40aecab6c969a..00e63938fc068f6eb6c8e0199fc16130f96426d4 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -694,6 +694,7 @@ sub _git_log {
                } else {
                        # normal regen is for for fresh data
                        $self->{regen_down} = $fcount;
+                       $self->{regen_down} += $high unless $opts->{reindex};
                }
        } else {
                # Give oldest messages the smallest numbers

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 913794316330bbc261d6f236f0e5a355c5b11ec5..a0f041dd3dc0d287918142a39ceecff2b0abda3d 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -128,12 +128,13 @@ sub new {
 
 # public (for now?)
 sub init_inbox {
-       my ($self, $shards, $skip_epoch) = @_;
+       my ($self, $shards, $skip_epoch, $skip_artnum) = @_;
        if (defined $shards) {
                $self->{parallel} = 0 if $shards == 0;
                $self->{shards} = $shards if $shards > 0;
        }
        $self->idx_init;
+       $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
        my $epoch_max = -1;
        git_dir_latest($self, \$epoch_max);
        if (defined $skip_epoch && $epoch_max == -1) {

diff --git a/script/public-inbox-init b/script/public-inbox-init
index e8dcf4fc5743d1813eae3797e6178515737329e3..c7f3da6f28a5f8fe10511c41e32f890ca70b40f3 100755 (executable)
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -24,14 +24,12 @@ use File::Path qw/mkpath/;
 use Fcntl qw(:DEFAULT);
 use Cwd qw/abs_path/;
 
-my $version = undef;
-my $indexlevel = undef;
-my $skip_epoch;
-my $jobs;
+my ($version, $indexlevel, $skip_epoch, $skip_artnum, $jobs);
 my %opts = (
        'V|version=i' => \$version,
        'L|indexlevel=s' => \$indexlevel,
        'S|skip|skip-epoch=i' => \$skip_epoch,
+       'N|skip-artnum=i' => \$skip_artnum,
        'j|jobs=i' => \$jobs,
 );
 GetOptions(%opts) or usage();
@@ -152,7 +150,8 @@ if (defined $jobs) {
        $creat_opt->{nproc} = $jobs;
 }
 
-PublicInbox::InboxWritable->new($ibx, $creat_opt)->init_inbox(0, $skip_epoch);
+$ibx = PublicInbox::InboxWritable->new($ibx, $creat_opt);
+$ibx->init_inbox(0, $skip_epoch, $skip_artnum);
 
 # needed for git prior to v2.1.0
 umask(0077) if defined $perm;

diff --git a/t/init.t b/t/init.t
index 94c6184ed891a3760c0f102ad3fdbc44c9683dd2..e3e8a2297964be2ef359d252b7047d4ddb341ce7 100644 (file)
--- a/t/init.t
+++ b/t/init.t
@@ -93,12 +93,38 @@ SKIP: {
                is_deeply($gits, ["$tmpdir/skip1/git/1.git"], 'skip OK');
        }
 
-
        $cmd = [ '-init', '-V2', '--skip-epoch=2', 'skip2', "$tmpdir/skip2",
                   qw(http://example.com/skip2 skip2@example.com) ];
        ok(run_script($cmd), "--skip-epoch 2");
        my $gits = [ glob("$tmpdir/skip2/git/*.git") ];
        is_deeply($gits, ["$tmpdir/skip2/git/2.git"], 'skipping 2 works, too');
+
+       xsys(qw(git config), "--file=$ENV{PI_DIR}/config",
+                       'publicinboxmda.spamcheck', 'none') == 0 or
+                       BAIL_OUT "git config $?";
+       my $addr = 'skip3@example.com';
+       $cmd = [ qw(-init -V2 -Lbasic -N12 skip3), "$tmpdir/skip3",
+                  qw(http://example.com/skip3), $addr ];
+       ok(run_script($cmd), '--skip-artnum -V2');
+       my $env = { ORIGINAL_RECIPIENT => $addr };
+       my $mid = 'skip-artnum@example.com';
+       my $msg = "Message-ID: <$mid>\n\n";
+       my $rdr = { 0 => \$msg, 2 => \(my $err = '')  };
+       ok(run_script([qw(-mda --no-precheck)], $env, $rdr), 'deliver V1');
+       my $mm = PublicInbox::Msgmap->new_file("$tmpdir/skip3/msgmap.sqlite3");
+       my $n = $mm->num_for($mid);
+       is($n, 13, 'V2 NNTP article numbers skipped via --skip-artnum');
+
+       $addr = 'skip4@example.com';
+       $env = { ORIGINAL_RECIPIENT => $addr };
+       $cmd = [ qw(-init -V1 -N12 -Lmedium skip4), "$tmpdir/skip4",
+                  qw(http://example.com/skip4), $addr ];
+       ok(run_script($cmd), '--skip-artnum -V1');
+       ok(run_script([qw(-mda --no-precheck)], $env, $rdr), 'deliver V1');
+       $mm = PublicInbox::Msgmap->new("$tmpdir/skip4");
+       system "find $tmpdir/skip4 >&2";
+       $n = $mm->num_for($mid);
+       is($n, 13, 'V1 NNTP article numbers skipped via --skip-artnum');
 }
 
 done_testing();