init: add --skip-artnum parameter

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index bf5a0df947aaa1d04275461663ae29167f581b3a..a0f041dd3dc0d287918142a39ceecff2b0abda3d 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -19,7 +19,6 @@ use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::SearchIdx;
-use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::MultiMidQueue;
 use IO::Handle; # ->autoflush
 use File::Temp qw(tempfile);
@@ -75,11 +74,14 @@ sub count_shards ($) {
        # Also, shard count may change while -watch is running
        # due to "xcpdb --reshard"
        if (-d $xpfx) {
-               require PublicInbox::Search;
-               PublicInbox::Search::load_xapian();
-               my $XapianDatabase = $PublicInbox::Search::X{Database};
+               my $XapianDatabase;
                foreach my $shard (<$xpfx/*>) {
                        -d $shard && $shard =~ m!/[0-9]+\z! or next;
+                       $XapianDatabase //= do {
+                               require PublicInbox::Search;
+                               PublicInbox::Search::load_xapian();
+                               $PublicInbox::Search::X{Database};
+                       };
                        eval {
                                $XapianDatabase->new($shard)->close;
                                $n++;
@@ -126,12 +128,13 @@ sub new {
 
 # public (for now?)
 sub init_inbox {
-       my ($self, $shards, $skip_epoch) = @_;
+       my ($self, $shards, $skip_epoch, $skip_artnum) = @_;
        if (defined $shards) {
                $self->{parallel} = 0 if $shards == 0;
                $self->{shards} = $shards if $shards > 0;
        }
        $self->idx_init;
+       $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
        my $epoch_max = -1;
        git_dir_latest($self, \$epoch_max);
        if (defined $skip_epoch && $epoch_max == -1) {
@@ -153,13 +156,13 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$) {
        my ($self, $msgref, $mime, $smsg) = @_;
-       $smsg->{ds} //= msg_datestamp($mime->header_obj, $self->{autime});
-       $smsg->{ts} //= msg_timestamp($mime->header_obj, $self->{cotime});
+       $smsg->{bytes} = $smsg->{raw_bytes} +
+                       PublicInbox::SearchIdx::crlf_adjust($$msgref);
        $self->{over}->add_overview($mime, $smsg);
        my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
        $idx->index_raw($msgref, $mime, $smsg);
-       my $n = $self->{transact_bytes} += $smsg->{bytes};
-       $n >= (PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
+       my $n = $self->{transact_bytes} += $smsg->{raw_bytes};
+       $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
 }
 
 sub _add {
@@ -179,7 +182,7 @@ sub _add {
 
        my ($num, $mid0) = v2_num_for($self, $mime);
        defined $num or return; # duplicate
-       defined $mid0 or die "BUG: $mid0 undefined\n";
+       defined $mid0 or die "BUG: \$mid0 undefined\n";
        my $im = $self->importer;
        my $smsg = bless { mid => $mid0, num => $num }, 'PublicInbox::Smsg';
        my $cmt = $im->add($mime, undef, $smsg); # sets $smsg->{ds|ts|blob}
@@ -568,10 +571,12 @@ W: $list
        for my $smsg (@$need_reindex) {
                my $new_smsg = bless {
                        blob => $blob,
-                       bytes => $bytes,
+                       raw_bytes => $bytes,
                        num => $smsg->{num},
                        mid => $smsg->{mid},
                }, 'PublicInbox::Smsg';
+               my $v2w = { autime => $smsg->{ds}, cotime => $smsg->{ts} };
+               $new_smsg->populate($new_mime, $v2w);
                do_idx($self, \$raw, $new_mime, $new_smsg);
        }
        $rewritten->{rewrites};
@@ -606,7 +611,7 @@ sub barrier_wait {
        my $bnote = $self->{bnote} or return;
        my $r = $bnote->[0];
        while (scalar keys %$barrier) {
-               defined(my $l = $r->getline) or die "EOF on barrier_wait: $!";
+               defined(my $l = readline($r)) or die "EOF on barrier_wait: $!";
                $l =~ /\Abarrier (\d+)/ or die "bad line on barrier_wait: $l";
                delete $barrier->{$1} or die "bad shard[$1] on barrier wait";
        }
@@ -960,11 +965,12 @@ sub reindex_oid_m ($$$$;$) {
        }
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1051,11 +1057,12 @@ sub reindex_oid ($$$$) {
                die "failed to delete <$mid0> for article #$num\n";
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1295,7 +1302,7 @@ sub index_epoch ($$$) {
                }
        }
        close $fh or die "git log failed: \$?=$?";
-       delete $self->{reindex_pipe};
+       delete @$self{qw(reindex_pipe autime cotime)};
        update_last_commit($self, $git, $i, $cmt) if defined $cmt;
 }