view: make /$INBOX/$MSGID/ permalink async

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index b393b31f0e205f3191169762fbd1af9d76fa3d52..528f5e9a5650df2866b60e5ad602306a8c0eda37 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -19,7 +19,6 @@ use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::SearchIdx;
-use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::MultiMidQueue;
 use IO::Handle; # ->autoflush
 use File::Temp qw(tempfile);
@@ -114,6 +113,7 @@ sub new {
                im => undef, #  PublicInbox::Import
                parallel => 1,
                transact_bytes => 0,
+               total_bytes => 0,
                current_info => '',
                xpfx => $xpfx,
                over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3", 1),
@@ -129,12 +129,13 @@ sub new {
 
 # public (for now?)
 sub init_inbox {
-       my ($self, $shards, $skip_epoch) = @_;
+       my ($self, $shards, $skip_epoch, $skip_artnum) = @_;
        if (defined $shards) {
                $self->{parallel} = 0 if $shards == 0;
                $self->{shards} = $shards if $shards > 0;
        }
        $self->idx_init;
+       $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
        my $epoch_max = -1;
        git_dir_latest($self, \$epoch_max);
        if (defined $skip_epoch && $epoch_max == -1) {
@@ -156,12 +157,12 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$) {
        my ($self, $msgref, $mime, $smsg) = @_;
-       $smsg->{ds} //= msg_datestamp($mime->header_obj, $self->{autime});
-       $smsg->{ts} //= msg_timestamp($mime->header_obj, $self->{cotime});
+       $smsg->{bytes} = $smsg->{raw_bytes} +
+                       PublicInbox::SearchIdx::crlf_adjust($$msgref);
        $self->{over}->add_overview($mime, $smsg);
        my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
        $idx->index_raw($msgref, $mime, $smsg);
-       my $n = $self->{transact_bytes} += $smsg->{bytes};
+       my $n = $self->{transact_bytes} += $smsg->{raw_bytes};
        $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
 }
 
@@ -170,7 +171,7 @@ sub _add {
 
        # spam check:
        if ($check_cb) {
-               $mime = $check_cb->($mime) or return;
+               $mime = $check_cb->($mime, $self->{-inbox}) or return;
        }
 
        # All pipes (> $^F) known to Perl 5.6+ have FD_CLOEXEC set,
@@ -182,7 +183,7 @@ sub _add {
 
        my ($num, $mid0) = v2_num_for($self, $mime);
        defined $num or return; # duplicate
-       defined $mid0 or die "BUG: $mid0 undefined\n";
+       defined $mid0 or die "BUG: \$mid0 undefined\n";
        my $im = $self->importer;
        my $smsg = bless { mid => $mid0, num => $num }, 'PublicInbox::Smsg';
        my $cmt = $im->add($mime, undef, $smsg); # sets $smsg->{ds|ts|blob}
@@ -571,10 +572,12 @@ W: $list
        for my $smsg (@$need_reindex) {
                my $new_smsg = bless {
                        blob => $blob,
-                       bytes => $bytes,
+                       raw_bytes => $bytes,
                        num => $smsg->{num},
                        mid => $smsg->{mid},
                }, 'PublicInbox::Smsg';
+               my $v2w = { autime => $smsg->{ds}, cotime => $smsg->{ts} };
+               $new_smsg->populate($new_mime, $v2w);
                do_idx($self, \$raw, $new_mime, $new_smsg);
        }
        $rewritten->{rewrites};
@@ -657,6 +660,7 @@ sub checkpoint ($;$) {
 
                $dbh->begin_work;
        }
+       $self->{total_bytes} += $self->{transact_bytes};
        $self->{transact_bytes} = 0;
 }
 
@@ -679,8 +683,9 @@ sub done {
        }
        $self->{over}->disconnect;
        delete $self->{bnote};
-       $self->{transact_bytes} = 0;
-       $self->lock_release if $shards;
+       my $nbytes = $self->{total_bytes};
+       $self->{total_bytes} = 0;
+       $self->lock_release(!!$nbytes) if $shards;
        $self->{-inbox}->git->cleanup;
 }
 
@@ -963,11 +968,12 @@ sub reindex_oid_m ($$$$;$) {
        }
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1054,11 +1060,12 @@ sub reindex_oid ($$$$) {
                die "failed to delete <$mid0> for article #$num\n";
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1298,7 +1305,7 @@ sub index_epoch ($$$) {
                }
        }
        close $fh or die "git log failed: \$?=$?";
-       delete $self->{reindex_pipe};
+       delete @$self{qw(reindex_pipe autime cotime)};
        update_last_commit($self, $git, $i, $cmt) if defined $cmt;
 }