wwwstream: use parent.pm and no warnings

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 79bee7f9f3d1125fbd918d27ac9d0c7c85cd88f3..528f5e9a5650df2866b60e5ad602306a8c0eda37 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -113,6 +113,7 @@ sub new {
                im => undef, #  PublicInbox::Import
                parallel => 1,
                transact_bytes => 0,
+               total_bytes => 0,
                current_info => '',
                xpfx => $xpfx,
                over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3", 1),
@@ -128,12 +129,13 @@ sub new {
 
 # public (for now?)
 sub init_inbox {
-       my ($self, $shards, $skip_epoch) = @_;
+       my ($self, $shards, $skip_epoch, $skip_artnum) = @_;
        if (defined $shards) {
                $self->{parallel} = 0 if $shards == 0;
                $self->{shards} = $shards if $shards > 0;
        }
        $self->idx_init;
+       $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
        my $epoch_max = -1;
        git_dir_latest($self, \$epoch_max);
        if (defined $skip_epoch && $epoch_max == -1) {
@@ -155,10 +157,12 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$) {
        my ($self, $msgref, $mime, $smsg) = @_;
+       $smsg->{bytes} = $smsg->{raw_bytes} +
+                       PublicInbox::SearchIdx::crlf_adjust($$msgref);
        $self->{over}->add_overview($mime, $smsg);
        my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
        $idx->index_raw($msgref, $mime, $smsg);
-       my $n = $self->{transact_bytes} += $smsg->{bytes};
+       my $n = $self->{transact_bytes} += $smsg->{raw_bytes};
        $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
 }
 
@@ -167,7 +171,7 @@ sub _add {
 
        # spam check:
        if ($check_cb) {
-               $mime = $check_cb->($mime) or return;
+               $mime = $check_cb->($mime, $self->{-inbox}) or return;
        }
 
        # All pipes (> $^F) known to Perl 5.6+ have FD_CLOEXEC set,
@@ -568,7 +572,7 @@ W: $list
        for my $smsg (@$need_reindex) {
                my $new_smsg = bless {
                        blob => $blob,
-                       bytes => $bytes,
+                       raw_bytes => $bytes,
                        num => $smsg->{num},
                        mid => $smsg->{mid},
                }, 'PublicInbox::Smsg';
@@ -656,6 +660,7 @@ sub checkpoint ($;$) {
 
                $dbh->begin_work;
        }
+       $self->{total_bytes} += $self->{transact_bytes};
        $self->{transact_bytes} = 0;
 }
 
@@ -678,8 +683,9 @@ sub done {
        }
        $self->{over}->disconnect;
        delete $self->{bnote};
-       $self->{transact_bytes} = 0;
-       $self->lock_release if $shards;
+       my $nbytes = $self->{total_bytes};
+       $self->{total_bytes} = 0;
+       $self->lock_release(!!$nbytes) if $shards;
        $self->{-inbox}->git->cleanup;
 }
 
@@ -962,7 +968,7 @@ sub reindex_oid_m ($$$$;$) {
        }
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
@@ -1054,7 +1060,7 @@ sub reindex_oid ($$$$) {
                die "failed to delete <$mid0> for article #$num\n";
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,