imap+nntp: share COMPRESS implementation

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 36b84f5708ae27ff52e01d082b630bbdb1c439a9..ed5182ae846029a5af0837060335c7d7d8a45066 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -267,9 +267,7 @@ sub _idx_init { # with_umask callback
 
        # Now that all subprocesses are up, we can open the FDs
        # for SQLite:
-       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file(
-                               "$ibx->{inboxdir}/msgmap.sqlite3",
-                               $ibx->{-no_fsync} ? 2 : 1);
+       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1);
        $mm->{dbh}->begin_work;
 }
 
@@ -549,11 +547,11 @@ sub checkpoint ($;$) {
        }
        my $shards = $self->{idx_shards};
        if ($shards) {
-               my $mm = $self->{mm};
-               my $dbh = $mm->{dbh} if $mm;
+               my $dbh = $self->{mm}->{dbh} if $self->{mm};
 
                # SQLite msgmap data is second in importance
                $dbh->commit if $dbh;
+               eval { $dbh->do('PRAGMA optimize') };
 
                # SQLite overview is third
                $self->{oidx}->commit_lazy;
@@ -623,6 +621,11 @@ sub done {
                eval { $mm->{dbh}->$m };
                $err .= "msgmap $m: $@\n" if $@;
        }
+       if ($self->{oidx} && $self->{oidx}->{dbh} && $err) {
+               eval { $self->{oidx}->rollback_lazy };
+               $err .= "overview rollback: $@\n" if $@;
+       }
+
        my $shards = delete $self->{idx_shards};
        if ($shards) {
                for (@$shards) {
@@ -810,8 +813,8 @@ sub index_oid { # cat_async callback
                        }
                }
        }
+       my $oidx = $self->{oidx};
        if (!defined($num)) { # reuse if reindexing (or duplicates)
-               my $oidx = $self->{oidx};
                for my $mid (@$mids) {
                        ($num, $mid0) = $oidx->num_mid0_for_oid($oid, $mid);
                        last if defined $num;
@@ -819,6 +822,11 @@ sub index_oid { # cat_async callback
        }
        $mid0 //= do { # is this a number we got before?
                $num = $arg->{mm_tmp}->num_for($mids->[0]);
+
+               # don't clobber existing if Message-ID is reused:
+               if (my $x = defined($num) ? $oidx->get_art($num) : undef) {
+                       undef($num) if $x->{blob} ne $oid;
+               }
                defined($num) ? $mids->[0] : undef;
        };
        if (!defined($num)) {
@@ -1031,7 +1039,7 @@ sub sync_prepare ($$) {
                        my $req = { %$sync, oid => $oid };
                        $self->git->cat_async($oid, $unindex_oid, $req);
                }
-               $self->git->cat_async_wait;
+               $self->git->async_wait_all;
        }
        return 0 if $sync->{quit};
        if (!$regen_max) {
@@ -1113,7 +1121,7 @@ sub unindex_todo ($$$) {
                $self->git->cat_async($1, $unindex_oid, { %$sync, oid => $1 });
        }
        close $fh or die "git log failed: \$?=$?";
-       $self->git->cat_async_wait;
+       $self->git->async_wait_all;
 
        return unless $sync->{-opt}->{prune};
        my $after = scalar keys %$unindexed;
@@ -1245,7 +1253,7 @@ sub xapian_only {
                        index_xap_step($self, $sync, $art_beg, 1);
                }
        }
-       $self->git->cat_async_wait;
+       $self->git->async_wait_all;
        $self->{ibx}->cleanup;
        $self->done;
 }