nntp: use Inbox->uidvalidity instead of ->mm->created_at

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 2d230dc18045e5bff145f4e2a0bcb6d513e2178c..7ab0c4af64a9a3c5951cf796bd017f51d3984947 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -21,6 +21,7 @@ use Carp qw(croak carp);
 use PublicInbox::Search;
 use PublicInbox::SearchIdx qw(crlf_adjust prepare_stack is_ancestor);
 use PublicInbox::OverIdx;
+use PublicInbox::MiscIdx;
 use PublicInbox::MID qw(mids);
 use PublicInbox::V2Writable;
 use PublicInbox::InboxWritable;
@@ -297,7 +298,7 @@ sub _sync_inbox ($$$) {
                my $epoch_max;
                defined($ibx->git_dir_latest(\$epoch_max)) or return;
                $sync->{epoch_max} = $epoch_max;
-               sync_prepare($self, $sync) or return; # fills $sync->{todo}
+               sync_prepare($self, $sync); # or return # TODO: once MiscIdx is stable
        } elsif ($v == 1) {
                my $uv = $ibx->uidvalidity;
                my $lc = $self->{oidx}->eidx_meta("lc-v1:$ekey//$uv");
@@ -308,7 +309,11 @@ sub _sync_inbox ($$$) {
                warn "E: $ekey unsupported inbox version (v$v)\n";
                return;
        }
-       index_todo($self, $sync, $_) for @{$sync->{todo}};
+       unless ($sync->{quit}) {
+               index_todo($self, $sync, $_) for @{delete($sync->{todo}) // []};
+               $self->{midx}->index_ibx($ibx) unless $sync->{quit};
+       }
+       $ibx->git->cleanup; # done with this inbox, now
 }
 
 sub eidx_sync { # main entry point
@@ -329,13 +334,18 @@ sub eidx_sync { # main entry point
                -regen_fmt => "%u/?\n",
        };
        local $SIG{USR1} = sub { $need_checkpoint = 1 };
+       my $quit = sub { $sync->{quit} = 1; warn "gracefully quitting\n"; };
+       local $SIG{QUIT} = $quit;
+       local $SIG{INT} = $quit;
+       local $SIG{TERM} = $quit;
 
        # don't use $_ here, it'll get clobbered by reindex_checkpoint
        for my $ibx (@{$self->{ibx_list}}) {
                _sync_inbox($self, $sync, $ibx);
+               last if $sync->{quit};
        }
 
-       $self->{oidx}->rethread_done($opt);
+       $self->{oidx}->rethread_done($opt) unless $sync->{quit};
 
        PublicInbox::V2Writable::done($self);
 }
@@ -369,6 +379,12 @@ sub update_last_commit { # overrides V2Writable
        $self->{oidx}->eidx_meta($meta_key, $latest_cmt);
 }
 
+sub _idx_init { # with_umask callback
+       my ($self, $opt) = @_;
+       PublicInbox::V2Writable::_idx_init($self, $opt);
+       $self->{midx} = PublicInbox::MiscIdx->new($self);
+}
+
 sub idx_init { # similar to V2Writable
        my ($self, $opt) = @_;
        return if $self->{idx_shards};
@@ -380,18 +396,28 @@ sub idx_init { # similar to V2Writable
        my $info_dir = "$ALL/objects/info";
        my $alt = "$info_dir/alternates";
        my $mode = 0644;
-       my (%old, @old, %new, @new);
+       my (@old, @new, %seen); # seen: st_dev + st_ino
        if (-e $alt) {
                open(my $fh, '<', $alt) or die "open $alt: $!";
                $mode = (stat($fh))[2] & 07777;
-               while (<$fh>) {
-                       push @old, $_ if !$old{$_}++;
+               while (my $line = <$fh>) {
+                       chomp(my $d = $line);
+                       if (my @st = stat($d)) {
+                               next if $seen{"$st[0]\0$st[1]"}++;
+                       } else {
+                               warn "W: stat($d) failed (from $alt): $!\n";
+                       }
+                       push @old, $line;
                }
        }
        for my $ibx (@{$self->{ibx_list}}) {
                my $line = $ibx->git->{git_dir} . "/objects\n";
-               next if $old{$line};
-               $new{$line} = 1;
+               chomp(my $d = $line);
+               if (my @st = stat($d)) {
+                       next if $seen{"$st[0]\0$st[1]"}++;
+               } else {
+                       warn "W: stat($d) failed (from $ibx->{inboxdir}): $!\n";
+               }
                push @new, $line;
        }
        if (scalar @new) {
@@ -401,9 +427,10 @@ sub idx_init { # similar to V2Writable
        }
        $self->parallel_init($self->{indexlevel});
        $self->umask_prepare;
-       $self->with_umask(\&PublicInbox::V2Writable::_idx_init, $self, $opt);
+       $self->with_umask(\&_idx_init, $self, $opt);
        $self->{oidx}->begin_lazy;
        $self->{oidx}->eidx_prep;
+       $self->{midx}->begin_txn;
 }
 
 no warnings 'once';