update copyrights for 2021

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 53ff2ca183656e5fd1a6dd72b58db9e46096691d..c358162889760cde1f4ecf6089918b325ba58329 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Detached/external index cross inbox search indexing support
@@ -881,9 +881,11 @@ sub eidx_sync { # main entry point
        }
 
        # don't use $_ here, it'll get clobbered by reindex_checkpoint
-       for my $ibx (@{$self->{ibx_list}}) {
-               last if $sync->{quit};
-               sync_inbox($self, $sync, $ibx);
+       if ($opt->{scan} // 1) {
+               for my $ibx (@{$self->{ibx_list}}) {
+                       last if $sync->{quit};
+                       sync_inbox($self, $sync, $ibx);
+               }
        }
        $self->{oidx}->rethread_done($opt) unless $sync->{quit};
        eidxq_process($self, $sync) unless $sync->{quit};
@@ -933,18 +935,31 @@ sub idx_init { # similar to V2Writable
        return if $self->{idx_shards};
 
        $self->git->cleanup;
-
+       my $mode = 0644;
        my $ALL = $self->git->{git_dir}; # ALL.git
-       PublicInbox::Import::init_bare($ALL) unless -d $ALL;
+       my $old = -d $ALL;
+       if ($opt->{-private}) { # LeiStore
+               $mode = 0600;
+               if (!$old) {
+                       umask 077; # don't bother restoring
+                       PublicInbox::Import::init_bare($ALL);
+                       $self->git->qx(qw(config core.sharedRepository 0600));
+               }
+       } else {
+               PublicInbox::Import::init_bare($ALL) unless $old;
+       }
        my $info_dir = "$ALL/objects/info";
        my $alt = "$info_dir/alternates";
-       my $mode = 0644;
        my (@old, @new, %seen); # seen: st_dev + st_ino
        if (-e $alt) {
                open(my $fh, '<', $alt) or die "open $alt: $!";
                $mode = (stat($fh))[2] & 07777;
                while (my $line = <$fh>) {
                        chomp(my $d = $line);
+
+                       # expand relative path (/local/ stuff)
+                       substr($d, 0, 3) eq '../' and
+                               $d = "$ALL/objects/$d";
                        if (my @st = stat($d)) {
                                next if $seen{"$st[0]\0$st[1]"}++;
                        } else {
@@ -954,6 +969,22 @@ sub idx_init { # similar to V2Writable
                        push @old, $line;
                }
        }
+
+       # for LeiStore, and possibly some mirror-only state
+       if (opendir(my $dh, my $local = "$self->{topdir}/local")) {
+               # highest numbered epoch first
+               for my $n (sort { $b <=> $a } map { substr($_, 0, -4) + 0 }
+                               grep(/\A[0-9]+\.git\z/, readdir($dh))) {
+                       my $d = "$local/$n.git/objects"; # absolute path
+                       if (my @st = stat($d)) {
+                               next if $seen{"$st[0]\0$st[1]"}++;
+                               # favor relative paths for rename-friendliness
+                               push @new, "../../local/$n.git/objects\n";
+                       } else {
+                               warn "W: stat($d) failed: $!\n";
+                       }
+               }
+       }
        for my $ibx (@{$self->{ibx_list}}) {
                my $line = $ibx->git->{git_dir} . "/objects\n";
                chomp(my $d = $line);
@@ -1008,6 +1039,7 @@ sub eidx_reload { # -extindex --watch SIGHUP handler
        if ($self->{cfg}) {
                my $pr = $self->{-watch_sync}->{-opt}->{-progress};
                $pr->('reloading ...') if $pr;
+               delete $self->{-resync_queue};
                @{$self->{ibx_list}} = ();
                %{$self->{ibx_map}} = ();
                delete $self->{-watch_sync}->{id2pos};
@@ -1043,6 +1075,10 @@ sub event_step { # PublicInbox::DS::requeue callback
 
 sub eidx_watch { # public-inbox-extindex --watch main loop
        my ($self, $opt) = @_;
+       local %SIG = %SIG;
+       for my $sig (qw(HUP USR1 TSTP QUIT INT TERM)) {
+               $SIG{$sig} = sub { warn "SIG$sig ignored while scanning\n" };
+       }
        require PublicInbox::InboxIdle;
        require PublicInbox::DS;
        require PublicInbox::Syscall;
@@ -1052,6 +1088,8 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
                $idler->watch_inbox($_) for @{$self->{ibx_list}};
        }
        $_->subscribe_unlock(__PACKAGE__, $self) for @{$self->{ibx_list}};
+       my $pr = $opt->{-progress};
+       $pr->("performing initial scan ...\n") if $pr;
        my $sync = eidx_sync($self, $opt); # initial sync
        return if $sync->{quit};
        my $oldset = PublicInbox::Sigfd::block_signals();
@@ -1067,7 +1105,7 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
        $sig->{QUIT} = $sig->{INT} = $sig->{TERM} = $quit;
        my $sigfd = PublicInbox::Sigfd->new($sig,
                                        $PublicInbox::Syscall::SFD_NONBLOCK);
-       local %SIG = (%SIG, %$sig) if !$sigfd;
+       %SIG = (%SIG, %$sig) if !$sigfd;
        local $self->{-watch_sync} = $sync; # for ->on_inbox_unlock
        if (!$sigfd) {
                # wake up every second to accept signals if we don't
@@ -1076,6 +1114,7 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
                PublicInbox::DS->SetLoopTimeout(1000);
        }
        PublicInbox::DS->SetPostLoopCallback(sub { !$sync->{quit} });
+       $pr->("initial scan complete, entering event loop\n") if $pr;
        PublicInbox::DS->EventLoop; # calls InboxIdle->event_step
        done($self);
 }