imap+nntp: share COMPRESS implementation

[public-inbox.git] / lib / PublicInbox / Admin.pm

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index d414e4e205d95167b5a2cad241936fc12cd55d6b..11ea8f8307c2082ec974d51763df3f5f2983fa8c 100644 (file)
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # common stuff for administrative command-line tools
@@ -10,6 +10,7 @@ our @EXPORT_OK = qw(setup_signals);
 use PublicInbox::Config;
 use PublicInbox::Inbox;
 use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Eml;
 *rel2abs_collapsed = \&PublicInbox::Config::rel2abs_collapsed;
 
 sub setup_signals {
@@ -27,6 +28,27 @@ sub setup_signals {
        };
 }
 
+sub resolve_eidxdir {
+       my ($cd) = @_;
+       my $try = $cd // '.';
+       my $root_dev_ino;
+       while (1) { # favor v2, first
+               if (-f "$try/ei.lock") {
+                       return rel2abs_collapsed($try);
+               } elsif (-d $try) {
+                       my @try = stat _;
+                       $root_dev_ino //= do {
+                               my @root = stat('/') or die "stat /: $!\n";
+                               "$root[0]\0$root[1]";
+                       };
+                       return undef if "$try[0]\0$try[1]" eq $root_dev_ino;
+                       $try .= '/..'; # continue, cd up
+               } else {
+                       die "`$try' is not a directory\n";
+               }
+       }
+}
+
 sub resolve_inboxdir {
        my ($cd, $ver) = @_;
        my $try = $cd // '.';
@@ -106,11 +128,24 @@ sub resolve_inboxes ($;$$) {
                $cfg or die "--all specified, but $cfgfile not readable\n";
                @$argv and die "--all specified, but directories specified\n";
        }
-
+       my (@old, @ibxs, @eidx);
+       if ($opt->{-eidx_ok}) {
+               require PublicInbox::ExtSearchIdx;
+               my $i = -1;
+               @$argv = grep {
+                       $i++;
+                       if (defined(my $ei = resolve_eidxdir($_))) {
+                               $ei = PublicInbox::ExtSearchIdx->new($ei, $opt);
+                               push @eidx, $ei;
+                               undef;
+                       } else {
+                               1;
+                       }
+               } @$argv;
+       }
        my $min_ver = $opt->{-min_inbox_version} || 0;
        # lookup inboxes by st_dev + st_ino instead of {inboxdir} pathnames,
        # pathnames are not unique due to symlinks and bind mounts
-       my (@old, @ibxs);
        if ($opt->{all}) {
                $cfg->each_inbox(sub {
                        my ($ibx) = @_;
@@ -160,11 +195,10 @@ sub resolve_inboxes ($;$$) {
                die "-V$min_ver inboxes not supported by $0\n\t",
                    join("\n\t", @old), "\n";
        }
-       @ibxs;
+       $opt->{-eidx_ok} ? (\@ibxs, \@eidx) : @ibxs;
 }
 
-# TODO: make Devel::Peek optional, only used for daemon
-my @base_mod = qw(Devel::Peek);
+my @base_mod = ();
 my @over_mod = qw(DBD::SQLite DBI);
 my %mod_groups = (
        -index => [ @base_mod, @over_mod ],
@@ -239,16 +273,14 @@ sub index_inbox {
        if (my $pr = $opt->{-progress}) {
                $pr->("indexing $ibx->{inboxdir} ...\n");
        }
-       local %SIG = %SIG;
+       local @SIG{keys %SIG} = values %SIG;
        setup_signals(\&index_terminate, $ibx);
-       my $warn_cb = $SIG{__WARN__} // \&CORE::warn;
        my $idx = { current_info => $ibx->{inboxdir} };
-       my $warn_ignore = PublicInbox::InboxWritable->can('warn_ignore');
        local $SIG{__WARN__} = sub {
-               return if $warn_ignore->(@_);
-               $warn_cb->($idx->{current_info}, ': ', @_);
+               return if PublicInbox::Eml::warn_ignore(@_);
+               warn($idx->{current_info}, ': ', @_);
        };
-       if (ref($ibx) && $ibx->version == 2) {
+       if ($ibx->version == 2) {
                eval { require PublicInbox::V2Writable };
                die "v2 requirements not met: $@\n" if $@;
                $ibx->{-creat_opt}->{nproc} = $jobs;
@@ -274,8 +306,8 @@ EOM
        $idx->{nidx} // 0; # returns number processed
 }
 
-sub progress_prepare ($) {
-       my ($opt) = @_;
+sub progress_prepare ($;$) {
+       my ($opt, $dst) = @_;
 
        # public-inbox-index defaults to quiet, -xcpdb and -compact do not
        if (defined($opt->{quiet}) && $opt->{quiet} < 0) {
@@ -287,7 +319,8 @@ sub progress_prepare ($) {
                $opt->{1} = $null; # suitable for spawn() redirect
        } else {
                $opt->{verbose} ||= 1;
-               $opt->{-progress} = sub { print STDERR @_ };
+               $dst //= *STDERR{GLOB};
+               $opt->{-progress} = sub { print $dst '# ', @_ };
        }
 }
 
@@ -327,15 +360,27 @@ sub index_prepare ($$) {
        $opt->{batch_size} and
                $env = { XAPIAN_FLUSH_THRESHOLD => '4294967295' };
 
-       for my $k (qw(sequential_shard)) {
+       for my $k (qw(sequential-shard)) {
                my $git_key = "publicInbox.index".ucfirst($k);
-               $git_key =~ s/_([a-z])/\U$1/g;
+               $git_key =~ s/-([a-z])/\U$1/g;
                defined(my $s = $opt->{$k} // $cfg->{lc($git_key)}) or next;
                defined(my $v = $cfg->git_bool($s))
                                        or die "`$git_key=$s' not boolean\n";
                $opt->{$k} = $v;
        }
+       for my $k (qw(since until)) {
+               my $v = $opt->{$k} // next;
+               $opt->{reindex} or die "--$k=$v requires --reindex\n";
+       }
        $env;
 }
 
+sub do_chdir ($) {
+       my $chdir = $_[0] // return;
+       for my $d (@$chdir) {
+               next if $d eq ''; # same as git(1)
+               chdir $d or die "cd $d: $!";
+       }
+}
+
 1;