v2writable: make remove return-compatible w/ Import::remove

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 72d8d5af6641aac8e0080132a92d26a11642c173..573a92aacdf14b2ae349353b25a4d33399236d10 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
+# Copyright (C) 2018-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # This interface wraps and mimics PublicInbox::Import
@@ -7,6 +7,7 @@ package PublicInbox::V2Writable;
 use strict;
 use warnings;
 use base qw(PublicInbox::Lock);
+use 5.010_001;
 use PublicInbox::SearchIdxShard;
 use PublicInbox::MIME;
 use PublicInbox::Git;
@@ -32,14 +33,29 @@ my $PACKING_FACTOR = 0.4;
 # to increase Xapian shards
 our $NPROC_MAX_DEFAULT = 4;
 
+sub detect_nproc () {
+       for my $nproc (qw(nproc gnproc)) { # GNU coreutils nproc
+               `$nproc 2>/dev/null` =~ /^(\d+)$/ and return $1;
+       }
+
+       # getconf(1) is POSIX, but *NPROCESSORS* vars are not
+       for (qw(_NPROCESSORS_ONLN NPROCESSORS_ONLN)) {
+               `getconf $_ 2>/dev/null` =~ /^(\d+)$/ and return $1;
+       }
+
+       # should we bother with `sysctl hw.ncpu`?  Those only give
+       # us total processor count, not online processor count.
+       undef
+}
+
 sub nproc_shards ($) {
        my ($creat_opt) = @_;
        my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH';
        $n //= $ENV{NPROC};
        if (!$n) {
-               chomp($n = `nproc 2>/dev/null`);
-               # assume 2 cores if GNU nproc(1) is not available
-               $n = 2 if !$n;
+               # assume 2 cores if not detectable or zero
+               state $NPROC_DETECTED = detect_nproc() || 2;
+               $n = $NPROC_DETECTED;
                $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT;
        }
 
@@ -365,7 +381,7 @@ sub rewrite_internal ($$;$$$) {
        }
        my $over = $self->{over};
        my $cids = content_ids($old_mime);
-       my $removed;
+       my @removed;
        my $mids = mids($old_mime->header_obj);
 
        # We avoid introducing new blobs into git since the raw content
@@ -375,7 +391,7 @@ sub rewrite_internal ($$;$$$) {
        my $mark;
 
        foreach my $mid (@$mids) {
-               my %gone; # num => [ smsg, raw ]
+               my %gone; # num => [ smsg, $mime, raw ]
                my ($id, $prev);
                while (my $smsg = $over->next_by_mid($mid, \$id, \$prev)) {
                        my $msg = get_blob($self, $smsg);
@@ -386,8 +402,7 @@ sub rewrite_internal ($$;$$$) {
                        my $orig = $$msg;
                        my $cur = PublicInbox::MIME->new($msg);
                        if (content_matches($cids, $cur)) {
-                               $smsg->{mime} = $cur;
-                               $gone{$smsg->{num}} = [ $smsg, \$orig ];
+                               $gone{$smsg->{num}} = [ $smsg, $cur, \$orig ];
                        }
                }
                my $n = scalar keys %gone;
@@ -397,15 +412,16 @@ sub rewrite_internal ($$;$$$) {
                                join(',', sort keys %gone), "\n";
                }
                foreach my $num (keys %gone) {
-                       my ($smsg, $orig) = @{$gone{$num}};
-                       # $removed should only be set once assuming
+                       my ($smsg, $mime, $orig) = @{$gone{$num}};
+                       # @removed should only be set once assuming
                        # no bugs in our deduplication code:
-                       $removed = $smsg;
+                       @removed = (undef, $mime, $smsg);
                        my $oid = $smsg->{blob};
                        if ($replace_map) {
                                $replace_map->{$oid} = $sref;
                        } else {
                                ($mark, undef) = $im->remove($orig, $cmt_msg);
+                               $removed[0] = $mark;
                        }
                        $orig = undef;
                        if ($need_reindex) { # ->replace
@@ -425,15 +441,18 @@ sub rewrite_internal ($$;$$$) {
                my $rewrites = _replace_oids($self, $new_mime, $replace_map);
                return { rewrites => $rewrites, need_reindex => $need_reindex };
        }
-       $removed;
+       defined($mark) ? @removed : undef;
 }
 
-# public
+# public (see PublicInbox::Import->remove), but note the 3rd element
+# (retval[2]) is not part of the stable API shared with Import->remove
 sub remove {
        my ($self, $mime, $cmt_msg) = @_;
+       my @ret;
        $self->{-inbox}->with_umask(sub {
-               rewrite_internal($self, $mime, $cmt_msg);
+               @ret = rewrite_internal($self, $mime, $cmt_msg);
        });
+       defined($ret[0]) ? @ret : undef;
 }
 
 sub _replace ($$;$$) {