lei: hexdigest mocks account for unwanted headers

PublicInbox::Import never imports @UNWANTED_HEADERS, so ensure
our mock blob OIDs do the same.  This ought to prevent
duplicates if the PSGI mboxrd download starts setting
"X-Status: F" like "lei q -tt .."

diff --git a/lib/PublicInbox/FakeImport.pm b/lib/PublicInbox/FakeImport.pm
index dea25cbe595b39534f2161142b069201a2640509..bccc3321a57be52ff7e5e9c1972319ea1882887a 100644 (file)
--- a/lib/PublicInbox/FakeImport.pm
+++ b/lib/PublicInbox/FakeImport.pm
@@ -4,12 +4,15 @@
 # pretend to do PublicInbox::Import::add for "lei index"
 package PublicInbox::FakeImport;
 use strict;
+use v5.10.1;
 use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
 
 sub new { bless { bytes_added => 0 }, __PACKAGE__ }
 
 sub add {
        my ($self, $eml, $check_cb, $smsg) = @_;
+       PublicInbox::Import::drop_unwanted_headers($eml);
        $smsg->populate($eml);
        my $raw = $eml->as_string;
        $smsg->{blob} = git_sha(1, \$raw)->hexdigest;

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7d0f63dca4a7b2dcbb519cdd9b32e9a7a237150a..347dd2806365340ef699d3489a4615cec166c023 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1420,4 +1420,9 @@ sub refresh_watches {
        }
 }
 
+sub git_blob_id {
+       my ($lei, $eml) = @_;
+       ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+}
+
 1;

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 37b660f9ce1ca46a31e5c9ed4a7230be16398f35..962ad49ec9e0bb748c508f7edfedd6db4d828464 100644 (file)
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -73,7 +73,7 @@ sub lxs_prepare {
        my @only = @{$opt->{only} // []};
        # --local is enabled by default unless --only is used
        # we'll allow "--only $LOCATION --local"
-       my $sto = $self->_lei_store(1); # FIXME: should not create
+       my $sto = $self->_lei_store(1);
        $self->{lse} = $sto->search;
        if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
                $lxs->prepare_external($self->{lse});

diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index e7deecb8b7c3847f717349b3935f4f788b6b621a..580787c042c4e317d7c79ddab425c05b0816d2f3 100644 (file)
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
                $smsg = $res if ref($res) eq ref($smsg);
        }
        $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
-                               : git_sha(1, $eml)->hexdigest;
+                               : $lei->git_blob_id($eml);
        $smsg->populate($eml);
        $smsg->{mid} //= '(none)';
        push @{$self->{smsg}}, $smsg;

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index ce66014fcb31fa2209e13230449e66b4b9b76144..3f33d114a6c7e85d5a3513696f2c86e00fffdfd1 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
 use PublicInbox::Import;
 use PublicInbox::InboxWritable qw(eml_from_path);
 use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
 use PublicInbox::MID qw(mids);
 use PublicInbox::LeiSearch;
 use PublicInbox::MDA;
@@ -508,4 +508,11 @@ sub write_prepare {
        $lei->{sto} = $self;
 }
 
+# TODO: support SHA-256
+sub git_blob_id { # called via LEI->git_blob_id
+       my ($self, $eml) = @_;
+       $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+       git_sha(1, $eml)->hexdigest;
+}
+
 1;

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 971f3a06a7fd49442582e509b6709c957bf18e7d..5e34d864e5844456ace1a61a4d40125dbc1dd25c 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -274,8 +274,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
                        $smsg->{kw} = []; # short-circuit xsmsg_vmd
                }
        }
-       $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
-                               : git_sha(1, $eml)->hexdigest;
+       $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
        _smsg_fill($smsg, $eml);
        wait_startq($lei);
        if ($lei->{-progress}) {