lib/PublicInbox/LeiDedupe.pm | 16 ++++++++-------- lib/PublicInbox/LeiToMail.pm | 6 +++--- t/lei_dedupe.t | 11 +++++++---- diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm index a62b3a7ccd6c737773f0bb3e6c82bcab0a7e4e51..378f748e5c4a7e46794af0e4bbebe7c5c323f36a 100644 --- a/lib/PublicInbox/LeiDedupe.pm +++ b/lib/PublicInbox/LeiDedupe.pm @@ -41,8 +41,8 @@ # the paranoid option sub dedupe_oid ($) { my ($skv) = @_; (sub { # may be called in a child process - my ($eml, $oid) = @_; - $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), ''); + my ($eml, $oidhex) = @_; + $skv->set_maybe(_oidbin($oidhex) // _regen_oid($eml), ''); }, sub { my ($smsg) = @_; $skv->set_maybe(_oidbin($smsg->{blob}), ''); @@ -53,9 +53,9 @@ # dangerous if there's duplicate messages with different Message-IDs sub dedupe_mid ($) { my ($skv) = @_; (sub { # may be called in a child process - my ($eml, $oid) = @_; - # TODO: lei will support non-public messages w/o Message-ID - my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) // + my ($eml, $oidhex) = @_; + # lei supports non-public drafts w/o Message-ID + my $mid = $eml->header_raw('Message-ID') // _oidbin($oidhex) // content_hash($eml); $skv->set_maybe($mid, ''); }, sub { @@ -71,7 +71,7 @@ # our default deduplication strategy (used by v2, also) sub dedupe_content ($) { my ($skv) = @_; (sub { # may be called in a child process - my ($eml) = @_; # oid = $_[1], ignored + my ($eml) = @_; # $oidhex = $_[1], ignored $skv->set_maybe(content_hash($eml), ''); }, sub { my ($smsg) = @_; @@ -104,8 +104,8 @@ # returns true on seen messages according to the deduplication strategy, # returns false if unseen sub is_dup { - my ($self, $eml, $oid) = @_; - !$self->[1]->($eml, $oid); + my ($self, $eml, $smsg) = @_; + !$self->[1]->($eml, $smsg ? $smsg->{blob} : undef); } sub is_smsg_dup { diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index 70164e40b672417fa01fcac159b69d151ae062d5..7adbffe7ac3aba411d132131cb00a0c13a762820 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -194,7 +194,7 @@ my $lse = $lei->{lse}; # may be undef sub { # for git_to_mail my ($buf, $smsg, $eml) = @_; $eml //= PublicInbox::Eml->new($buf); - return if $dedupe->is_dup($eml, $smsg->{blob}); + return if $dedupe->is_dup($eml, $smsg); $lse->xsmsg_vmd($smsg) if $lse; $buf = $eml2mbox->($eml, $smsg); return atomic_append($lei, $buf) if $atomic_append; @@ -280,7 +280,7 @@ $buf //= \($eml->as_string); $lse->xsmsg_vmd($smsg) if $lse; return _buf2maildir($dst, $buf, $smsg) if !$dedupe; $eml //= PublicInbox::Eml->new($$buf); # copy buf - return if $dedupe->is_dup($eml, $smsg->{blob}); + return if $dedupe->is_dup($eml, $smsg); undef $eml; _buf2maildir($dst, $buf, $smsg); } @@ -299,7 +299,7 @@ my ($bref, $smsg, $eml) = @_; $mic // return $lei->fail; # mic may be undef-ed in last run if ($dedupe) { $eml //= PublicInbox::Eml->new($$bref); # copy bref - return if $dedupe->is_dup($eml, $smsg->{blob}); + return if $dedupe->is_dup($eml, $smsg); } $lse->xsmsg_vmd($smsg) if $lse; eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) }; diff --git a/t/lei_dedupe.t b/t/lei_dedupe.t index bcb06a0a3c36055ef6b91fa637483f060aeecb9d..e1944d025b5fb56f26f845de1af971926039cf19 100644 --- a/t/lei_dedupe.t +++ b/t/lei_dedupe.t @@ -74,10 +74,13 @@ ok(!$dd->is_dup($different), 'different is_dup with mid dedupe (augment)'); $different->header_set('Status', 'RO'); ok($dd->is_dup($different), 'different seen with oid dedupe Status removed'); -ok(!$dd->is_dup($eml, '01d'), '1st is_dup with oid dedupe'); -ok($dd->is_dup($different, '01d'), 'different content ignored if oid matches'); -ok($dd->is_dup($eml, '01D'), 'case insensitive oid comparison :P'); -ok(!$dd->is_dup($eml, '01dbad'), 'case insensitive oid comparison :P'); +$smsg = { blob => '01d' }; +ok(!$dd->is_dup($eml, $smsg), '1st is_dup with oid dedupe'); +ok($dd->is_dup($different, $smsg), 'different content ignored if oid matches'); +$smsg->{blob} = uc($smsg->{blob}); +ok($dd->is_dup($eml, $smsg), 'case insensitive oid comparison :P'); +$smsg->{blob} = '01dbad'; +ok(!$dd->is_dup($eml, $smsg), 'case insensitive oid comparison :P'); $smsg->{blob} = 'dead'; ok(!$dd->is_smsg_dup($smsg), 'smsg dedupe pass');