lib/PublicInbox/LeiDedupe.pm | 29 ++++++++++++++++++++++++++++- lib/PublicInbox/LeiQuery.pm | 5 +++++ t/lei_dedupe.t | 14 ++++++++++++++ diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm index c4e5dffbad3be62316ee759fd92d72718932691b..58eee53314777ecac2801762fd52e3bb274416b3 100644 --- a/lib/PublicInbox/LeiDedupe.pm +++ b/lib/PublicInbox/LeiDedupe.pm @@ -33,12 +33,24 @@ } sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef } +sub smsg_hash ($) { + my ($smsg) = @_; + my $dig = Digest::SHA->new(256); + my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)}); + utf8::encode($x); + $dig->add($x); + $dig->digest; +} + # the paranoid option sub dedupe_oid () { my $skv = PublicInbox::SharedKV->new; ($skv, sub { # may be called in a child process my ($eml, $oid) = @_; $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), ''); + }, sub { + my ($smsg) = @_; + $skv->set_maybe(_oidbin($smsg->{blob}), ''); }); } @@ -51,6 +63,12 @@ # TODO: lei will support non-public messages w/o Message-ID my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) // content_hash($eml); $skv->set_maybe($mid, ''); + }, sub { + my ($smsg) = @_; + my $mid = $smsg->{mid}; + $mid = undef if $mid eq ''; + $mid //= smsg_hash($smsg) // _oidbin($smsg->{blob}); + $skv->set_maybe($mid, ''); }); } @@ -60,11 +78,15 @@ my $skv = PublicInbox::SharedKV->new; ($skv, sub { # may be called in a child process my ($eml) = @_; # oid = $_[1], ignored $skv->set_maybe(content_hash($eml), ''); + }, sub { + my ($smsg) = @_; + $skv->set_maybe(smsg_hash($smsg), ''); }); } # no deduplication at all -sub dedupe_none () { (undef, sub { 1 }) } +sub true { 1 } +sub dedupe_none () { (undef, \&true, \&true) } sub new { my ($cls, $lei, $dst) = @_; @@ -83,6 +105,11 @@ # returns false if seen sub is_dup { my ($self, $eml, $oid) = @_; !$self->[1]->($eml, $oid); +} + +sub is_smsg_dup { + my ($self, $smsg) = @_; + !$self->[2]->($smsg); } sub prepare_dedupe { diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index d14da1bc4f17c0ea2241d7a6ca75ec8774509172..f69dccadbd5b167357e849cc6936f2e8d2477f13 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -69,6 +69,8 @@ /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_ } @argv); $opt->{limit} //= 10000; my $lxs; + require PublicInbox::LeiDedupe; + my $dd = PublicInbox::LeiDedupe->new($self); # --local is enabled by default my @src = $opt->{'local'} ? ($sto->search) : (); @@ -135,6 +137,7 @@ my ($smsg) = @_; delete @$smsg{qw(tid num)}; # only makes sense if single src chomp($buf = $json->encode(_smsg_unbless($smsg))); }; + $dd->prepare_dedupe; for my $src (@src) { my $srch = $src->search; my $over = $src->over; @@ -145,6 +148,7 @@ my $ctx = {}; if ($smsg_for) { for my $it ($mset->items) { my $smsg = $smsg_for->($srch, $it) or next; + next if $dd->is_smsg_dup($smsg); $self->out($buf .= $ORS) if defined $buf; $smsg->{relevance} = get_pct($it); $emit_cb->($smsg); @@ -160,6 +164,7 @@ undef $mset; while ($over && $over->expand_thread($ctx)) { for my $n (@{$ctx->{xids}}) { my $t = $over->get_art($n) or next; + next if $dd->is_smsg_dup($t); if (my $p = delete $n2p{$t->{num}}) { $t->{relevance} = $p; } diff --git a/t/lei_dedupe.t b/t/lei_dedupe.t index b5e2b8f9db651a1fdc7697e54f78b7e21bb30906..6e971b9b07d319d51a98d2badb7934f885374882 100644 --- a/t/lei_dedupe.t +++ b/t/lei_dedupe.t @@ -6,12 +6,16 @@ use v5.10.1; use Test::More; use PublicInbox::TestCommon; use PublicInbox::Eml; +use PublicInbox::Smsg; require_mods(qw(DBD::SQLite)); use_ok 'PublicInbox::LeiDedupe'; my $eml = eml_load('t/plack-qp.eml'); my $mid = $eml->header_raw('Message-ID'); my $different = eml_load('t/msg_iter-order.eml'); $different->header_set('Message-ID', $mid); +my $smsg = bless { ds => time }, 'PublicInbox::Smsg'; +$smsg->populate($eml); +$smsg->{$_} //= '' for (qw(to cc references)) ; my $lei = { opt => { dedupe => 'none' } }; my $dd = PublicInbox::LeiDedupe->new($lei); @@ -19,6 +23,8 @@ $dd->prepare_dedupe; ok(!$dd->is_dup($eml), '1st is_dup w/o dedupe'); ok(!$dd->is_dup($eml), '2nd is_dup w/o dedupe'); ok(!$dd->is_dup($different), 'different is_dup w/o dedupe'); +ok(!$dd->is_smsg_dup($smsg), 'smsg dedupe none 1'); +ok(!$dd->is_smsg_dup($smsg), 'smsg dedupe none 2'); for my $strat (undef, 'content') { $lei->{opt}->{dedupe} = $strat; @@ -28,6 +34,8 @@ my $desc = $strat // 'default'; ok(!$dd->is_dup($eml), "1st is_dup with $desc dedupe"); ok($dd->is_dup($eml), "2nd seen with $desc dedupe"); ok(!$dd->is_dup($different), "different is_dup with $desc dedupe"); + ok(!$dd->is_smsg_dup($smsg), "is_smsg_dup pass w/ $desc dedupe"); + ok($dd->is_smsg_dup($smsg), "is_smsg_dup reject w/ $desc dedupe"); } $lei->{opt}->{dedupe} = 'bogus'; eval { PublicInbox::LeiDedupe->new($lei) }; @@ -39,6 +47,8 @@ $dd->prepare_dedupe; ok(!$dd->is_dup($eml), '1st is_dup with mid dedupe'); ok($dd->is_dup($eml), '2nd seen with mid dedupe'); ok($dd->is_dup($different), 'different seen with mid dedupe'); +ok(!$dd->is_smsg_dup($smsg), 'smsg mid dedupe pass'); +ok($dd->is_smsg_dup($smsg), 'smsg mid dedupe reject'); $lei->{opt}->{dedupe} = 'oid'; $dd = PublicInbox::LeiDedupe->new($lei); @@ -55,5 +65,9 @@ ok(!$dd->is_dup($eml, '01d'), '1st is_dup with oid dedupe'); ok($dd->is_dup($different, '01d'), 'different content ignored if oid matches'); ok($dd->is_dup($eml, '01D'), 'case insensitive oid comparison :P'); ok(!$dd->is_dup($eml, '01dbad'), 'case insensitive oid comparison :P'); + +$smsg->{blob} = 'dead'; +ok(!$dd->is_smsg_dup($smsg), 'smsg dedupe pass'); +ok($dd->is_smsg_dup($smsg), 'smsg dedupe reject'); done_testing;