No ext_urls

[public-inbox.git] / lib / PublicInbox / LeiDedupe.pm

diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index a62b3a7ccd6c737773f0bb3e6c82bcab0a7e4e51..22864508b84e180adbadc262159bcb9d120b317c 100644 (file)
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::LeiDedupe;
 use strict;
 use v5.10.1;
 use PublicInbox::ContentHash qw(content_hash git_sha);
-use Digest::SHA ();
+use PublicInbox::SHA ();
 
 # n.b. mutt sets most of these headers not sure about Bytes
 our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
@@ -30,7 +30,7 @@ sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef }
 
 sub smsg_hash ($) {
        my ($smsg) = @_;
-       my $dig = Digest::SHA->new(256);
+       my $dig = PublicInbox::SHA->new(256);
        my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)});
        utf8::encode($x);
        $dig->add($x);
@@ -41,8 +41,8 @@ sub smsg_hash ($) {
 sub dedupe_oid ($) {
        my ($skv) = @_;
        (sub { # may be called in a child process
-               my ($eml, $oid) = @_;
-               $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), '');
+               my ($eml, $oidhex) = @_;
+               $skv->set_maybe(_oidbin($oidhex) // _regen_oid($eml), '');
        }, sub {
                my ($smsg) = @_;
                $skv->set_maybe(_oidbin($smsg->{blob}), '');
@@ -53,9 +53,9 @@ sub dedupe_oid ($) {
 sub dedupe_mid ($) {
        my ($skv) = @_;
        (sub { # may be called in a child process
-               my ($eml, $oid) = @_;
-               # TODO: lei will support non-public messages w/o Message-ID
-               my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) //
+               my ($eml, $oidhex) = @_;
+               # lei supports non-public drafts w/o Message-ID
+               my $mid = $eml->header_raw('Message-ID') // _oidbin($oidhex) //
                        content_hash($eml);
                $skv->set_maybe($mid, '');
        }, sub {
@@ -71,7 +71,7 @@ sub dedupe_mid ($) {
 sub dedupe_content ($) {
        my ($skv) = @_;
        (sub { # may be called in a child process
-               my ($eml) = @_; # oid = $_[1], ignored
+               my ($eml) = @_; # $oidhex = $_[1], ignored
                $skv->set_maybe(content_hash($eml), '');
        }, sub {
                my ($smsg) = @_;
@@ -104,8 +104,8 @@ sub new {
 # returns true on seen messages according to the deduplication strategy,
 # returns false if unseen
 sub is_dup {
-       my ($self, $eml, $oid) = @_;
-       !$self->[1]->($eml, $oid);
+       my ($self, $eml, $smsg) = @_;
+       !$self->[1]->($eml, $smsg ? $smsg->{blob} : undef);
 }
 
 sub is_smsg_dup {
@@ -127,4 +127,9 @@ sub pause_dedupe {
        delete($skv->{dbh}) if $skv;
 }
 
+sub has_entries {
+       my $skv = $_[0]->[0] or return undef;
+       $skv->has_entries;
+}
+
 1;