www: improve visibility of coderepos

[public-inbox.git] / lib / PublicInbox / LeiDedupe.pm

diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index c4e5dffbad3be62316ee759fd92d72718932691b..5fec9384f1e305cae739e17562377f6377504f61 100644 (file)
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -3,8 +3,8 @@
 package PublicInbox::LeiDedupe;
 use strict;
 use v5.10.1;
-use PublicInbox::SharedKV;
 use PublicInbox::ContentHash qw(content_hash);
+use Digest::SHA ();
 
 # n.b. mutt sets most of these headers not sure about Bytes
 our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
@@ -33,67 +33,102 @@ sub _regen_oid ($) {
 
 sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef }
 
+sub smsg_hash ($) {
+       my ($smsg) = @_;
+       my $dig = Digest::SHA->new(256);
+       my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)});
+       utf8::encode($x);
+       $dig->add($x);
+       $dig->digest;
+}
+
 # the paranoid option
-sub dedupe_oid () {
-       my $skv = PublicInbox::SharedKV->new;
-       ($skv, sub { # may be called in a child process
+sub dedupe_oid ($) {
+       my ($skv) = @_;
+       (sub { # may be called in a child process
                my ($eml, $oid) = @_;
                $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), '');
+       }, sub {
+               my ($smsg) = @_;
+               $skv->set_maybe(_oidbin($smsg->{blob}), '');
        });
 }
 
 # dangerous if there's duplicate messages with different Message-IDs
-sub dedupe_mid () {
-       my $skv = PublicInbox::SharedKV->new;
-       ($skv, sub { # may be called in a child process
+sub dedupe_mid ($) {
+       my ($skv) = @_;
+       (sub { # may be called in a child process
                my ($eml, $oid) = @_;
                # TODO: lei will support non-public messages w/o Message-ID
                my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) //
                        content_hash($eml);
                $skv->set_maybe($mid, '');
+       }, sub {
+               my ($smsg) = @_;
+               my $mid = $smsg->{mid};
+               $mid = undef if $mid eq '';
+               $mid //= smsg_hash($smsg) // _oidbin($smsg->{blob});
+               $skv->set_maybe($mid, '');
        });
 }
 
 # our default deduplication strategy (used by v2, also)
-sub dedupe_content () {
-       my $skv = PublicInbox::SharedKV->new;
-       ($skv, sub { # may be called in a child process
+sub dedupe_content ($) {
+       my ($skv) = @_;
+       (sub { # may be called in a child process
                my ($eml) = @_; # oid = $_[1], ignored
                $skv->set_maybe(content_hash($eml), '');
+       }, sub {
+               my ($smsg) = @_;
+               $skv->set_maybe(smsg_hash($smsg), '');
        });
 }
 
 # no deduplication at all
-sub dedupe_none () { (undef, sub { 1 }) }
+sub true { 1 }
+sub dedupe_none ($) { (\&true, \&true) }
 
 sub new {
-       my ($cls, $lei, $dst) = @_;
+       my ($cls, $lei) = @_;
        my $dd = $lei->{opt}->{dedupe} // 'content';
+       my $dst = $lei->{ovv}->{dst};
 
        # allow "none" to bypass Eml->new if writing to directory:
        return if ($dd eq 'none' && substr($dst // '', -1) eq '/');
-
-       my $dd_new = $cls->can("dedupe_$dd") //
-                       die "unsupported dedupe strategy: $dd\n";
-       bless [ $dd_new->() ], $cls; # [ $skv, $cb ]
+       my $m = "dedupe_$dd";
+       $cls->can($m) or die "unsupported dedupe strategy: $dd\n";
+       my $skv;
+       if ($dd ne 'none') {
+               require PublicInbox::SharedKV;
+               $skv = PublicInbox::SharedKV->new;
+       }
+       # [ $skv, $eml_cb, $smsg_cb, "dedupe_$dd" ]
+       bless [ $skv, undef, undef, $m ], $cls;
 }
 
-# returns true on unseen messages according to the deduplication strategy,
-# returns false if seen
+# returns true on seen messages according to the deduplication strategy,
+# returns false if unseen
 sub is_dup {
        my ($self, $eml, $oid) = @_;
        !$self->[1]->($eml, $oid);
 }
 
+sub is_smsg_dup {
+       my ($self, $smsg) = @_;
+       !$self->[2]->($smsg);
+}
+
 sub prepare_dedupe {
        my ($self) = @_;
        my $skv = $self->[0];
+       $self->[1] or @$self[1,2] = $self->can($self->[3])->($skv);
        $skv ? $skv->dbh : undef;
 }
 
 sub pause_dedupe {
        my ($self) = @_;
-       my $skv = $self->[0];
+       my $skv = $self->[0] or return;
+       $skv->dbh_release;
        delete($skv->{dbh}) if $skv;
 }