1 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
3 package PublicInbox::LeiDedupe;
6 use PublicInbox::SharedKV;
7 use PublicInbox::ContentHash qw(content_hash);
9 # n.b. mutt sets most of these headers not sure about Bytes
10 our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
12 # best-effort regeneration of OID when augmenting existing results
15 my @stash; # stash away headers we shouldn't have in git
16 for my $k (@OID_IGNORE) {
17 my @v = $eml->header_raw($k) or next;
18 push @stash, [ $k, \@v ];
19 $eml->header_set($k); # restore below
21 my $dig = Digest::SHA->new(1); # XXX SHA256 later
22 my $buf = $eml->as_string;
23 $dig->add('blob '.length($buf)."\0");
27 for my $kv (@stash) { # restore stashed headers
29 $eml->header_set($k, @v);
34 sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef }
38 my $skv = PublicInbox::SharedKV->new;
39 ($skv, sub { # may be called in a child process
41 $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), '');
45 # dangerous if there's duplicate messages with different Message-IDs
47 my $skv = PublicInbox::SharedKV->new;
48 ($skv, sub { # may be called in a child process
50 # TODO: lei will support non-public messages w/o Message-ID
51 my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) //
53 $skv->set_maybe($mid, '');
57 # our default deduplication strategy (used by v2, also)
58 sub dedupe_content () {
59 my $skv = PublicInbox::SharedKV->new;
60 ($skv, sub { # may be called in a child process
61 my ($eml) = @_; # oid = $_[1], ignored
62 $skv->set_maybe(content_hash($eml), '');
66 # no deduplication at all
67 sub dedupe_none () { (undef, sub { 1 }) }
70 my ($cls, $lei, $dst) = @_;
71 my $dd = $lei->{opt}->{dedupe} // 'content';
73 # allow "none" to bypass Eml->new if writing to directory:
74 return if ($dd eq 'none' && substr($dst // '', -1) eq '/');
76 my $dd_new = $cls->can("dedupe_$dd") //
77 die "unsupported dedupe strategy: $dd\n";
78 bless [ $dd_new->() ], $cls; # [ $skv, $cb ]
81 # returns true on unseen messages according to the deduplication strategy,
82 # returns false if seen
84 my ($self, $eml, $oid) = @_;
85 !$self->[1]->($eml, $oid);
91 $skv ? $skv->dbh : undef;
97 delete($skv->{dbh}) if $skv;