lei: implement various deduplication strategies

[public-inbox.git] / t / lei_to_mail.t

diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 089a422ea5ab6161324242aec164b027d1c17421..5be4e285efc65ddce089fd3f1fc969f26297afa0 100644 (file)
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -6,6 +6,7 @@ use v5.10.1;
 use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
+require_mods(qw(DBD::SQLite));
 use_ok 'PublicInbox::LeiToMail';
 my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n";
 my $noeol = "Subject: x\n\nFrom hell";
@@ -62,4 +63,72 @@ for my $mbox (qw(mboxrd mboxo mboxcl mboxcl2)) {
        }
 }
 
+my ($tmpdir, $for_destroy) = tmpdir();
+local $ENV{TMPDIR} = $tmpdir;
+open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
+my $lei = { 2 => $err };
+my $buf = <<'EOM';
+From: x@example.com
+Subject: x
+
+blah
+EOM
+my $fn = "$tmpdir/x.mbox";
+my $orig = do {
+       my $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei);
+       is(ref $wcb, 'CODE', 'write_cb returned callback');
+       ok(-f $fn && !-s _, 'empty file created');
+       $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
+       undef $wcb;
+       open my $fh, '<', $fn or BAIL_OUT $!;
+       my $raw = do { local $/; <$fh> };
+       like($raw, qr/^blah\n/sm, 'wrote content');
+       unlink $fn or BAIL_OUT $!;
+
+       local $lei->{opt} = { jobs => 2 };
+       $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei);
+       $lei->{dedupe}->prepare_dedupe;
+       $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
+       undef $wcb;
+       open $fh, '<', $fn or BAIL_OUT $!;
+       is($raw, do { local $/; <$fh> }, 'jobs > 1');
+       $raw;
+};
+for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
+       my $zsfx2cmd = PublicInbox::LeiToMail->can('zsfx2cmd');
+       SKIP: {
+               my $cmd = eval { $zsfx2cmd->($zsfx, 0, $lei) };
+               skip $@, 3 if $@;
+               my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) };
+               ok($dc_cmd, "decompressor for .$zsfx");
+               my $f = "$fn.$zsfx";
+               my $dst = "mboxcl2:$f";
+               my $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
+               $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
+               undef $wcb;
+               my $uncompressed = xqx([@$dc_cmd, $f]);
+               is($uncompressed, $orig, "$zsfx works unlocked");
+
+               local $lei->{opt} = { jobs => 2 }; # for atomic writes
+               unlink $f or BAIL_OUT "unlink $!";
+               $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
+               $lei->{dedupe}->prepare_dedupe;
+               $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
+               undef $wcb;
+               is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
+       }
+}
+
+unlink $fn or BAIL_OUT $!;
+if ('default deduplication uses content_hash') {
+       my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
+       $wcb->(\(my $x = $buf), 'deadbeef', []) for (1..2);
+       undef $wcb; # undef to commit changes
+       my $cmp = '';
+       open my $fh, '<', $fn or BAIL_OUT $!;
+       require PublicInbox::MboxReader;
+       PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= shift->as_string });
+       is($cmp, $buf, 'only one message written');
+}
+
 done_testing;