#!perl -w # Copyright (C) 2020 all contributors # License: AGPL-3.0+ use strict; use v5.10.1; use Test::More; use PublicInbox::TestCommon; use PublicInbox::Eml; require_mods(qw(DBD::SQLite)); use_ok 'PublicInbox::LeiToMail'; my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n"; my $noeol = "Subject: x\n\nFrom hell"; my $crlf = $noeol; $crlf =~ s/\n/\r\n/g; my $kw = [qw(seen answered flagged)]; for my $mbox (qw(mboxrd mboxo mboxcl mboxcl2)) { my $m = "eml2$mbox"; my $cb = PublicInbox::LeiToMail->can($m); my $s = $cb->(PublicInbox::Eml->new($from), $kw); is(substr($$s, -1, 1), "\n", "trailing LF in normal $mbox"); my $eml = PublicInbox::Eml->new($s); is($eml->header('Status'), 'R', "Status: set by $m"); is($eml->header('X-Status'), 'AF', "X-Status: set by $m"); if ($mbox eq 'mboxcl2') { like($eml->body_raw, qr/^From /, "From not escaped $m"); } else { like($eml->body_raw, qr/^>From /, "From escaped once by $m"); } my @cl = $eml->header('Content-Length'); if ($mbox =~ /mboxcl/) { is(scalar(@cl), 1, "$m only has one Content-Length header"); is($cl[0] + length("\n"), length($eml->body_raw), "$m Content-Length matches"); } else { is(scalar(@cl), 0, "$m clobbered Content-Length"); } $s = $cb->(PublicInbox::Eml->new($noeol), $kw); is(substr($$s, -1, 1), "\n", "trailing LF added by $m when original lacks EOL"); $eml = PublicInbox::Eml->new($s); if ($mbox eq 'mboxcl2') { is($eml->body_raw, "From hell\n", "From not escaped by $m"); } else { is($eml->body_raw, ">From hell\n", "From escaped once by $m"); } $s = $cb->(PublicInbox::Eml->new($crlf), $kw); is(substr($$s, -2, 2), "\r\n", "trailing CRLF added $m by original lacks EOL"); $eml = PublicInbox::Eml->new($s); if ($mbox eq 'mboxcl2') { is($eml->body_raw, "From hell\r\n", "From not escaped by $m"); } else { is($eml->body_raw, ">From hell\r\n", "From escaped once by $m"); } if ($mbox =~ /mboxcl/) { is($eml->header('Content-Length') + length("\r\n"), length($eml->body_raw), "$m Content-Length matches"); } elsif ($mbox eq 'mboxrd') { $s = $cb->($eml, $kw); $eml = PublicInbox::Eml->new($s); is($eml->body_raw, ">>From hell\r\n\r\n", "From escaped again by $m"); } } my ($tmpdir, $for_destroy) = tmpdir(); local $ENV{TMPDIR} = $tmpdir; open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!; my $lei = { 2 => $err }; my $buf = <<'EOM'; From: x@example.com Subject: x blah EOM my $fn = "$tmpdir/x.mbox"; my $orig = do { my $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei); is(ref $wcb, 'CODE', 'write_cb returned callback'); ok(-f $fn && !-s _, 'empty file created'); $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; open my $fh, '<', $fn or BAIL_OUT $!; my $raw = do { local $/; <$fh> }; like($raw, qr/^blah\n/sm, 'wrote content'); unlink $fn or BAIL_OUT $!; local $lei->{opt} = { jobs => 2 }; $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei); $lei->{dedupe}->prepare_dedupe; $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; open $fh, '<', $fn or BAIL_OUT $!; is($raw, do { local $/; <$fh> }, 'jobs > 1'); $raw; }; for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma? my $zsfx2cmd = PublicInbox::LeiToMail->can('zsfx2cmd'); SKIP: { my $cmd = eval { $zsfx2cmd->($zsfx, 0, $lei) }; skip $@, 3 if $@; my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) }; ok($dc_cmd, "decompressor for .$zsfx"); my $f = "$fn.$zsfx"; my $dst = "mboxcl2:$f"; my $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei); $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; my $uncompressed = xqx([@$dc_cmd, $f]); is($uncompressed, $orig, "$zsfx works unlocked"); local $lei->{opt} = { jobs => 2 }; # for atomic writes unlink $f or BAIL_OUT "unlink $!"; $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei); $lei->{dedupe}->prepare_dedupe; $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock"); } } unlink $fn or BAIL_OUT $!; if ('default deduplication uses content_hash') { my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei); $wcb->(\(my $x = $buf), 'deadbeef', []) for (1..2); undef $wcb; # undef to commit changes my $cmp = ''; open my $fh, '<', $fn or BAIL_OUT $!; require PublicInbox::MboxReader; PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= shift->as_string }); is($cmp, $buf, 'only one message written'); } done_testing;