X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Flei_to_mail.t;h=e8958c64c46c73b6ae38feeef8718c2b7bd9e2f3;hb=2231c8b183be0be5d8a9738a3e417b5c3a09c7c7;hp=bd864754b4238fd62bf4ee0a3a305bb4cf260558;hpb=abd0a85b212ce1467ddc94d523152d9a65028960;p=public-inbox.git
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index bd864754..e8958c64 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -1,30 +1,34 @@
#!perl -w
-# Copyright (C) 2020 all contributors
+# Copyright (C) 2020-2021 all contributors
# License: AGPL-3.0+
use strict;
use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
use PublicInbox::Eml;
-use Fcntl qw(SEEK_SET);
-use PublicInbox::Spawn qw(popen_rd which);
+use Fcntl qw(SEEK_SET O_RDONLY O_NONBLOCK);
+use PublicInbox::Spawn qw(popen_rd);
use List::Util qw(shuffle);
require_mods(qw(DBD::SQLite));
+require PublicInbox::MdirReader;
require PublicInbox::MboxReader;
+require PublicInbox::LeiOverview;
+require PublicInbox::LEI;
use_ok 'PublicInbox::LeiToMail';
my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n";
my $noeol = "Subject: x\n\nFrom hell";
my $crlf = $noeol;
$crlf =~ s/\n/\r\n/g;
my $kw = [qw(seen answered flagged)];
+my $smsg = { kw => $kw, blob => '0'x40 };
my @MBOX = qw(mboxcl2 mboxrd mboxcl mboxo);
for my $mbox (@MBOX) {
my $m = "eml2$mbox";
my $cb = PublicInbox::LeiToMail->can($m);
- my $s = $cb->(PublicInbox::Eml->new($from), $kw);
+ my $s = $cb->(PublicInbox::Eml->new($from), $smsg);
is(substr($$s, -1, 1), "\n", "trailing LF in normal $mbox");
my $eml = PublicInbox::Eml->new($s);
- is($eml->header('Status'), 'R', "Status: set by $m");
+ is($eml->header('Status'), 'RO', "Status: set by $m");
is($eml->header('X-Status'), 'AF', "X-Status: set by $m");
if ($mbox eq 'mboxcl2') {
like($eml->body_raw, qr/^From /, "From not escaped $m");
@@ -39,7 +43,7 @@ for my $mbox (@MBOX) {
} else {
is(scalar(@cl), 0, "$m clobbered Content-Length");
}
- $s = $cb->(PublicInbox::Eml->new($noeol), $kw);
+ $s = $cb->(PublicInbox::Eml->new($noeol), $smsg);
is(substr($$s, -1, 1), "\n",
"trailing LF added by $m when original lacks EOL");
$eml = PublicInbox::Eml->new($s);
@@ -48,7 +52,7 @@ for my $mbox (@MBOX) {
} else {
is($eml->body_raw, ">From hell\n", "From escaped once by $m");
}
- $s = $cb->(PublicInbox::Eml->new($crlf), $kw);
+ $s = $cb->(PublicInbox::Eml->new($crlf), $smsg);
is(substr($$s, -2, 2), "\r\n",
"trailing CRLF added $m by original lacks EOL");
$eml = PublicInbox::Eml->new($s);
@@ -61,7 +65,7 @@ for my $mbox (@MBOX) {
is($eml->header('Content-Length') + length("\r\n"),
length($eml->body_raw), "$m Content-Length matches");
} elsif ($mbox eq 'mboxrd') {
- $s = $cb->($eml, $kw);
+ $s = $cb->($eml, $smsg);
$eml = PublicInbox::Eml->new($s);
is($eml->body_raw,
">>From hell\r\n\r\n", "From escaped again by $m");
@@ -71,7 +75,11 @@ for my $mbox (@MBOX) {
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{TMPDIR} = $tmpdir;
open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
-my $lei = { 2 => $err };
+my $lei = bless { 2 => $err, cmd => 'test' }, 'PublicInbox::LEI';
+my $commit = sub {
+ $_[0] = undef; # wcb
+ delete $lei->{1};
+};
my $buf = <<'EOM';
From: x@example.com
Subject: x
@@ -80,55 +88,90 @@ blah
EOM
my $fn = "$tmpdir/x.mbox";
my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter
+my $wcb_get = sub {
+ my ($fmt, $dst) = @_;
+ delete $lei->{dedupe}; # to be recreated
+ $lei->{ovv} = bless {
+ fmt => $fmt,
+ dst => $dst
+ }, 'PublicInbox::LeiOverview';
+ my $l2m = PublicInbox::LeiToMail->new($lei);
+ SKIP: {
+ require_mods('Storable', 1);
+ my $dup = Storable::thaw(Storable::freeze($l2m));
+ is_deeply($dup, $l2m, "$fmt round-trips through storable");
+ }
+ $l2m->pre_augment($lei);
+ $l2m->do_augment($lei);
+ $l2m->post_augment($lei);
+ $l2m->write_cb($lei);
+};
+
+my $deadbeef = { blob => 'deadbeef', kw => [ qw(seen) ] };
my $orig = do {
- my $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei);
+ my $wcb = $wcb_get->($mbox, $fn);
is(ref $wcb, 'CODE', 'write_cb returned callback');
ok(-f $fn && !-s _, 'empty file created');
- $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
- undef $wcb;
+ $wcb->(\(my $dup = $buf), $deadbeef);
+ $commit->($wcb);
open my $fh, '<', $fn or BAIL_OUT $!;
my $raw = do { local $/; <$fh> };
like($raw, qr/^blah\n/sm, 'wrote content');
unlink $fn or BAIL_OUT $!;
- local $lei->{opt} = { jobs => 2 };
- $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei);
+ $wcb = $wcb_get->($mbox, $fn);
ok(-f $fn && !-s _, 'truncated mbox destination');
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
- undef $wcb;
+ $wcb->(\($dup = $buf), $deadbeef);
+ $commit->($wcb);
open $fh, '<', $fn or BAIL_OUT $!;
- is($raw, do { local $/; <$fh> }, 'jobs > 1');
+ is(do { local $/; <$fh> }, $raw, 'wrote identical content');
$raw;
};
-for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
- my $zsfx2cmd = PublicInbox::LeiToMail->can('zsfx2cmd');
+
+test_lei({tmpdir => "$tmpdir/using -F"}, sub {
+ lei_ok(qw(import -F), $mbox, $fn, \'imported mbox');
+ lei_ok(qw(q s:x), \'lei q works') or diag $lei_err;
+ my $res = json_utf8->decode($lei_out);
+ my $x = $res->[0];
+ is($x->{'s'}, 'x', 'subject imported') or diag $lei_out;
+ is_deeply($x->{'kw'}, ['seen'], 'kw imported') or diag $lei_out;
+ is($res->[1], undef, 'only one result');
+});
+
+test_lei({tmpdir => "$tmpdir/using TYPE: prefix"}, sub {
+ lei_ok('import', "$mbox:$fn", \'imported mbox:/path') or diag $lei_err;
+ lei_ok(qw(q s:x), \'lei q works') or diag $lei_err;
+ my $res = json_utf8->decode($lei_out);
+ my $x = $res->[0];
+ is($x->{'s'}, 'x', 'subject imported') or diag $lei_out;
+ is_deeply($x->{'kw'}, ['seen'], 'kw imported') or diag $lei_out;
+ is($res->[1], undef, 'only one result');
+});
+
+my $zsfx2cmd = PublicInbox::MboxReader->can('zsfx2cmd');
+for my $zsfx (qw(gz bz2 xz)) {
SKIP: {
my $cmd = eval { $zsfx2cmd->($zsfx, 0, $lei) };
skip $@, 3 if $@;
my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) };
ok($dc_cmd, "decompressor for .$zsfx");
my $f = "$fn.$zsfx";
- my $dst = "$mbox:$f";
- my $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
- $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
- undef $wcb;
+ my $wcb = $wcb_get->($mbox, $f);
+ $wcb->(\(my $dup = $buf), { %$deadbeef });
+ $commit->($wcb);
my $uncompressed = xqx([@$dc_cmd, $f]);
is($uncompressed, $orig, "$zsfx works unlocked");
- local $lei->{opt} = { jobs => 2 }; # for atomic writes
unlink $f or BAIL_OUT "unlink $!";
- $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
- undef $wcb;
+ $wcb = $wcb_get->($mbox, $f);
+ $wcb->(\($dup = $buf), { %$deadbeef });
+ $commit->($wcb);
is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
local $lei->{opt} = { augment => 1 };
- $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($dup = $buf . "\nx\n"), 'deadbeef', [ qw(seen) ]);
- undef $wcb; # commit
+ $wcb = $wcb_get->($mbox, $f);
+ $wcb->(\($dup = $buf . "\nx\n"), { %$deadbeef });
+ $commit->($wcb);
my $cat = popen_rd([@$dc_cmd, $f]);
my @raw;
@@ -137,11 +180,10 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx");
like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx");
- local $lei->{opt} = { augment => 1, jobs => 2 };
- $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($dup = $buf . "\ny\n"), 'deadbeef', [ qw(seen) ]);
- undef $wcb; # commit
+ local $lei->{opt} = { augment => 1 };
+ $wcb = $wcb_get->($mbox, $f);
+ $wcb->(\($dup = $buf . "\ny\n"), { %$deadbeef });
+ $commit->($wcb);
my @raw3;
$cat = popen_rd([@$dc_cmd, $f]);
@@ -153,23 +195,30 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
}
}
+my $as_orig = sub {
+ my ($eml) = @_;
+ $eml->header_set('Status');
+ $eml->as_string;
+};
+
unlink $fn or BAIL_OUT $!;
if ('default deduplication uses content_hash') {
- my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
- $wcb->(\(my $x = $buf), 'deadbeef', []) for (1..2);
- undef $wcb; # undef to commit changes
+ my $wcb = $wcb_get->('mboxo', $fn);
+ $deadbeef->{kw} = [];
+ $wcb->(\(my $x = $buf), $deadbeef) for (1..2);
+ $commit->($wcb);
my $cmp = '';
open my $fh, '<', $fn or BAIL_OUT $!;
- PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= shift->as_string });
+ PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= $as_orig->(@_) });
is($cmp, $buf, 'only one message written');
local $lei->{opt} = { augment => 1 };
- $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
- $wcb->(\($x = $buf . "\nx\n"), 'deadbeef', []) for (1..2);
- undef $wcb; # undef to commit changes
+ $wcb = $wcb_get->('mboxo', $fn);
+ $wcb->(\($x = $buf . "\nx\n"), $deadbeef) for (1..2);
+ $commit->($wcb);
open $fh, '<', $fn or BAIL_OUT $!;
my @x;
- PublicInbox::MboxReader->mboxo($fh, sub { push @x, shift->as_string });
+ PublicInbox::MboxReader->mboxo($fh, sub { push @x, $as_orig->(@_) });
is(scalar(@x), 2, 'augmented mboxo');
is($x[0], $cmp, 'original message preserved');
is($x[1], $buf . "\nx\n", 'new message appended');
@@ -178,12 +227,12 @@ if ('default deduplication uses content_hash') {
{ # stdout support
open my $tmp, '+>', undef or BAIL_OUT $!;
local $lei->{1} = $tmp;
- my $wcb = PublicInbox::LeiToMail->write_cb("mboxrd:/dev/stdout", $lei);
- $wcb->(\(my $x = $buf), 'deadbeef', []);
- undef $wcb; # commit
+ my $wcb = $wcb_get->('mboxrd', '/dev/stdout');
+ $wcb->(\(my $x = $buf), $deadbeef);
+ $commit->($wcb);
seek($tmp, 0, SEEK_SET) or BAIL_OUT $!;
my $cmp = '';
- PublicInbox::MboxReader->mboxrd($tmp, sub { $cmp .= shift->as_string });
+ PublicInbox::MboxReader->mboxrd($tmp, sub { $cmp .= $as_orig->(@_) });
is($cmp, $buf, 'message written to stdout');
}
@@ -191,49 +240,46 @@ SKIP: { # FIFO support
use POSIX qw(mkfifo);
my $fn = "$tmpdir/fifo";
mkfifo($fn, 0600) or skip("mkfifo not supported: $!", 1);
- my $cat = popen_rd([which('cat'), $fn]);
- my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
- $wcb->(\(my $x = $buf), 'deadbeef', []);
- undef $wcb; # commit
+ sysopen(my $cat, $fn, O_RDONLY|O_NONBLOCK) or BAIL_OUT $!;
+ my $wcb = $wcb_get->('mboxo', $fn);
+ $wcb->(\(my $x = $buf), $deadbeef);
+ $commit->($wcb);
my $cmp = '';
- PublicInbox::MboxReader->mboxo($cat, sub { $cmp .= shift->as_string });
+ $cat->blocking(1);
+ PublicInbox::MboxReader->mboxo($cat, sub { $cmp .= $as_orig->(@_) });
is($cmp, $buf, 'message written to FIFO');
}
{ # Maildir support
+ my $mdr = PublicInbox::MdirReader->new;
my $md = "$tmpdir/maildir/";
- my $wcb = PublicInbox::LeiToMail->write_cb("Maildir:$md", $lei);
- ok($wcb, 'got Maildir callback');
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\(my $x = $buf), 'badc0ffee', []);
- undef $wcb; # commit
+ my $wcb = $wcb_get->('maildir', $md);
+ is(ref($wcb), 'CODE', 'got Maildir callback');
+ my $b4dc0ffee = { blob => 'badc0ffee', kw => [] };
+ $wcb->(\(my $x = $buf), $b4dc0ffee);
my @f;
- PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
- is(scalar(@f), 1, 'wrote one file');
+ $mdr->maildir_each_file($md, sub { push @f, shift });
open my $fh, $f[0] or BAIL_OUT $!;
is(do { local $/; <$fh> }, $buf, 'wrote to Maildir');
- $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei);
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($x = $buf."\nx\n"), 'deadcafe', []);
- undef $wcb; # commit
+ $wcb = $wcb_get->('maildir', $md);
+ my $deadcafe = { blob => 'deadcafe', kw => [] };
+ $wcb->(\($x = $buf."\nx\n"), $deadcafe);
my @x = ();
- PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @x, shift });
+ $mdr->maildir_each_file($md, sub { push @x, shift });
is(scalar(@x), 1, 'wrote one new file');
ok(!-f $f[0], 'old file clobbered');
open $fh, $x[0] or BAIL_OUT $!;
is(do { local $/; <$fh> }, $buf."\nx\n", 'wrote new file to Maildir');
local $lei->{opt}->{augment} = 1;
- $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei);
- $lei->{dedupe}->prepare_dedupe;
- $wcb->(\($x = $buf."\ny\n"), 'deadcafe', []);
- $wcb->(\($x = $buf."\ny\n"), 'b4dc0ffee', []); # skipped by dedupe
- undef $wcb; # commit
+ $wcb = $wcb_get->('maildir', $md);
+ $wcb->(\($x = $buf."\ny\n"), $deadcafe);
+ $wcb->(\($x = $buf."\ny\n"), $b4dc0ffee); # skipped by dedupe
@f = ();
- PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
+ $mdr->maildir_each_file($md, sub { push @f, shift });
is(scalar grep(/\A\Q$x[0]\E\z/, @f), 1, 'old file still there');
my @new = grep(!/\A\Q$x[0]\E\z/, @f);
is(scalar @new, 1, '1 new file written (b4dc0ffee skipped)');