2 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
3 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::TestCommon;
9 use Fcntl qw(SEEK_SET);
10 use PublicInbox::Spawn qw(popen_rd which);
11 use List::Util qw(shuffle);
12 require_mods(qw(DBD::SQLite));
13 require PublicInbox::MboxReader;
14 require PublicInbox::LeiOverview;
15 use_ok 'PublicInbox::LeiToMail';
16 my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n";
17 my $noeol = "Subject: x\n\nFrom hell";
20 my $kw = [qw(seen answered flagged)];
21 my $smsg = { kw => $kw, blob => '0'x40 };
22 my @MBOX = qw(mboxcl2 mboxrd mboxcl mboxo);
23 for my $mbox (@MBOX) {
25 my $cb = PublicInbox::LeiToMail->can($m);
26 my $s = $cb->(PublicInbox::Eml->new($from), $smsg);
27 is(substr($$s, -1, 1), "\n", "trailing LF in normal $mbox");
28 my $eml = PublicInbox::Eml->new($s);
29 is($eml->header('Status'), 'OR', "Status: set by $m");
30 is($eml->header('X-Status'), 'AF', "X-Status: set by $m");
31 if ($mbox eq 'mboxcl2') {
32 like($eml->body_raw, qr/^From /, "From not escaped $m");
34 like($eml->body_raw, qr/^>From /, "From escaped once by $m");
36 my @cl = $eml->header('Content-Length');
37 if ($mbox =~ /mboxcl/) {
38 is(scalar(@cl), 1, "$m only has one Content-Length header");
39 is($cl[0] + length("\n"),
40 length($eml->body_raw), "$m Content-Length matches");
42 is(scalar(@cl), 0, "$m clobbered Content-Length");
44 $s = $cb->(PublicInbox::Eml->new($noeol), $smsg);
45 is(substr($$s, -1, 1), "\n",
46 "trailing LF added by $m when original lacks EOL");
47 $eml = PublicInbox::Eml->new($s);
48 if ($mbox eq 'mboxcl2') {
49 is($eml->body_raw, "From hell\n", "From not escaped by $m");
51 is($eml->body_raw, ">From hell\n", "From escaped once by $m");
53 $s = $cb->(PublicInbox::Eml->new($crlf), $smsg);
54 is(substr($$s, -2, 2), "\r\n",
55 "trailing CRLF added $m by original lacks EOL");
56 $eml = PublicInbox::Eml->new($s);
57 if ($mbox eq 'mboxcl2') {
58 is($eml->body_raw, "From hell\r\n", "From not escaped by $m");
60 is($eml->body_raw, ">From hell\r\n", "From escaped once by $m");
62 if ($mbox =~ /mboxcl/) {
63 is($eml->header('Content-Length') + length("\r\n"),
64 length($eml->body_raw), "$m Content-Length matches");
65 } elsif ($mbox eq 'mboxrd') {
66 $s = $cb->($eml, $smsg);
67 $eml = PublicInbox::Eml->new($s);
69 ">>From hell\r\n\r\n", "From escaped again by $m");
73 my ($tmpdir, $for_destroy) = tmpdir();
74 local $ENV{TMPDIR} = $tmpdir;
75 open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
76 my $lei = { 2 => $err };
83 my $fn = "$tmpdir/x.mbox";
84 my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter
87 delete $lei->{dedupe};
91 }, 'PublicInbox::LeiOverview';
92 my $l2m = PublicInbox::LeiToMail->new($lei);
94 require_mods('Storable', 1);
95 my $dup = Storable::thaw(Storable::freeze($l2m));
96 is_deeply($dup, $l2m, "$fmt round-trips through storable");
98 my $zpipe = $l2m->pre_augment($lei);
99 $l2m->do_augment($lei);
100 $l2m->post_augment($lei, $zpipe);
101 my $cb = $l2m->write_cb($lei);
106 my $deadbeef = { blob => 'deadbeef', kw => [ qw(seen) ] };
108 my $wcb = $wcb_get->($mbox, $fn);
109 is(ref $wcb, 'CODE', 'write_cb returned callback');
110 ok(-f $fn && !-s _, 'empty file created');
111 $wcb->(\(my $dup = $buf), $deadbeef);
113 open my $fh, '<', $fn or BAIL_OUT $!;
114 my $raw = do { local $/; <$fh> };
115 like($raw, qr/^blah\n/sm, 'wrote content');
116 unlink $fn or BAIL_OUT $!;
118 local $lei->{opt} = { jobs => 2 };
119 $wcb = $wcb_get->($mbox, $fn);
120 ok(-f $fn && !-s _, 'truncated mbox destination');
121 $wcb->(\($dup = $buf), $deadbeef);
123 open $fh, '<', $fn or BAIL_OUT $!;
124 is(do { local $/; <$fh> }, $raw, 'jobs > 1');
127 for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
128 my $zsfx2cmd = PublicInbox::LeiToMail->can('zsfx2cmd');
130 my $cmd = eval { $zsfx2cmd->($zsfx, 0, $lei) };
132 my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) };
133 ok($dc_cmd, "decompressor for .$zsfx");
135 my $wcb = $wcb_get->($mbox, $f);
136 $wcb->(\(my $dup = $buf), $deadbeef);
138 my $uncompressed = xqx([@$dc_cmd, $f]);
139 is($uncompressed, $orig, "$zsfx works unlocked");
141 local $lei->{opt} = { jobs => 2 }; # for atomic writes
142 unlink $f or BAIL_OUT "unlink $!";
143 $wcb = $wcb_get->($mbox, $f);
144 $wcb->(\($dup = $buf), $deadbeef);
146 is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
148 local $lei->{opt} = { augment => 1 };
149 $wcb = $wcb_get->($mbox, $f);
150 $wcb->(\($dup = $buf . "\nx\n"), $deadbeef);
153 my $cat = popen_rd([@$dc_cmd, $f]);
155 PublicInbox::MboxReader->$mbox($cat,
156 sub { push @raw, shift->as_string });
157 like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx");
158 like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx");
160 local $lei->{opt} = { augment => 1, jobs => 2 };
161 $wcb = $wcb_get->($mbox, $f);
162 $wcb->(\($dup = $buf . "\ny\n"), $deadbeef);
166 $cat = popen_rd([@$dc_cmd, $f]);
167 PublicInbox::MboxReader->$mbox($cat,
168 sub { push @raw3, shift->as_string });
170 is_deeply(\@raw3, \@raw, 'previous messages preserved');
171 like($y, qr/\nblah\n\ny\n\z/s, "augmented $zsfx (atomic)");
177 $eml->header_set('Status');
181 unlink $fn or BAIL_OUT $!;
182 if ('default deduplication uses content_hash') {
183 my $wcb = $wcb_get->('mboxo', $fn);
184 $deadbeef->{kw} = [];
185 $wcb->(\(my $x = $buf), $deadbeef) for (1..2);
186 undef $wcb; # undef to commit changes
188 open my $fh, '<', $fn or BAIL_OUT $!;
189 PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= $as_orig->(@_) });
190 is($cmp, $buf, 'only one message written');
192 local $lei->{opt} = { augment => 1 };
193 $wcb = $wcb_get->('mboxo', $fn);
194 $wcb->(\($x = $buf . "\nx\n"), $deadbeef) for (1..2);
195 undef $wcb; # undef to commit changes
196 open $fh, '<', $fn or BAIL_OUT $!;
198 PublicInbox::MboxReader->mboxo($fh, sub { push @x, $as_orig->(@_) });
199 is(scalar(@x), 2, 'augmented mboxo');
200 is($x[0], $cmp, 'original message preserved');
201 is($x[1], $buf . "\nx\n", 'new message appended');
205 open my $tmp, '+>', undef or BAIL_OUT $!;
206 local $lei->{1} = $tmp;
207 my $wcb = $wcb_get->('mboxrd', '/dev/stdout');
208 $wcb->(\(my $x = $buf), $deadbeef);
210 seek($tmp, 0, SEEK_SET) or BAIL_OUT $!;
212 PublicInbox::MboxReader->mboxrd($tmp, sub { $cmp .= $as_orig->(@_) });
213 is($cmp, $buf, 'message written to stdout');
216 SKIP: { # FIFO support
217 use POSIX qw(mkfifo);
218 my $fn = "$tmpdir/fifo";
219 mkfifo($fn, 0600) or skip("mkfifo not supported: $!", 1);
220 my $cat = popen_rd([which('cat'), $fn]);
221 my $wcb = $wcb_get->('mboxo', $fn);
222 $wcb->(\(my $x = $buf), $deadbeef);
225 PublicInbox::MboxReader->mboxo($cat, sub { $cmp .= $as_orig->(@_) });
226 is($cmp, $buf, 'message written to FIFO');
230 my $md = "$tmpdir/maildir/";
231 my $wcb = $wcb_get->('maildir', $md);
232 is(ref($wcb), 'CODE', 'got Maildir callback');
233 my $b4dc0ffee = { blob => 'badc0ffee', kw => [] };
234 $wcb->(\(my $x = $buf), $b4dc0ffee);
237 PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
238 open my $fh, $f[0] or BAIL_OUT $!;
239 is(do { local $/; <$fh> }, $buf, 'wrote to Maildir');
241 $wcb = $wcb_get->('maildir', $md);
242 my $deadcafe = { blob => 'deadcafe', kw => [] };
243 $wcb->(\($x = $buf."\nx\n"), $deadcafe);
246 PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @x, shift });
247 is(scalar(@x), 1, 'wrote one new file');
248 ok(!-f $f[0], 'old file clobbered');
249 open $fh, $x[0] or BAIL_OUT $!;
250 is(do { local $/; <$fh> }, $buf."\nx\n", 'wrote new file to Maildir');
252 local $lei->{opt}->{augment} = 1;
253 $wcb = $wcb_get->('maildir', $md);
254 $wcb->(\($x = $buf."\ny\n"), $deadcafe);
255 $wcb->(\($x = $buf."\ny\n"), $b4dc0ffee); # skipped by dedupe
257 PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
258 is(scalar grep(/\A\Q$x[0]\E\z/, @f), 1, 'old file still there');
259 my @new = grep(!/\A\Q$x[0]\E\z/, @f);
260 is(scalar @new, 1, '1 new file written (b4dc0ffee skipped)');
261 open $fh, $x[0] or BAIL_OUT $!;
262 is(do { local $/; <$fh> }, $buf."\nx\n", 'old file untouched');
263 open $fh, $new[0] or BAIL_OUT $!;
264 is(do { local $/; <$fh> }, $buf."\ny\n", 'new file written');