Unindexed v1 inboxes were leaving $smsg objects unpopulated when
using public-inbox-httpd (but not generic PSGI servers) and
causing missing HTML content and uninitialized value warnings.
Our existing tests for unindexed v1 inboxes only assumed generic
PSGI servers and synchronous blob retrieval. Due to changes
several years ago to make git blob retrieval async for slow
storage using public-inbox-httpd, our tests were insufficient to
detect this regression.
So ensure $smsg->populate runs in a few places and rewrite
t/plack.t to test against both generic PSGI and -httpd
implementations.
Fortunately, unindexed v1 inboxes are uncommon, and this
bug was only (finally) discovered while developing other
features.
For ensuring we can test (and not blindly follow) redirects with
-httpd, we now provide our own LWP::UserAgent (used internally
by Plack::Test::ExternalServer) with redirect following
disabled to P:T:ES::test_psgi.
my ($ctx, $eml) = @_;
$ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
my ($ctx, $eml) = @_;
$ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
- $eml and return PublicInbox::View::eml_entry($ctx, $eml);
+ if ($eml) {
+ $ctx->{smsg}->populate($eml) if !$ctx->{ibx}->{over};
+ return PublicInbox::View::eml_entry($ctx, $eml);
+ }
my $smsg = shift @{$ctx->{msgs}} or
$ctx->zmore(PublicInbox::View::pagination_footer(
$ctx, './new.html'));
my $smsg = shift @{$ctx->{msgs}} or
$ctx->zmore(PublicInbox::View::pagination_footer(
$ctx, './new.html'));
$env->{$_} or BAIL_OUT "$_ unset";
}
SKIP: {
$env->{$_} or BAIL_OUT "$_ unset";
}
SKIP: {
- require_mods(qw(Plack::Test::ExternalServer), $skip // 1);
+ require_mods(qw(Plack::Test::ExternalServer LWP::UserAgent),
+ $skip // 1);
my $sock = tcp_server() or die;
my ($out, $err) = map { "$env->{TMPDIR}/std$_.log" } qw(out err);
my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
my $td = start_script($cmd, $env, { 3 => $sock });
my ($h, $p) = tcp_host_port($sock);
local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
my $sock = tcp_server() or die;
my ($out, $err) = map { "$env->{TMPDIR}/std$_.log" } qw(out err);
my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
my $td = start_script($cmd, $env, { 3 => $sock });
my ($h, $p) = tcp_host_port($sock);
local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
- Plack::Test::ExternalServer::test_psgi(client => $client);
+ my $ua = LWP::UserAgent->new;
+ $ua->max_redirect(0);
+ Plack::Test::ExternalServer::test_psgi(client => $client,
+ ua => $ua);
$td->join('TERM');
open my $fh, '<', $err or BAIL_OUT $!;
my $e = do { local $/; <$fh> };
$td->join('TERM');
open my $fh, '<', $err or BAIL_OUT $!;
my $e = do { local $/; <$fh> };
sub async_eml { # for async_blob_cb
my ($ctx, $eml) = @_;
my $smsg = delete $ctx->{smsg};
sub async_eml { # for async_blob_cb
my ($ctx, $eml) = @_;
my $smsg = delete $ctx->{smsg};
+ $smsg->{mid} // $smsg->populate($eml);
$ctx->write(feed_entry($ctx, $smsg, $eml));
}
$ctx->write(feed_entry($ctx, $smsg, $eml));
}
require_mods(@mods);
foreach my $mod (@mods) { use_ok $mod; }
ok(-f $psgi, "psgi example file found");
require_mods(@mods);
foreach my $mod (@mods) { use_ok $mod; }
ok(-f $psgi, "psgi example file found");
+my ($tmpdir, $for_destroy) = tmpdir();
my $pfx = 'http://example.com/test';
my $eml = eml_load('t/iso-2202-jp.eml');
# ensure successful message deliveries
my $pfx = 'http://example.com/test';
my $eml = eml_load('t/iso-2202-jp.eml');
# ensure successful message deliveries
close $fh or BAIL_OUT "close: $!";
});
close $fh or BAIL_OUT "close: $!";
});
-local $ENV{PI_CONFIG} = "$ibx->{inboxdir}/pi_config";
-my $app = require $psgi;
-test_psgi($app, sub {
+my $env = { PI_CONFIG => "$ibx->{inboxdir}/pi_config", TMPDIR => $tmpdir };
+local @ENV{keys %$env} = values %$env;
+my $c1 = sub {
+ my $uri = $ENV{PLACK_TEST_EXTERNALSERVER_URI} // 'http://example.com';
+ $pfx = "$uri/test";
+
foreach my $u (qw(robots.txt favicon.ico .well-known/foo)) {
foreach my $u (qw(robots.txt favicon.ico .well-known/foo)) {
- my $res = $cb->(GET("http://example.com/$u"));
+ my $res = $cb->(GET("$uri/$u"));
is($res->code, 404, "$u is missing");
}
is($res->code, 404, "$u is missing");
}
-test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET('http://example.com/test/crlf@example.com/'));
+ my $res = $cb->(GET("$uri/test/crlf\@example.com/"));
is($res->code, 200, 'retrieved CRLF as HTML');
like($res->content, qr/mailto:me\@example/, 'no %40, per RFC 6068');
unlike($res->content, qr/\r/, 'no CR in HTML');
is($res->code, 200, 'retrieved CRLF as HTML');
like($res->content, qr/mailto:me\@example/, 'no %40, per RFC 6068');
unlike($res->content, qr/\r/, 'no CR in HTML');
- $res = $cb->(GET('http://example.com/test/crlf@example.com/raw'));
+ $res = $cb->(GET("$uri/test/crlf\@example.com/raw"));
is($res->code, 200, 'retrieved CRLF raw');
like($res->content, qr/\r/, 'CR preserved in raw message');
is($res->code, 200, 'retrieved CRLF raw');
like($res->content, qr/\r/, 'CR preserved in raw message');
- $res = $cb->(GET('http://example.com/test/bogus@example.com/raw'));
+ $res = $cb->(GET("$uri/test/bogus\@example.com/raw"));
is($res->code, 404, 'missing /raw is 404');
is($res->code, 404, 'missing /raw is 404');
-# redirect with newsgroup
-test_psgi($app, sub {
- my ($cb) = @_;
- my $from = 'http://example.com/inbox.test';
- my $to = 'http://example.com/test/';
- my $res = $cb->(GET($from));
+ # redirect with newsgroup
+ my $from = "$uri/inbox.test";
+ my $to = "http://example.com/test/";
+ $res = $cb->(GET($from));
is($res->code, 301, 'newsgroup name is permanent redirect');
is($to, $res->header('Location'), 'redirect location matches');
$from .= '/';
is($res->code, 301, 'newsgroup name/ is permanent redirect');
is($to, $res->header('Location'), 'redirect location matches');
is($res->code, 301, 'newsgroup name is permanent redirect');
is($to, $res->header('Location'), 'redirect location matches');
$from .= '/';
is($res->code, 301, 'newsgroup name/ is permanent redirect');
is($to, $res->header('Location'), 'redirect location matches');
-# redirect with trailing /
-test_psgi($app, sub {
- my ($cb) = @_;
- my $from = 'http://example.com/test';
- my $to = "$from/";
- my $res = $cb->(GET($from));
+ # redirect with trailing /
+ $from = "$uri/test";
+ $to = "$from/";
+ $res = $cb->(GET($from));
is(301, $res->code, 'is permanent redirect');
is($to, $res->header('Location'),
'redirect location matches with trailing slash');
is(301, $res->code, 'is permanent redirect');
is($to, $res->header('Location'),
'redirect location matches with trailing slash');
-foreach my $t (qw(t T)) {
- test_psgi($app, sub {
- my ($cb) = @_;
my $u = $pfx . "/blah\@example.com/$t";
my $u = $pfx . "/blah\@example.com/$t";
- my $res = $cb->(GET($u));
is(301, $res->code, "redirect for missing /");
my $location = $res->header('Location');
like($location, qr!/\Q$t\E/#u\z!,
'redirected with missing /');
is(301, $res->code, "redirect for missing /");
my $location = $res->header('Location');
like($location, qr!/\Q$t\E/#u\z!,
'redirected with missing /');
- });
-}
-foreach my $t (qw(f)) {
- test_psgi($app, sub {
- my ($cb) = @_;
+ }
+
+ for my $t (qw(f)) { # legacy redirect
my $u = $pfx . "/blah\@example.com/$t";
my $u = $pfx . "/blah\@example.com/$t";
- my $res = $cb->(GET($u));
is(301, $res->code, "redirect for legacy /f");
my $location = $res->header('Location');
like($location, qr!/blah\@example\.com/\z!,
'redirected with missing /');
is(301, $res->code, "redirect for legacy /f");
my $location = $res->header('Location');
like($location, qr!/blah\@example\.com/\z!,
'redirected with missing /');
-test_psgi($app, sub {
- my ($cb) = @_;
- my $atomurl = 'http://example.com/test/new.atom';
- my $res = $cb->(GET('http://example.com/test/new.html'));
+ my $atomurl = "$uri/test/new.atom";
+ $res = $cb->(GET("$uri/test/new.html"));
is(200, $res->code, 'success response received');
like($res->content, qr!href="new\.atom"!,
'atom URL generated');
like($res->content, qr!href="blah\@example\.com/"!,
'index generated');
like($res->content, qr!1993-10-02!, 'date set');
is(200, $res->code, 'success response received');
like($res->content, qr!href="new\.atom"!,
'atom URL generated');
like($res->content, qr!href="blah\@example\.com/"!,
'index generated');
like($res->content, qr!1993-10-02!, 'date set');
-test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET($pfx . '/atom.xml'));
+ $res = $cb->(GET($pfx . '/atom.xml'));
is(200, $res->code, 'success response received for atom');
my $body = $res->content;
like($body, qr!link\s+href="\Q$pfx\E/blah\@example\.com/"!s,
is(200, $res->code, 'success response received for atom');
my $body = $res->content;
like($body, qr!link\s+href="\Q$pfx\E/blah\@example\.com/"!s,
like($body, qr/zzzzzz/, 'body included');
$res = $cb->(GET($pfx . '/description'));
like($res->content, qr/test for public-inbox/, 'got description');
like($body, qr/zzzzzz/, 'body included');
$res = $cb->(GET($pfx . '/description'));
like($res->content, qr/test for public-inbox/, 'got description');
-test_psgi($app, sub {
- my ($cb) = @_;
my $path = '/blah@example.com/';
my $path = '/blah@example.com/';
- my $res = $cb->(GET($pfx . $path));
+ $res = $cb->(GET($pfx . $path));
is(200, $res->code, "success for $path");
my $html = $res->content;
like($html, qr!<title>hihi - Me</title>!, 'HTML returned');
is(200, $res->code, "success for $path");
my $html = $res->content;
like($html, qr!<title>hihi - Me</title>!, 'HTML returned');
$res = $cb->(GET($pfx . '/qp@example.com/'));
like($res->content, qr/\bhi = bye\b/, "HTML output decoded QP");
$res = $cb->(GET($pfx . '/qp@example.com/'));
like($res->content, qr/\bhi = bye\b/, "HTML output decoded QP");
-test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET($pfx . '/blah@example.com/raw'));
+
+ $res = $cb->(GET($pfx . '/blah@example.com/raw'));
is(200, $res->code, 'success response received for /*/raw');
like($res->content, qr!^From !sm, "mbox returned");
is($res->header('Content-Type'), 'text/plain; charset=iso-8859-1',
is(200, $res->code, 'success response received for /*/raw');
like($res->content, qr!^From !sm, "mbox returned");
is($res->header('Content-Type'), 'text/plain; charset=iso-8859-1',
$res = $cb->(GET($pfx . '/199707281508.AAA24167@hoyogw.example/raw'));
is($res->header('Content-Type'), 'text/plain; charset=ISO-2022-JP',
'ISO-2002-JP returned');
$res = $cb->(GET($pfx . '/199707281508.AAA24167@hoyogw.example/raw'));
is($res->header('Content-Type'), 'text/plain; charset=ISO-2022-JP',
'ISO-2002-JP returned');
- chomp(my $body = $res->content);
+ chomp($body = $res->content);
my $raw = PublicInbox::Eml->new(\$body);
is($raw->body_raw, $eml->body_raw, 'ISO-2022-JP body unmodified');
$res = $cb->(GET($pfx . '/blah@example.com/t.mbox.gz'));
is(501, $res->code, '501 when overview missing');
like($res->content, qr!\bOverview\b!, 'overview omission noted');
my $raw = PublicInbox::Eml->new(\$body);
is($raw->body_raw, $eml->body_raw, 'ISO-2022-JP body unmodified');
$res = $cb->(GET($pfx . '/blah@example.com/t.mbox.gz'));
is(501, $res->code, '501 when overview missing');
like($res->content, qr!\bOverview\b!, 'overview omission noted');
-# legacy redirects
-foreach my $t (qw(m f)) {
- test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET($pfx . "/$t/blah\@example.com.txt"));
+ # legacy redirects
+ for my $t (qw(m f)) {
+ $res = $cb->(GET($pfx . "/$t/blah\@example.com.txt"));
is(301, $res->code, "redirect for old $t .txt link");
is(301, $res->code, "redirect for old $t .txt link");
- my $location = $res->header('Location');
+ $location = $res->header('Location');
like($location, qr!/blah\@example\.com/raw\z!,
".txt redirected to /raw");
like($location, qr!/blah\@example\.com/raw\z!,
".txt redirected to /raw");
- });
-}
-
-my %umap = (
- 'm' => '',
- 'f' => '',
- 't' => 't/',
-);
-while (my ($t, $e) = each %umap) {
- test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET($pfx . "/$t/blah\@example.com.html"));
+ }
+
+ my %umap = (
+ 'm' => '',
+ 'f' => '',
+ 't' => 't/',
+ );
+ while (my ($t, $e) = each %umap) {
+ $res = $cb->(GET($pfx . "/$t/blah\@example.com.html"));
is(301, $res->code, "redirect for old $t .html link");
is(301, $res->code, "redirect for old $t .html link");
- my $location = $res->header('Location');
- like($location,
- qr!/blah\@example\.com/$e(?:#u)?\z!,
- ".html redirected to new location");
- });
-}
-foreach my $sfx (qw(mbox mbox.gz)) {
- test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(GET($pfx . "/t/blah\@example.com.$sfx"));
+ $location = $res->header('Location');
+ like($location, qr!/blah\@example\.com/$e(?:#u)?\z!,
+ ".html redirected to new location");
+ }
+
+ for my $sfx (qw(mbox mbox.gz)) {
+ $res = $cb->(GET($pfx . "/t/blah\@example.com.$sfx"));
is(301, $res->code, 'redirect for old thread link');
is(301, $res->code, 'redirect for old thread link');
- my $location = $res->header('Location');
+ $location = $res->header('Location');
like($location,
qr!/blah\@example\.com/t\.mbox(?:\.gz)?\z!,
"$sfx redirected to /mbox.gz");
like($location,
qr!/blah\@example\.com/t\.mbox(?:\.gz)?\z!,
"$sfx redirected to /mbox.gz");
- });
-}
-test_psgi($app, sub {
- my ($cb) = @_;
# for a while, we used to support /$INBOX/$X40/
# when we "compressed" long Message-IDs to SHA-1
# Now we're stuck supporting them forever :<
# for a while, we used to support /$INBOX/$X40/
# when we "compressed" long Message-IDs to SHA-1
# Now we're stuck supporting them forever :<
- foreach my $path ('f2912279bd7bcd8b7ab3033234942d58746d56f7') {
- my $from = "http://example.com/test/$path/";
- my $res = $cb->(GET($from));
+ for my $path ('f2912279bd7bcd8b7ab3033234942d58746d56f7') {
+ $from = "$uri/test/$path/";
+ $res = $cb->(GET($from));
is(301, $res->code, 'is permanent redirect');
like($res->header('Location'),
qr!/test/blah\@example\.com/!,
'redirect from x40 MIDs works');
}
is(301, $res->code, 'is permanent redirect');
like($res->header('Location'),
qr!/test/blah\@example\.com/!,
'redirect from x40 MIDs works');
}
-# dumb HTTP clone/fetch support
-test_psgi($app, sub {
- my ($cb) = @_;
- my $path = '/test/info/refs';
+
+ # dumb HTTP clone/fetch support
+ $path = '/test/info/refs';
my $req = HTTP::Request->new('GET' => $path);
my $req = HTTP::Request->new('GET' => $path);
is(200, $res->code, 'refs readable');
my $orig = $res->content;
is(200, $res->code, 'refs readable');
my $orig = $res->content;
$res = $cb->($req);
is(206, $res->code, 'got partial another response');
is($res->content, substr($orig, 5), 'partial body OK past end');
$res = $cb->($req);
is(206, $res->code, 'got partial another response');
is($res->content, substr($orig, 5), 'partial body OK past end');
-# things which should fail
-test_psgi($app, sub {
- my ($cb) = @_;
- my $res = $cb->(PUT('/'));
+ # things which should fail
+ $res = $cb->(PUT('/'));
is(405, $res->code, 'no PUT to / allowed');
$res = $cb->(PUT('/test/'));
is(405, $res->code, 'no PUT /$INBOX allowed');
is(405, $res->code, 'no PUT to / allowed');
$res = $cb->(PUT('/test/'));
is(405, $res->code, 'no PUT /$INBOX allowed');
-
- # TODO
- # $res = $cb->(GET('/'));
-});
-
-done_testing();
+};
+test_psgi(require $psgi, $c1);
+test_httpd($env, $c1);
+done_testing;