#!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# manifest.js.gz generation and grok-pull integration test
-use strict;
-use v5.10.1;
-use Test::More;
-use PublicInbox::Spawn qw(which);
-use PublicInbox::TestCommon;
+use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Import;
use IO::Uncompress::Gunzip qw(gunzip);
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
+require_mods(qw(json URI::Escape Plack::Builder HTTP::Tiny));
require PublicInbox::WwwListing;
require PublicInbox::ManifestJsGz;
use PublicInbox::Config;
is(xsys(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i")
}
ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox');
+ open($fh, '>', "$v2/description") or xbail "open $v2/description: $!";
+ print $fh "a v2 inbox\n" or xbail "print $!";
+ close $fh or xbail "write: $v2/description $!";
+
open $fh, '>', "$alt/description" or xbail "open $alt/description $!";
print $fh "we're \xc4\x80ll clones\n" or xbail "print $!";
close $fh or xbail "write: $alt/description $!";
is(xsys('git', "--git-dir=$alt", qw(config gitweb.owner),
"lorelei \xc4\x80"), 0,
'set gitweb user');
- ok(unlink("$bare->{git_dir}/description"), 'removed bare/description');
open $fh, '>', $cfgfile or xbail "open $cfgfile: $!";
$fh->autoflush(1);
print $fh <<"" or xbail "print $!";
my $env = { PI_CONFIG => $cfgfile };
my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+ my $psgi = "$tmpdir/pfx.psgi";
+ {
+ open my $psgi_fh, '>', $psgi or xbail "open: $!";
+ print $psgi_fh <<'EOM' or xbail "print $!";
+use PublicInbox::WWW;
+use Plack::Builder;
+my $www = PublicInbox::WWW->new;
+builder {
+ enable 'Head';
+ mount '/pfx/' => sub { $www->call(@_) }
+}
+EOM
+ close $psgi_fh or xbail "close: $!";
+ }
+
+ # ensure prefixed mount full clones work:
+ $td = start_script([@$cmd, $psgi], $env, { 3 => $sock });
+ my $opt = { 2 => \(my $clone_err = '') };
+ ok(run_script(['-clone', "http://$host:$port/pfx", "$tmpdir/pfx" ],
+ undef, $opt), 'pfx clone w/pfx') or diag "clone_err=$clone_err";
+
+ open my $mh, '<', "$tmpdir/pfx/manifest.js.gz" or xbail "open: $!";
+ gunzip(\(do { local $/; <$mh> }) => \(my $mjs = ''));
+ my $mf = $json->decode($mjs);
+ is_deeply([sort keys %$mf], [ qw(/alt /bare /v2/git/0.git
+ /v2/git/1.git /v2/git/2.git) ],
+ 'manifest saved');
+ for (keys %$mf) { ok(-d "$tmpdir/pfx$_", "pfx/$_ cloned") }
+ open my $desc, '<', "$tmpdir/pfx/v2/description" or xbail "open: $!";
+ $desc = <$desc>;
+ is($desc, "a v2 inbox\n", 'v2 description retrieved');
+
+ $clone_err = '';
+ ok(run_script(['-clone', '--include=*/alt',
+ "http://$host:$port/pfx", "$tmpdir/incl" ],
+ undef, $opt), 'clone w/include') or diag "clone_err=$clone_err";
+ ok(-d "$tmpdir/incl/alt", 'alt cloned');
+ ok(!-d "$tmpdir/incl/v2" && !-d "$tmpdir/incl/bare", 'only alt cloned');
+ is(xqx([qw(git config -f), "$tmpdir/incl/alt/config", 'gitweb.owner']),
+ "lorelei \xc4\x80\n", 'gitweb.owner set by -clone');
+
+ $clone_err = '';
+ ok(run_script(['-clone', '--dry-run',
+ "http://$host:$port/pfx", "$tmpdir/dry-run" ],
+ undef, $opt), 'clone --dry-run') or diag "clone_err=$clone_err";
+ ok(!-d "$tmpdir/dry-run", 'nothing cloned with --dry-run');
+
+ undef $td;
+
+ open $mh, '<', "$tmpdir/incl/manifest.js.gz" or xbail "open: $!";
+ gunzip(\(do { local $/; <$mh> }) => \($mjs = ''));
+ $mf = $json->decode($mjs);
+ is_deeply([keys %$mf], [ '/alt' ], 'excluded keys skipped in manifest');
+
$td = start_script($cmd, $env, { 3 => $sock });
# default publicinboxGrokManifest match=domain default
tiny_test($json, $host, $port);
+
+ # normal full clone on /
+ $clone_err = '';
+ ok(run_script(['-clone', "http://$host:$port/", "$tmpdir/full" ],
+ undef, $opt), 'full clone') or diag "clone_err=$clone_err";
+ ok(-d "$tmpdir/full/$_", "$_ cloned") for qw(alt v2 bare);
+
undef $td;
print $fh <<"" or xbail "print $!";
close $fh or xbail "close $!";
$td = start_script($cmd, $env, { 3 => $sock });
- tiny_test($json, $host, $port, 1);
undef $sock;
+ tiny_test($json, $host, $port, 1);
- my $grok_pull = which('grok-pull') or
+ # grok-pull sleeps a long while some places:
+ # https://lore.kernel.org/tools/20211013110344.GA10632@dcvr/
+ skip 'TEST_GROK unset', 12 unless $ENV{TEST_GROK};
+ my $grok_pull = require_cmd('grok-pull', 1) or
skip('grok-pull not available', 12);
my ($grok_version) = (xqx([$grok_pull, "--version"])
=~ /(\d+)\.(?:\d+)(?:\.(\d+))?/);
$grok_version >= 2 or
skip('grok-pull v2 or later not available', 12);
+ my $grok_loglevel = $ENV{TEST_GROK_LOGLEVEL} // 'info';
ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest');
- open $fh, '>', "$tmpdir/repos.conf" or die;
- print $fh <<"" or die;
+ my $tail = tail_f("$tmpdir/grok.log");
+ open $fh, '>', "$tmpdir/repos.conf" or xbail $!;
+ print $fh <<"" or xbail $!;
[core]
toplevel = $tmpdir/mirror
manifest = $tmpdir/local-manifest.js.gz
+log = $tmpdir/grok.log
+loglevel = $grok_loglevel
[remote]
site = http://$host:$port
manifest = \${site}/manifest.js.gz
[pull]
[fsck]
- close $fh or die;
-
+ close $fh or xbail $!;
xsys($grok_pull, '-c', "$tmpdir/repos.conf");
is($? >> 8, 0, 'grok-pull exit code as expected');
for (qw(alt bare v2/git/0.git v2/git/1.git v2/git/2.git)) {
# support per-inbox manifests, handy for v2:
# /$INBOX/v2/manifest.js.gz
- open $fh, '>', "$tmpdir/per-inbox.conf" or die;
- print $fh <<"" or die;
+ open $fh, '>', "$tmpdir/per-inbox.conf" or xbail $!;
+ print $fh <<"" or xbail $!;
[core]
toplevel = $tmpdir/per-inbox
manifest = $tmpdir/per-inbox-manifest.js.gz
+log = $tmpdir/grok.log
+loglevel = $grok_loglevel
[remote]
site = http://$host:$port
manifest = \${site}/v2/manifest.js.gz
[pull]
[fsck]
- close $fh or die;
+ close $fh or xbail $!;
ok(mkdir("$tmpdir/per-inbox"), 'prepare single-v2-inbox mirror');
xsys($grok_pull, '-c', "$tmpdir/per-inbox.conf");
is($? >> 8, 0, 'grok-pull exit code as expected');