MANIFEST | 1 + lib/PublicInbox/Git.pm | 2 +- lib/PublicInbox/WWW.pm | 17 ++++++++++++++--- lib/PublicInbox/WwwListing.pm | 174 +++++++++++++++++++++++++++++++++++++++++++++-------- t/psgi_v2.t | 2 ++ t/v2mirror.t | 8 +++++--- t/www_listing.t | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/MANIFEST b/MANIFEST index a44632a98151cfa25bc39efdad2872f097ed3ddc..ae637f24aa91f1d7e569e4f798348102d8bc462f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -263,3 +263,4 @@ t/view.t t/watch_filter_rubylang.t t/watch_maildir.t t/watch_maildir_v2.t +t/www_listing.t diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 6a87661c290faaa4ea0275f70a1fe23bf15898ba..f5c7a95c34d7aedad7860a1189f8e121167eb1fb 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -303,7 +303,7 @@ foreach my $oid (<$fh>) { chomp $oid; my $buf = cat_file($self, $oid) or next; $$buf =~ /^committer .*?> ([0-9]+) [\+\-]?[0-9]+/sm or next; - my $cmt_time = $1; + my $cmt_time = $1 + 0; $modified = $cmt_time if $cmt_time > $modified; } $modified || time; diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 7ea982041b7daede1c9ddb567995136f12fa7a50..e468263699bbcdd13af2e8efeb020f743b06907b 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -74,7 +74,7 @@ $path_info //= $env->{PATH_INFO}; my $method = $env->{REQUEST_METHOD}; if ($method eq 'POST') { - if ($path_info =~ m!$INBOX_RE/(?:([0-9]+)/)? + if ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)? (git-upload-pack)\z!x) { my ($part, $path) = ($2, $3); return invalid_inbox($ctx, $1) || @@ -88,7 +88,7 @@ return r(405, 'Method Not Allowed'); } # top-level indices and feeds - if ($path_info eq '/') { + if ($path_info eq '/' || $path_info eq '/manifest.js.gz') { www_listing($self)->call($env); } elsif ($path_info =~ m!$INBOX_RE\z!o) { invalid_inbox($ctx, $1) || r301($ctx, $1); @@ -98,7 +98,7 @@ } elsif ($path_info =~ m!$INBOX_RE/(?:atom\.xml|new\.atom)\z!o) { invalid_inbox($ctx, $1) || get_atom($ctx); } elsif ($path_info =~ m!$INBOX_RE/new\.html\z!o) { invalid_inbox($ctx, $1) || get_new($ctx); - } elsif ($path_info =~ m!$INBOX_RE/(?:([0-9]+)/)? + } elsif ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)? ($PublicInbox::GitHTTPBackend::ANY)\z!ox) { my ($part, $path) = ($2, $3); invalid_inbox($ctx, $1) || serve_git($ctx, $part, $path); @@ -126,6 +126,8 @@ } elsif ($path_info =~ m!$INBOX_RE/_/text(?:/(.*))?\z!o) { get_text($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/([a-zA-Z0-9_\-\.]+)\.css\z!o) { get_css($ctx, $1, $2); + } elsif ($path_info =~ m!$INBOX_RE/manifest\.js\.gz\z!o) { + get_inbox_manifest($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s/\z!o) { get_vcs_object($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s/ @@ -488,6 +490,15 @@ $self->{www_listing} ||= do { require PublicInbox::WwwListing; PublicInbox::WwwListing->new($self); } +} + +# GET $INBOX/manifest.js.gz +sub get_inbox_manifest ($$$) { + my ($ctx, $inbox, $key) = @_; + my $r404 = invalid_inbox($ctx, $inbox); + return $r404 if $r404; + require PublicInbox::WwwListing; + PublicInbox::WwwListing::js($ctx->{env}, [$ctx->{-inbox}]); } sub get_attach { diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm index e1473b3deec7135b4aa9557a65371ce9dd469d7a..e2724cc49e631dc63a38dc26410d0b6a9577707b 100644 --- a/lib/PublicInbox/WwwListing.pm +++ b/lib/PublicInbox/WwwListing.pm @@ -9,26 +9,33 @@ use warnings; use PublicInbox::Hval qw(ascii_html); use PublicInbox::Linkify; use PublicInbox::View; +use bytes (); +use HTTP::Date qw(time2str); +require Digest::SHA; +require File::Spec; +{ no warnings 'once'; *try_cat = *PublicInbox::Inbox::try_cat }; -sub list_all ($$) { - my ($self, undef) = @_; +sub list_all ($$$) { + my ($self, $env, $hide_key) = @_; my @list; $self->{pi_config}->each_inbox(sub { my ($ibx) = @_; - push @list, $ibx unless $ibx->{-hide}->{www}; + push @list, $ibx unless $ibx->{-hide}->{$hide_key}; }); \@list; } -sub list_match_domain ($$) { - my ($self, $env) = @_; +sub list_match_domain ($$$) { + my ($self, $env, $hide_key) = @_; my @list; my $host = $env->{HTTP_HOST} // $env->{SERVER_NAME}; $host =~ s/:[0-9]+\z//; my $re = qr!\A(?:https?:)?//\Q$host\E(?::[0-9]+)?/!i; $self->{pi_config}->each_inbox(sub { my ($ibx) = @_; - push @list, $ibx if !$ibx->{-hide}->{www} && $ibx->{url} =~ $re; + if (!$ibx->{-hide}->{$hide_key} && $ibx->{url} =~ $re) { + push @list, $ibx; + } }); \@list; } @@ -42,21 +49,27 @@ 'match=domain' => *list_match_domain, 404 => *list_404, ); +sub set_cb ($$$) { + my ($pi_config, $k, $default) = @_; + my $v = $pi_config->{lc $k} // $default; + $VALID{$v} || do { + warn <<""; +`$v' is not a valid value for `$k' +$k be one of `all', `match=domain', or `404' + + $VALID{$default}; + }; +} + sub new { my ($class, $www) = @_; - my $k = 'publicinbox.wwwListing'; my $pi_config = $www->{pi_config}; - my $v = $pi_config->{lc($k)} // 404; bless { pi_config => $pi_config, style => $www->style("\0"), - list_cb => $VALID{$v} || do { - warn <<""; -`$v' is not a valid value for `$k' -$k be one of `all', `match=domain', or `404' - - *list_404; - }, + www_cb => set_cb($pi_config, 'publicInbox.wwwListing', 404), + manifest_cb => set_cb($pi_config, 'publicInbox.grokManifest', + 'match=domain'), }, $class; } @@ -74,22 +87,20 @@ } $tmp; } -# not really a stand-alone PSGI app, but maybe it could be... -sub call { - my ($self, $env) = @_; - my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ]; - my $list = $self->{list_cb}->($self, $env); - my $code = 404; +sub html ($$) { + my ($env, $list) = @_; my $title = 'public-inbox'; my $out = ''; + my $code = 404; if (@$list) { + $title .= ' - listing'; + $code = 200; + # Swartzian transform since ->modified is expensive @$list = sort { $b->[0] <=> $a->[0] } map { [ $_->modified, $_ ] } @$list; - $code = 200; - $title .= ' - listing'; my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list); my $l = PublicInbox::Linkify->new; $l->linkify_1($tmp); @@ -98,7 +109,122 @@ } $out = "$title" . $out; $out .= '
'. PublicInbox::WwwStream::code_footer($env) .
 		'
'; - [ $code, $h, [ $out ] ] + + my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ]; + [ $code, $h, [ $out ] ]; +} + +my $json; +sub _json () { + for my $mod (qw(JSON::MaybeXS JSON JSON::PP)) { + eval "require $mod" or next; + # ->ascii encodes non-ASCII to "\uXXXX" + return $mod->new->ascii(1); + } + die; +} + +sub fingerprint ($) { + my ($git) = @_; + my $fh = $git->popen('show-ref') or + die "popen($git->{git_dir} show-ref) failed: $!"; + + my $dig = Digest::SHA->new(1); + while (read($fh, my $buf, 65536)) { + $dig->add($buf); + } + close $fh; + return if $?; # empty, uninitialized git repo + $dig->hexdigest; +} + +sub manifest_add ($$;$) { + my ($manifest, $ibx, $epoch) = @_; + my $url_path = "/$ibx->{name}"; + my $git_dir = $ibx->{mainrepo}; + if (defined $epoch) { + $git_dir .= "/git/$epoch.git"; + $url_path .= "/git/$epoch.git"; + } + return unless -d $git_dir; + my $git = PublicInbox::Git->new($git_dir); + my $fingerprint = fingerprint($git) or return; # no empty repos + + chomp(my $owner = $git->qx('config', 'gitweb.owner')); + chomp(my $desc = try_cat("$git_dir/description")); + $owner = undef if $owner eq ''; + $desc = 'Unnamed repository' if $desc eq ''; + + my $reference; + chomp(my $alt = try_cat("$git_dir/objects/info/alternates")); + if ($alt) { + # n.b.: GitPython doesn't seem to handle comments or C-quoted + # strings like native git does; and we don't for now, either. + my @alt = split(/\n+/, $alt); + + # grokmirror only supports 1 alternate for "reference", + if (scalar(@alt) == 1) { + my $objdir = "$git_dir/objects"; + $reference = File::Spec->rel2abs($alt[0], $objdir); + $reference =~ s!/[^/]+/?\z!!; # basename + } + } + $manifest->{-abs2urlpath}->{$git_dir} = $url_path; + my $modified = $git->modified; + if ($modified > $manifest->{-mtime}) { + $manifest->{-mtime} = $modified; + } + $manifest->{$url_path} = { + owner => $owner, + reference => $reference, + description => $desc, + modified => $modified, + fingerprint => $fingerprint, + }; +} + +# manifest.js.gz +sub js ($$) { + my ($env, $list) = @_; + eval { require IO::Compress::Gzip } or return [ 404, [], [] ]; + + my $manifest = { -abs2urlpath => {}, -mtime => 0 }; + for my $ibx (@$list) { + if (defined(my $max = $ibx->max_git_part)) { + for my $epoch (0..$max) { + manifest_add($manifest, $ibx, $epoch); + } + } else { + manifest_add($manifest, $ibx); + } + } + my $abs2urlpath = delete $manifest->{-abs2urlpath}; + my $mtime = delete $manifest->{-mtime}; + while (my ($url_path, $repo) = each %$manifest) { + defined(my $abs = $repo->{reference}) or next; + $repo->{reference} = $abs2urlpath->{$abs}; + } + my $out; + IO::Compress::Gzip::gzip(\(($json ||= _json())->encode($manifest)) => + \$out); + $manifest = undef; + [ 200, [ qw(Content-Type application/gzip), + 'Last-Modified', time2str($mtime), + 'Content-Length', bytes::length($out) ], [ $out ] ]; +} + +# not really a stand-alone PSGI app, but maybe it could be... +sub call { + my ($self, $env) = @_; + + if ($env->{PATH_INFO} eq '/manifest.js.gz') { + # grokmirror uses relative paths, so it's domain-dependent + my $list = $self->{manifest_cb}->($self, $env, 'manifest'); + js($env, $list); + } else { # / + my $list = $self->{www_cb}->($self, $env, 'www'); + html($env, $list); + } } 1; diff --git a/t/psgi_v2.t b/t/psgi_v2.t index 98112494b32cf90c202c02d103659961fd5fe6ad..5c358cdecb591720c785cce6ecc6a8fd1a718c96 100644 --- a/t/psgi_v2.t +++ b/t/psgi_v2.t @@ -202,6 +202,8 @@ like($raw, qr/\b4\+ messages\b/, 'thread overview shown with /t/'); $res = $cb->(GET('/v2test/0/info/refs')); is($res->code, 200, 'got info refs for dumb clones'); + $res = $cb->(GET('/v2test/0.git/info/refs')); + is($res->code, 200, 'got info refs for dumb clones w/ .git suffix'); $res = $cb->(GET('/v2test/info/refs')); is($res->code, 404, 'unpartitioned git URL fails'); diff --git a/t/v2mirror.t b/t/v2mirror.t index fe05ec4d2c30916a6a01d424571e944080f7ca0f..c31dcd5bf7cdc2ed6c8df36ab89656eb70f8d7d2 100644 --- a/t/v2mirror.t +++ b/t/v2mirror.t @@ -80,11 +80,13 @@ $sock = undef; my @cmd; foreach my $i (0..$epoch_max) { - @cmd = (qw(git clone --mirror -q), "http://$host:$port/v2/$i", + my $sfx = $i == 0 ? '.git' : ''; + @cmd = (qw(git clone --mirror -q), + "http://$host:$port/v2/$i$sfx", "$tmpdir/m/git/$i.git"); - is(system(@cmd), 0, 'cloned OK'); - ok(-d "$tmpdir/m/git/$i.git", 'mirror OK'); + is(system(@cmd), 0, "cloned $i.git"); + ok(-d "$tmpdir/m/git/$i.git", "mirror $i OK"); } @cmd = ("$script-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m', diff --git a/t/www_listing.t b/t/www_listing.t new file mode 100644 index 0000000000000000000000000000000000000000..1f2929808f54c21ab1dd73ddda4a7f600608029a --- /dev/null +++ b/t/www_listing.t @@ -0,0 +1,159 @@ +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# manifest.js.gz generation and grok-pull integration test +use strict; +use warnings; +use Test::More; +use PublicInbox::Spawn qw(which); +use File::Temp qw/tempdir/; +require './t/common.perl'; +my @mods = qw(URI::Escape Plack::Builder IPC::Run Digest::SHA HTTP::Tiny + IO::Compress::Gzip IO::Uncompress::Gunzip Net::HTTP); +foreach my $mod (@mods) { + eval("require $mod") or plan skip_all => "$mod missing for $0"; +} +use_ok 'PublicInbox::WwwListing'; +use_ok 'PublicInbox::Git'; + +my $fi_data = './t/git.fast-import-data'; +my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $bare = PublicInbox::Git->new("$tmpdir/bare.git"); +is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare'); +is(PublicInbox::WwwListing::fingerprint($bare), undef, + 'empty repo has no fingerprint'); + +my $cmd = [ 'git', "--git-dir=$bare->{git_dir}", qw(fast-import --quiet) ]; +ok(IPC::Run::run($cmd, '<', $fi_data), 'fast-import'); + +like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/, + 'got fingerprint with non-empty repo'); + +my $pid; +END { kill 'TERM', $pid if defined $pid }; +SKIP: { + my $json = eval { PublicInbox::WwwListing::_json() }; + skip "JSON module missing: $@", 1 if $@; + my $err = "$tmpdir/stderr.log"; + my $out = "$tmpdir/stdout.log"; + my $alt = "$tmpdir/alt.git"; + my $cfgfile = "$tmpdir/config"; + my $v2 = "$tmpdir/v2"; + my $httpd = 'blib/script/public-inbox-httpd'; + use IO::Socket::INET; + my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => 1024, + ); + my $sock = IO::Socket::INET->new(%opts); + ok($sock, 'sock created'); + my ($host, $port) = ($sock->sockhost, $sock->sockport); + my @clone = qw(git clone -q -s --bare); + is(system(@clone, $bare->{git_dir}, $alt), 0, 'clone shared repo'); + + for my $i (0..2) { + is(system(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i"); + } + ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox'); + open $fh, '>', "$alt/description" or die; + print $fh "we're all clones\n" or die; + close $fh or die; + is(system('git', "--git-dir=$alt", qw(config gitweb.owner lorelei)), 0, + 'set gitweb user'); + ok(unlink("$bare->{git_dir}/description"), 'removed bare/description'); + open $fh, '>', $cfgfile or die; + print $fh <<"" or die; +[publicinbox "bare"] + mainrepo = $bare->{git_dir} + url = http://$host/bare + address = bare\@example.com +[publicinbox "alt"] + mainrepo = $alt + url = http://$host/alt + address = alt\@example.com +[publicinbox "v2"] + mainrepo = $v2 + url = http://$host/v2 + address = v2\@example.com + + close $fh or die; + my $env = { PI_CONFIG => $cfgfile }; + my $cmd = [ $httpd, "--stdout=$out", "--stderr=$err" ]; + $pid = spawn_listener($env, $cmd, [$sock]); + $sock = undef; + my $http = Net::HTTP->new(Host => "$host:$port"); + $http->write_request(GET => '/manifest.js.gz'); + my ($code, undef, %h) = $http->read_response_headers; + is($code, 200, 'got manifest'); + my $tmp; + my $body = ''; + while (1) { + my $n = $http->read_entity_body(my $buf, 65536); + die unless defined $n; + last if $n == 0; + $body .= $buf; + } + IO::Uncompress::Gunzip::gunzip(\$body => \$tmp); + unlike($tmp, qr/"modified":\s*"/, 'modified is an integer'); + my $manifest = $json->decode($tmp); + ok(my $clone = $manifest->{'/alt'}, '/alt in manifest'); + is($clone->{owner}, 'lorelei', 'owner set'); + is($clone->{reference}, '/bare', 'reference detected'); + is($clone->{description}, "we're all clones", 'description read'); + ok(my $bare = $manifest->{'/bare'}, '/bare in manifest'); + is($bare->{description}, 'Unnamed repository', + 'missing $GIT_DIR/description fallback'); + + like($bare->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'fingerprint'); + is($clone->{fingerprint}, $bare->{fingerprint}, 'fingerprint matches'); + + is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'}, + 'modified field and Last-Modified header match'); + + ok($manifest->{'/v2/git/0.git'}, 'v2 epoch appeared'); + + skip 'skipping grok-pull integration test', 2 if !which('grok-pull'); + + ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest'); + open $fh, '>', "$tmpdir/repos.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[test] +site = http://$host:$port +manifest = http://$host:$port/manifest.js.gz +toplevel = $tmpdir/mirror +mymanifest = $tmpdir/local-manifest.js.gz + + close $fh or die; + + system(qw(grok-pull -c), "$tmpdir/repos.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(alt bare v2/git/0.git v2/git/1.git v2/git/2.git)) { + ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); + } + + # support per-inbox manifests, handy for v2: + # /$INBOX/v2/manifest.js.gz + open $fh, '>', "$tmpdir/per-inbox.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[v2] +site = http://$host:$port +manifest = http://$host:$port/v2/manifest.js.gz +toplevel = $tmpdir/per-inbox +mymanifest = $tmpdir/per-inbox-manifest.js.gz + + close $fh or die; + ok(mkdir("$tmpdir/per-inbox"), 'prepare single-v2-inbox mirror'); + system(qw(grok-pull -c), "$tmpdir/per-inbox.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(v2/git/0.git v2/git/1.git v2/git/2.git)) { + ok(-d "$tmpdir/per-inbox/$_", "grok-pull created $_"); + } +} + +done_testing();