From: Eric Wong <e@80x24.org>
Date: Wed, 8 May 2019 19:24:07 +0000 (+0000)
Subject: Merge remote-tracking branch 'origin/danga-bundle'
X-Git-Tag: v1.2.0~296
X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=fe86054ee1e2d93b19762a34ed1aaebe43635ff0;hp=90c5a78c8f6239b27c32b816d1cd029196ba2fe2

Merge remote-tracking branch 'origin/danga-bundle'

* origin/danga-bundle:
  DS: epoll: fix misordered EPOLL_CTL_DEL call
  DS: drop unused "_undef" sub
  syscall: drop readahead wrapper
  build: do not manify DS and Syscall pods
  DS: handle EINTR in IO::Poll path, too
  DS: workaround IO::Kqueue EINTR (mis-)handling
  DS: drop profiling support
  DS: remove unused fields and functions
  listener: use EPOLLEXCLUSIVE for listen sockets
  bundle Danga::Socket and Sys::Syscall
---

diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index 29150220..d44c8f30 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -188,6 +188,14 @@ be treated as the default value.
 
 Default: 25
 
+=item publicinbox.<name>.hide
+
+A comma-delimited list of listings to hide the inbox from.
+
+Valid values are currently "www".
+
+Default: none
+
 =item coderepo.<nick>.dir
 
 The path to a git repository for "publicinbox.<name>.coderepo"
@@ -226,6 +234,35 @@ C<publicinbox.cgitbin>, but may be overridden.
 Default: basename of C<publicinbox.cgitbin>, /var/www/htdocs/cgit/
 or /usr/share/cgit/
 
+=item publicinbox.wwwlisting
+
+Enable a HTML listing style when the root path of the URL '/' is accessed.
+Valid values are:
+
+=over 8
+
+=item all
+
+Show all inboxes
+
+=item 404
+
+Return a 404 page.  This is useful to allow customization with
+L<Plack::App::Cascade(3pm)>
+
+=item match=domain
+
+Only show inboxes with URLs which belong to the domain of the HTTP
+request
+
+=for TODO comment
+
+support showing cgit listing
+
+=back
+
+Default: 404
+
 =back
 
 =head2 NAMED LIMITER (PSGI)
diff --git a/INSTALL b/INSTALL
index 3c0b9108..fafaf577 100644
--- a/INSTALL
+++ b/INSTALL
@@ -22,7 +22,7 @@ public-inbox requires a number of other packages to access its full
 functionality.  The core tools are, of course:
 
 * Git (1.8.0+, 2.6+ for writing v2 repositories)
-* Perl 5.8+
+* Perl 5.10.1+
 * SQLite (needed for Xapian use)
 
 To accept incoming mail into a public inbox, you'll likely want:
diff --git a/MANIFEST b/MANIFEST
index afe5ae1c..da9e3645 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -131,6 +131,7 @@ lib/PublicInbox/WatchMaildir.pm
 lib/PublicInbox/WwwAtomStream.pm
 lib/PublicInbox/WwwAttach.pm
 lib/PublicInbox/WwwHighlight.pm
+lib/PublicInbox/WwwListing.pm
 lib/PublicInbox/WwwStream.pm
 lib/PublicInbox/WwwText.pm
 sa_config/Makefile
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 631c7880..09f9179b 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -389,7 +389,7 @@ sub _fill {
 	}
 	# TODO: more arrays, we should support multi-value for
 	# more things to encourage decentralization
-	foreach my $k (qw(address altid nntpmirror coderepo)) {
+	foreach my $k (qw(address altid nntpmirror coderepo hide)) {
 		if (defined(my $v = $self->{"$pfx.$k"})) {
 			$ibx->{$k} = _array($v);
 		}
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 0d28dd04..286555f6 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -95,6 +95,15 @@ sub new {
 	if (defined $dir && -f "$dir/inbox.lock") {
 		$opts->{version} = 2;
 	}
+
+	# allow any combination of multi-line or comma-delimited hide entries
+	my $hide = {};
+	if (defined(my $h = $opts->{hide})) {
+		foreach my $v (@$h) {
+			$hide->{$_} = 1 foreach (split(/\s*,\s*/, $v));
+		}
+		$opts->{-hide} = $hide;
+	}
 	bless $opts, $class;
 }
 
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f756e92c..5c5df7b0 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -122,7 +122,7 @@ sub args_ok ($$) {
 sub process_line ($$) {
 	my ($self, $l) = @_;
 	my ($req, @args) = split(/\s+/, $l);
-	return unless defined($req);
+	return 1 unless defined($req); # skip blank line
 	$req = lc($req);
 	$req = eval {
 		no strict 'refs';
diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxPart.pm
index 7fe2120a..51d81a0a 100644
--- a/lib/PublicInbox/SearchIdxPart.pm
+++ b/lib/PublicInbox/SearchIdxPart.pm
@@ -48,8 +48,15 @@ sub spawn_worker {
 sub partition_worker_loop ($$$$) {
 	my ($self, $r, $part, $bnote) = @_;
 	$0 = "pi-v2-partition[$part]";
+	my $current_info = '';
+	my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
+	local $SIG{__WARN__} = sub {
+		chomp $current_info;
+		$warn_cb->("[$part] $current_info: ", @_);
+	};
 	$self->begin_txn_lazy;
 	while (my $line = $r->getline) {
+		$current_info = $line;
 		if ($line eq "commit\n") {
 			$self->commit_txn_lazy;
 		} elsif ($line eq "close\n") {
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 7b0f3bdd..66b916df 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -26,22 +26,35 @@ my $vfork_spawn = <<'VFORK_SPAWN';
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <unistd.h>
-#include <alloca.h>
-#include <signal.h>
-#include <assert.h>
+#include <stdlib.h>
 
-#define AV_ALLOCA(av, max) alloca((max = (av_len((av)) + 1)) * sizeof(char *))
+/* some platforms need alloca.h, but some don't */
+#if defined(__GNUC__) && !defined(alloca)
+#  define alloca(sz) __builtin_alloca(sz)
+#endif
 
-static void av2c_copy(char **dst, AV *src, I32 max)
-{
-	I32 i;
+#include <signal.h>
+#include <assert.h>
 
-	for (i = 0; i < max; i++) {
-		SV **sv = av_fetch(src, i, 0);
-		dst[i] = sv ? SvPV_nolen(*sv) : 0;
-	}
-	dst[max] = 0;
-}
+/*
+ * From the av_len apidoc:
+ *   Note that, unlike what the name implies, it returns
+ *   the highest index in the array, so to get the size of
+ *   the array you need to use "av_len(av) + 1".
+ *   This is unlike "sv_len", which returns what you would expect.
+ */
+#define AV2C_COPY(dst, src) do { \
+	I32 i; \
+	I32 top_index = av_len(src); \
+	I32 real_len = top_index + 1; \
+	I32 capa = real_len + 1; \
+	dst = alloca(capa * sizeof(char *)); \
+	for (i = 0; i < real_len; i++) { \
+		SV **sv = av_fetch(src, i, 0); \
+		dst[i] = SvPV_nolen(*sv); \
+	} \
+	dst[real_len] = 0; \
+} while (0)
 
 static void *deconst(const char *s)
 {
@@ -86,15 +99,11 @@ int pi_fork_exec(int in, int out, int err,
 	const char *filename = SvPV_nolen(file);
 	pid_t pid;
 	char **argv, **envp;
-	I32 max;
 	sigset_t set, old;
 	int ret, errnum;
 
-	argv = AV_ALLOCA(cmd, max);
-	av2c_copy(argv, cmd, max);
-
-	envp = AV_ALLOCA(env, max);
-	av2c_copy(envp, env, max);
+	AV2C_COPY(argv, cmd);
+	AV2C_COPY(envp, env);
 
 	ret = sigfillset(&set);
 	assert(ret == 0 && "BUG calling sigfillset");
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 6829a343..87e8f3eb 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -72,6 +72,7 @@ sub new {
 		im => undef, #  PublicInbox::Import
 		parallel => 1,
 		transact_bytes => 0,
+		current_info => '',
 		xpfx => $xpfx,
 		over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3", 1),
 		lock_path => "$dir/inbox.lock",
@@ -949,8 +950,10 @@ sub index_sync {
 		my $fh = $self->{reindex_pipe} = $git->popen(@cmd, $range);
 		my $cmt;
 		while (<$fh>) {
+			chomp;
+			$self->{current_info} = "$i.git $_";
 			if (/\A$x40$/o && !defined($cmt)) {
-				chomp($cmt = $_);
+				$cmt = $_;
 			} elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) {
 				$self->reindex_oid($mm_tmp, $D, $git, $1,
 						$regen, $reindex);
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 6e69001c..1f3ca157 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -11,7 +11,7 @@
 # - Must not rely on static content
 # - UTF-8 is only for user-content, 7-bit US-ASCII for us
 package PublicInbox::WWW;
-use 5.008;
+use 5.010_001;
 use strict;
 use warnings;
 use bytes (); # only for bytes::length
@@ -68,8 +68,9 @@ sub call {
 	} split(/[&;]+/, $env->{QUERY_STRING});
 	$ctx->{qp} = \%qp;
 
-	# not using $env->{PATH_INFO} here since that's already decoded
+	# avoiding $env->{PATH_INFO} here since that's already decoded
 	my ($path_info) = ($env->{REQUEST_URI} =~ path_re($env));
+	$path_info //= $env->{PATH_INFO};
 	my $method = $env->{REQUEST_METHOD};
 
 	if ($method eq 'POST') {
@@ -87,7 +88,7 @@ sub call {
 
 	# top-level indices and feeds
 	if ($path_info eq '/') {
-		r404();
+		www_listing($self)->call($env);
 	} elsif ($path_info =~ m!$INBOX_RE\z!o) {
 		invalid_inbox($ctx, $1) || r301($ctx, $1);
 	} elsif ($path_info =~ m!$INBOX_RE(?:/|/index\.html)?\z!o) {
@@ -157,6 +158,7 @@ sub preload {
 	if (ref($self)) {
 		$self->cgit;
 		$self->stylesheets_prepare($_) for ('', '../', '../../');
+		$self->www_listing;
 	}
 }
 
@@ -489,6 +491,14 @@ sub cgit {
 	}
 }
 
+sub www_listing {
+	my ($self) = @_;
+	$self->{www_listing} ||= do {
+		require PublicInbox::WwwListing;
+		PublicInbox::WwwListing->new($self);
+	}
+}
+
 sub get_attach {
 	my ($ctx, $idx, $fn) = @_;
 	require PublicInbox::WwwAttach;
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
new file mode 100644
index 00000000..e8dad4b8
--- /dev/null
+++ b/lib/PublicInbox/WwwListing.pm
@@ -0,0 +1,104 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Provide an HTTP-accessible listing of inboxes.
+# Used by PublicInbox::WWW
+package PublicInbox::WwwListing;
+use strict;
+use warnings;
+use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::Linkify;
+use PublicInbox::View;
+
+sub list_all ($$) {
+	my ($self, undef) = @_;
+	my @list;
+	$self->{pi_config}->each_inbox(sub {
+		my ($ibx) = @_;
+		push @list, $ibx unless $ibx->{-hide}->{www};
+	});
+	\@list;
+}
+
+sub list_match_domain ($$) {
+	my ($self, $env) = @_;
+	my @list;
+	my $host = $env->{HTTP_HOST} // $env->{SERVER_NAME};
+	$host =~ s/:\d+\z//;
+	my $re = qr!\A(?:https?:)?//\Q$host\E(?::\d+)?/!i;
+	$self->{pi_config}->each_inbox(sub {
+		my ($ibx) = @_;
+		push @list, $ibx if !$ibx->{-hide}->{www} && $ibx->{url} =~ $re;
+	});
+	\@list;
+}
+
+sub list_404 ($$) { [] }
+
+# TODO: +cgit
+my %VALID = (
+	all => *list_all,
+	'match=domain' => *list_match_domain,
+	404 => *list_404,
+);
+
+sub new {
+	my ($class, $www) = @_;
+	my $k = 'publicinbox.wwwListing';
+	my $pi_config = $www->{pi_config};
+	my $v = $pi_config->{lc($k)} // 404;
+	bless {
+		pi_config => $pi_config,
+		style => $www->style("\0"),
+		list_cb => $VALID{$v} || do {
+			warn <<"";
+`$v' is not a valid value for `$k'
+$k be one of `all', `match=domain', or `404'
+
+			*list_404;
+		},
+	}, $class;
+}
+
+sub ibx_entry {
+	my ($mtime, $ibx, $env) = @_;
+	my $ts = PublicInbox::View::fmt_ts($mtime);
+	my $url = PublicInbox::Hval::prurl($env, $ibx->{url});
+	my $tmp = <<"";
+* $ts - $url
+  ${\$ibx->description}
+
+	if (defined(my $info_url = $ibx->{info_url})) {
+		$tmp .= "\n$info_url";
+	}
+	$tmp;
+}
+
+# not really a stand-alone PSGI app, but maybe it could be...
+sub call {
+	my ($self, $env) = @_;
+	my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ];
+	my $list = $self->{list_cb}->($self, $env);
+	my $code = 404;
+	my $title = 'public-inbox';
+	my $out = '';
+	if (@$list) {
+		# Swartzian transform since ->modified is expensive
+		@$list = sort {
+			$b->[0] <=> $a->[0]
+		} map { [ $_->modified, $_ ] } @$list;
+
+		$code = 200;
+		$title .= ' - listing';
+		my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list);
+		my $l = PublicInbox::Linkify->new;
+		$l->linkify_1($tmp);
+		$out = '<pre>'.$l->linkify_2(ascii_html($tmp)).'</pre><hr>';
+	}
+	$out = "<html><head><title>$title</title></head><body>" . $out;
+	$out .= '<pre>'. PublicInbox::WwwStream::code_footer($env) .
+		'</pre></body></html>';
+	[ $code, $h, [ $out ] ]
+}
+
+1;
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index ea7aaad0..f6c50496 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -10,7 +10,6 @@ package PublicInbox::WwwStream;
 use strict;
 use warnings;
 use PublicInbox::Hval qw(ascii_html);
-use URI;
 our $TOR_URL = 'https://www.torproject.org/';
 our $CODE_URL = 'https://public-inbox.org/';
 our $PROJECT = 'public-inbox';
@@ -70,6 +69,12 @@ sub _html_top ($) {
 		"</head><body>". $top . $tip;
 }
 
+sub code_footer ($) {
+	my ($env) = @_;
+	my $u = PublicInbox::Hval::prurl($env, $CODE_URL);
+	qq(AGPL code for this site: git clone <a\nhref="$u">$u</a> $PROJECT)
+}
+
 sub _html_end {
 	my ($self) = @_;
 	my $urls = 'Archives are clonable:';
@@ -134,12 +139,10 @@ EOF
 		$urls .= "\n note: .onion URLs require Tor: ";
 		$urls .= qq[<a\nhref="$TOR_URL">$TOR_URL</a>];
 	}
-	my $url = PublicInbox::Hval::prurl($ctx->{env}, $CODE_URL);
 	'<hr><pre>'.join("\n\n",
 		$desc,
 		$urls,
-		'AGPL code for this site: '.
-		qq(git clone <a\nhref="$url">$url</a> $PROJECT)
+		code_footer($ctx->{env})
 	).'</pre></body></html>';
 }
 
diff --git a/script/public-inbox-index b/script/public-inbox-index
index 5adb6e74..2f810a56 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -85,6 +85,9 @@ sub index_dir {
 				}
 			}
 		}
+		local $SIG{__WARN__} = sub {
+			print STDERR $v2w->{current_info}, ': ', @_;
+		};
 		$v2w->index_sync({ reindex => $reindex, prune => $prune });
 	} else {
 		my $s = PublicInbox::SearchIdx->new($repo, 1);
diff --git a/t/search.t b/t/search.t
index 6415a644..35d71473 100644
--- a/t/search.t
+++ b/t/search.t
@@ -430,13 +430,23 @@ $ibx->with_umask(sub {
 	is($ro->lookup_article($art->{num}), undef, 'gone from OVER DB') if defined($art);
 });
 
+my $all_mask = 07777;
+my $dir_mask = 02770;
+
+# FreeBSD does not allow non-root users to set S_ISGID, so
+# git doesn't set it, either (see DIR_HAS_BSD_GROUP_SEMANTICS in git.git)
+if ($^O =~ /freebsd/i) {
+	$all_mask = 0777;
+	$dir_mask = 0770;
+}
+
 foreach my $f ("$git_dir/public-inbox/msgmap.sqlite3",
 		"$git_dir/public-inbox",
 		glob("$git_dir/public-inbox/xapian*/"),
 		glob("$git_dir/public-inbox/xapian*/*")) {
 	my @st = stat($f);
 	my ($bn) = (split(m!/!, $f))[-1];
-	is($st[2] & 07777, -f _ ? 0660 : 02770,
+	is($st[2] & $all_mask, -f _ ? 0660 : $dir_mask,
 		"sharedRepository respected for $bn");
 }
 
diff --git a/t/v2writable.t b/t/v2writable.t
index 06b22519..f8ef415a 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -253,10 +253,9 @@ EOF
 }
 
 {
-	my @warn;
 	my $x = 'x'x250;
 	my $y = 'y'x250;
-	local $SIG{__WARN__} = sub { push @warn, @_ };
+	local $SIG{__WARN__} = sub {};
 	$mime->header_set('Subject', 'long mid');
 	$mime->header_set('Message-ID', "<$x>");
 	ok($im->add($mime), 'add excessively long Message-ID');