From: Eric Wong (Contractor, The Linux Foundation) Date: Wed, 18 Apr 2018 20:58:35 +0000 (+0000) Subject: Merge remote-tracking branch 'origin/master' into v2 X-Git-Tag: v1.1.0-pre1~23 X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=cfb8d16578e7f2f2e300f9f436205e4a8fc7f322;hp=1dc0f0c10fd856166a6726034de8aec53392901a Merge remote-tracking branch 'origin/master' into v2 * origin/master: nntp: allow and ignore empty commands mbox: do not barf on queries which return no results nntp: fix NEWNEWS command searchview: fix non-numeric comparison Allow specification of the number of search results to return githttpbackend: avoid infinite loop on generic PSGI servers http: fix modification of read-only value extmsg: use news.gmane.org for Message-ID lookups extmsg: rework partial MID matching to favor current inbox Update the installation instructions with Fedora package names nntp: do not drain rbuf if there is a command pending nntp: improve fairness during XOVER and similar commands searchidx: do not modify Xapian DB while iterating Don't use LIMIT in UPDATE statements --- diff --git a/INSTALL b/INSTALL index 11d844cf..87aa6961 100644 --- a/INSTALL +++ b/INSTALL @@ -8,60 +8,117 @@ if they want to import mail into their personal inboxes. TODO: this still needs to be documented better, also see the scripts/ and sa_config/ directories in the source tree -It should also be possible to use public-inbox with only IMAP -(or even POP(!)) access to a mailbox. - -standard MakeMaker installation (Perl) --------------------------------------- - - perl Makefile.PL - make - make test - make install # root permissions may be needed - Requirements ------------ -* git -* Perl and several modules: (Debian package name) - - Date::Parse libtimedate-perl - - Email::MIME libemail-mime-perl - - Email::MIME::ContentType libemail-mime-contenttype-perl - - Encode::MIME::Header perl +public-inbox requires a number of other packages to access its full +functionality. The core tools are, of course: -Optional components: +* Git +* Perl +* SQLite (needed for Xapian use) + +To accept incoming mail into a public inbox, you'll likely want: * MTA - postfix is recommended (for public-inbox-mda) * SpamAssassin (spamc/spamd) (for public-inbox-watch/public-inbox-mda) -Optional Perl modules: - - - Plack[1] libplack-perl - - URI::Escape[1] liburi-perl - - Search::Xapian[2][3] libsearch-xapian-perl - - IO::Compress::Gzip[3] perl-modules (or libio-compress-perl) - - DBI[3] libdbi-perl - - DBD::SQLite[2][3] libdbd-sqlite3-perl - - Danga::Socket[4] libdanga-socket-perl - - Net::Server[5] libnet-server-perl - - Filesys::Notify::Simple[6] libfilesys-notify-simple-perl - - Inline::C[7] libinline-c-perl - - Plack::Middleware::ReverseProxy[8] libplack-middleware-reverseproxy-perl - - Plack::Middleware::Deflater[8] libplack-middleware-deflater-perl - -[1] - Optional, needed for serving/generating Atom and HTML pages -[2] - Optional, only required for NNTP server -[3] - Optional, needed for gzipped mbox support over HTTP -[4] - Optional, needed for bundled HTTP and NNTP servers -[5] - Optional, needed for standalone daemonization of HTTP+NNTP servers -[6] - Optional, needed for public-inbox-watch Maildir watcher -[7] - Optional, allows speeds up spawning on Linux (see public-inbox-daemon(8)) -[8] - Optional, recommended for PSGI interface +Beyond that, there is a long list of Perl modules required, starting with: + +* Date::Parse deb: libdatetime-perl + rpm: perl-Time-ParseDate + +* Email::MIME deb: libemail-mime-perl + rpm: perl-Email-MIME + +* Email::MIME::ContentType deb: libemail-mime-contenttype-perl + rpm: perl-Email-MIME-ContentType + +* Encode::MIME::Header deb: libencode-perl + rpm: perl-Encode + +Where "deb" indicates package names for Debian-derived distributions and +"rpm" is for RPM-based distributions (only known to work on Fedora). + +Numerous optional modules are likely to be useful as well: + + - Plack deb: libplack-perl + rpm: perl-Plack, perl-Plack-Test, + (for HTML/Atom generation) + + - URI::Escape deb: liburi-perl + rpm: perl-URI + (for HTML/Atom generation) + + - Search::Xapian deb: libsearch-xapian-perl + rpm: perl-Search-Xapian + (for NNTP service or gzipped mbox over HTTP) + + - IO::Compress::Gzip deb: perl-modules (or libio-compress-perl) + rpm: perl-PerlIO-gzip + (for gzipped mbox over HTTP) + + - DBI deb: libdbi-perl + rpm: perl-DBI + (for gzipped mbox over HTTP) + + - DBD::SQLite deb: libdbd-sqlite3-perl + rpm: perl-DBD-SQLite + (for NNTP service or gzipped mbox over HTTP) + + - Danga::Socket deb: libdanga-socket-perl + rpm: perl-Danga-Socket + (for bundled HTTP and NNTP servers) + + - Net::Server deb: libnet-server-perl + rpm: perl-Net-Server + (for HTTP/NNTP servers as standalone daemons) + + - Filesys::Notify::Simple deb: libfilesys-notify-simple-perl + rpm: perl-Filesys-Notify-Simple + (for public-inbox-watch) + + - Inline::C[7] deb: libinline-c-perl + (speeds up spawning on Linux + (see public-inbox-daemon(8)) + + - Plack::Middleware::ReverseProxy + + deb: libplack-middleware-reverseproxy-perl + rpm: perl-Plack-Middleware-ReverseProxy + (ensures redirects are correct when running + behind nginx or Varnish) + + - Plack::Middleware::Deflater + + deb: libplack-middleware-deflater-perl + rpm: perl-Plack-Middleware-Deflater + (saves bandwidth on responses) + + +On Fedora systems, you'll probably also end up wanting +perl-Test-HTTP-Server-Simple, perl-Devel-Peek, and perl-IPC-Run to run the +test suite. On Debian systems, libxml-feed-perl and libipc-run-perl(*) +will aid in running the test suite (XML::Feed and IPC::Run respectively, +on CPAN). + +(*) we hope to drop this dependency someday + +standard MakeMaker installation (Perl) +-------------------------------------- + +Once the dependencies are installed, you should be able to build and +install the system (into /usr/local) with: + + perl Makefile.PL + make + make test + make install # root permissions may be needed When installing Search::Xapian, make sure the underlying Xapian installation is not affected by an index corruption bug: - https://bugs.debian.org/808610 + https://bugs.debian.org/808610 For Debian 8.x (jessie), this means using Debian 8.5 or later. @@ -69,13 +126,8 @@ public-inbox will never store unregeneratable data in Xapian or any other search database we might use; Xapian corruption will not destroy critical data. -Optional Perl modules (for developers): - - - XML::Feed[9] libxml-feed-perl - - IPC::Run[10] libipc-run-perl - -[9] - Optional, for testing Atom feeds -[10] - Optional, for some tests (we hope to drop this dependency someday) +See the public-inbox-overview(7) man page for the next steps once the +installation is complete. Copyright --------- diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm index a6f516df..04cb4062 100644 --- a/lib/PublicInbox/ExtMsg.pm +++ b/lib/PublicInbox/ExtMsg.pm @@ -17,7 +17,7 @@ our @EXT_URL = ( # leading "//" denotes protocol-relative (http:// or https://) '//marc.info/?i=%s', '//www.mail-archive.com/search?l=mid&q=%s', - 'http://mid.gmane.org/%s', + 'nntp://news.gmane.org/%s', 'https://lists.debian.org/msgid-search/%s', '//docs.FreeBSD.org/cgi/mid.cgi?db=mid&id=%s', 'https://www.w3.org/mid/%s', diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 3237a5ed..ec3d4f9d 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -78,7 +78,7 @@ sub meta_accessor { $prev = $dbh->selectrow_array($sql, undef, $key); if (defined $prev) { - $sql = 'UPDATE meta SET val = ? WHERE key = ? LIMIT 1'; + $sql = 'UPDATE meta SET val = ? WHERE key = ?'; $dbh->do($sql, undef, $value, $key); } else { $sql = 'INSERT INTO meta (key,val) VALUES (?,?)'; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index ace56e7a..cdbd8e98 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -115,6 +115,7 @@ sub args_ok ($$) { sub process_line ($$) { my ($self, $l) = @_; my ($req, @args) = split(/\s+/, $l); + return unless defined($req); $req = lc($req); $req = eval { no strict 'refs'; @@ -943,11 +944,13 @@ sub event_write { sub event_read { my ($self) = @_; use constant LINE_MAX => 512; # RFC 977 section 2.3 - my $r = 1; - my $buf = $self->read(LINE_MAX) or return $self->close; - $self->{rbuf} .= $$buf; - while ($r > 0 && $self->{rbuf} =~ s/\A\s*([^\r\n]+)\r?\n//) { + if (index($self->{rbuf}, "\n") < 0) { + my $buf = $self->read(LINE_MAX) or return $self->close; + $self->{rbuf} .= $$buf; + } + my $r = 1; + while ($r > 0 && $self->{rbuf} =~ s/\A\s*([^\r\n]*)\r?\n//) { my $line = $1; return $self->close if $line =~ /[[:cntrl:]]/s; my $t0 = now(); @@ -967,7 +970,7 @@ sub event_read { sub watch_read { my ($self, $bool) = @_; my $rv = $self->SUPER::watch_read($bool); - if ($bool && $self->{rbuf} ne '') { + if ($bool && index($self->{rbuf}, "\n") >= 0) { # Force another read if there is a pipelined request. # We don't know if the socket has anything for us to read, # and we must double-check again by the time the timer fires diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index d038dfca..5d500c1b 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -22,6 +22,7 @@ sub mbox_results { my ($ctx) = @_; my $q = PublicInbox::SearchQuery->new($ctx->{qp}); my $x = $q->{x}; + require PublicInbox::Mbox; return PublicInbox::Mbox::mbox_all($ctx, $q->{'q'}) if $x eq 'm'; sres_top_html($ctx); } @@ -35,7 +36,7 @@ sub sres_top_html { my $code = 200; # double the limit for expanded views: my $opts = { - limit => $LIM, + limit => $q->{l}, offset => $q->{o}, mset => 1, relevance => $q->{r}, @@ -181,6 +182,7 @@ sub search_nav_bot { my ($mset, $q) = @_; my $total = $mset->get_matches_estimated; my $o = $q->{o}; + my $l = $q->{l}; my $end = $o + $mset->size; my $beg = $o + 1; my $rv = '
';
@@ -190,15 +192,15 @@ sub search_nav_bot {
 	} else {
 		$rv .= "No more results, only $total";
 	}
-	my $n = $o + $LIM;
+	my $n = $o + $l;
 
 	if ($n < $total) {
-		my $qs = $q->qs_html(o => $n);
+		my $qs = $q->qs_html(o => $n, l => $l);
 		$rv .= qq{  next}
 	}
 	if ($o > 0) {
 		$rv .= $n < $total ? '/' : '       ';
-		my $p = $o - $LIM;
+		my $p = $o - $l;
 		my $qs = $q->qs_html(o => ($p > 0 ? $p : 0));
 		$rv .= qq{prev};
 	}
@@ -305,10 +307,13 @@ sub new {
 	my ($class, $qp) = @_;
 
 	my $r = $qp->{r};
+	my ($l) = (($qp->{l} || '') =~ /(\d+)/);
+	$l = $LIM if !$l || $l > $LIM;
 	bless {
 		q => $qp->{'q'},
 		x => $qp->{x} || '',
 		o => (($qp->{o} || '0') =~ /(\d+)/),
+		l => $l,
 		r => (defined $r && $r ne '0'),
 	}, $class;
 }
@@ -331,6 +336,9 @@ sub qs_html {
 	if (my $o = $self->{o}) { # ignore o == 0
 		$qs .= "&o=$o";
 	}
+	if (my $l = $self->{l}) {
+		$qs .= "&l=$l";
+	}
 	if (my $r = $self->{r}) {
 		$qs .= "&r";
 	}
diff --git a/t/nntpd.t b/t/nntpd.t
index c6e34ed3..3698f98b 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -147,6 +147,8 @@ EOF
 	is($buf, "201 server ready - post via email\r\n", 'got greeting');
 	$s->autoflush(1);
 
+	ok(syswrite($s, "   \r\n"), 'wrote spaces');
+	ok(syswrite($s, "\r\n"), 'wrote nothing');
 	syswrite($s, "NEWGROUPS\t19990424 000000 \033GMT\007\r\n");
 	is(0, sysread($s, $buf, 4096), 'GOT EOF on cntrl');
 
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 60a44bde..2f033016 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -62,6 +62,16 @@ test_psgi(sub { $www->call(@_) }, sub {
 	is('%C3%86var', (keys %uniq)[0], 'matches original query');
 	ok(index($html, 'by Ævar Arnfjörð Bjarmason') >= 0,
 		"displayed Ævar's name properly in HTML");
+
+	my $warn = [];
+	local $SIG{__WARN__} = sub { push @$warn, @_ };
+	$res = $cb->(GET('/test/?q=s:test&l=5e'));
+	is($res->code, 200, 'successful search result');
+	is_deeply([], $warn, 'no warnings from non-numeric comparison');
+
+	$res = $cb->(POST('/test/?q=s:bogus&x=m'));
+	is($res->code, 404, 'failed search result gives 404');
+	is_deeply([], $warn, 'no warnings');
 });
 
 done_testing();