X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FWWW.pm;h=755d75585df22a18ab495d844705fd075cf57f45;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hp=406802a95767f409f49fb2247ee8e52e7328e512;hpb=c2d12f79e78eb4ea909cac6106880c0f41e8c5fd;p=public-inbox.git diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 406802a9..755d7558 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2018 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # # Main web interface for mailing list archives @@ -11,28 +11,27 @@ # - Must not rely on static content # - UTF-8 is only for user-content, 7-bit US-ASCII for us package PublicInbox::WWW; -use 5.008; use strict; -use warnings; +use v5.10.1; use PublicInbox::Config; use PublicInbox::Hval; use URI::Escape qw(uri_unescape); use PublicInbox::MID qw(mid_escape); -require PublicInbox::Git; use PublicInbox::GitHTTPBackend; use PublicInbox::UserContent; +use PublicInbox::WwwStatic qw(r path_info_raw); +use PublicInbox::Eml; # TODO: consider a routing tree now that we have more endpoints: our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!; our $MID_RE = qr!([^/]+)!; our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!; -our $ATTACH_RE = qr!(\d[\.\d]*)-([[:alnum:]][\w\.-]+[[:alnum:]])!i; -our $OID_RE = qr![a-f0-9]{7,40}!; +our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!; +our $OID_RE = qr![a-f0-9]{7,}!; sub new { - my ($class, $pi_config) = @_; - $pi_config ||= PublicInbox::Config->new; - bless { pi_config => $pi_config }, $class; + my ($class, $pi_cfg) = @_; + bless { pi_cfg => $pi_cfg // PublicInbox::Config->new }, $class; } # backwards compatibility, do not use @@ -41,51 +40,49 @@ sub run { PublicInbox::WWW->new->call($req->env); } -my %path_re_cache; - -sub path_re ($) { - my $sn = $_[0]->{SCRIPT_NAME}; - $path_re_cache{$sn} ||= do { - $sn = '/'.$sn unless index($sn, '/') == 0; - $sn =~ s!/\z!!; - qr!\A(?:https?://[^/]+)?\Q$sn\E(/[^\?\#]+)!; - }; -} - sub call { my ($self, $env) = @_; my $ctx = { env => $env, www => $self }; # we don't care about multi-value - my %qp = map { + %{$ctx->{qp}} = map { utf8::decode($_); - my ($k, $v) = split('=', uri_unescape($_), 2); - $v = '' unless defined $v; - $v =~ tr/+/ /; - ($k, $v) + tr/+/ /; + my ($k, $v) = split(/=/, $_, 2); + # none of the keys we care about will need escaping + ($k // '', uri_unescape($v // '')) } split(/[&;]+/, $env->{QUERY_STRING}); - $ctx->{qp} = \%qp; - # not using $env->{PATH_INFO} here since that's already decoded - my ($path_info) = ($env->{REQUEST_URI} =~ path_re($env)); + my $path_info = path_info_raw($env); my $method = $env->{REQUEST_METHOD}; if ($method eq 'POST') { - if ($path_info =~ m!$INBOX_RE/(?:(\d+)/)?(git-upload-pack)\z!) { - my ($part, $path) = ($2, $3); + if ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)? + (git-upload-pack)\z!x) { + my ($epoch, $path) = ($2, $3); return invalid_inbox($ctx, $1) || - serve_git($ctx, $part, $path); + serve_git($ctx, $epoch, $path); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + return get_altid_dump($ctx, $1, $2); + } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/$ATTACH_RE\z!o) { + my ($idx, $fn) = ($3, $4); + return invalid_inbox_mid($ctx, $1, $2) || + get_attach($ctx, $idx, $fn); } elsif ($path_info =~ m!$INBOX_RE/!o) { return invalid_inbox($ctx, $1) || mbox_results($ctx); } } - elsif ($method !~ /\AGET|HEAD\z/) { - return r(405, 'Method Not Allowed'); + elsif ($method !~ /\A(?:GET|HEAD)\z/) { + return r(405); } # top-level indices and feeds if ($path_info eq '/') { - r404(); + require PublicInbox::WwwListing; + PublicInbox::WwwListing->response($ctx); + } elsif ($path_info eq '/manifest.js.gz') { + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz->response($ctx); } elsif ($path_info =~ m!$INBOX_RE\z!o) { invalid_inbox($ctx, $1) || r301($ctx, $1); } elsif ($path_info =~ m!$INBOX_RE(?:/|/index\.html)?\z!o) { @@ -94,11 +91,13 @@ sub call { invalid_inbox($ctx, $1) || get_atom($ctx); } elsif ($path_info =~ m!$INBOX_RE/new\.html\z!o) { invalid_inbox($ctx, $1) || get_new($ctx); - } elsif ($path_info =~ m!$INBOX_RE/(?:(\d+)/)? + } elsif ($path_info =~ m!$INBOX_RE/description\z!o) { + get_description($ctx, $1); + } elsif ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)? ($PublicInbox::GitHTTPBackend::ANY)\z!ox) { - my ($part, $path) = ($2, $3); - invalid_inbox($ctx, $1) || serve_git($ctx, $part, $path); - } elsif ($path_info =~ m!$INBOX_RE/([\w-]+).mbox\.gz\z!o) { + my ($epoch, $path) = ($2, $3); + invalid_inbox($ctx, $1) || serve_git($ctx, $epoch, $path); + } elsif ($path_info =~ m!$INBOX_RE/([a-zA-Z0-9_\-]+).mbox\.gz\z!o) { serve_mbox_range($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/$END_RE\z!o) { msg_page($ctx, $1, $2, $3); @@ -120,40 +119,65 @@ sub call { r301($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/_/text(?:/(.*))?\z!o) { get_text($ctx, $1, $2); - } elsif ($path_info =~ m!$INBOX_RE/([\w\-\.]+)\.css\z!o) { + } elsif ($path_info =~ m!$INBOX_RE/([a-zA-Z0-9_\-\.]+)\.css\z!o) { get_css($ctx, $1, $2); + } elsif ($path_info =~ m!$INBOX_RE/manifest\.js\.gz\z!o) { + get_inbox_manifest($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s/\z!o) { get_vcs_object($ctx, $1, $2); - } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s/([\w\.\-]+)\z!o) { + } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s/ + ($PublicInbox::Hval::FN)\z!ox) { get_vcs_object($ctx, $1, $2, $3); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s\z!o) { r301($ctx, $1, $2, 's/'); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + get_altid_dump($ctx, $1, $2); # convenience redirects order matters } elsif ($path_info =~ m!$INBOX_RE/([^/]{2,})\z!o) { r301($ctx, $1, $2); - + } elsif ($path_info =~ m!\A/\+/([a-zA-Z0-9_\-\.]+)\.css\z!) { + get_css($ctx, undef, $1); # for WwwListing } else { legacy_redirects($ctx, $path_info); } } -# for CoW-friendliness, MOOOOO! +# for CoW-friendliness, MOOOOO! Even for single-process setups, +# we want to get all immortal allocations done early to avoid heap +# fragmentation since common allocators favor a large contiguous heap. sub preload { my ($self) = @_; + + # populate caches used by Encode internally, since emails + # may show up with any encoding. + require Encode; + Encode::find_encoding($_) for Encode->encodings(':all'); + + require PublicInbox::ExtMsg; require PublicInbox::Feed; require PublicInbox::View; require PublicInbox::SearchThread; - require PublicInbox::MIME; - require Digest::SHA; - require POSIX; - - foreach (qw(PublicInbox::Search PublicInbox::SearchView - PublicInbox::Mbox IO::Compress::Gzip - PublicInbox::NewsWWW)) { - eval "require $_;"; + require PublicInbox::Eml; + require PublicInbox::Mbox; + require PublicInbox::ViewVCS; + require PublicInbox::WwwText; + require PublicInbox::WwwAttach; + eval { + require PublicInbox::Search; + PublicInbox::Search::load_xapian(); + }; + for (qw(SearchView MboxGz WwwAltId)) { + eval "require PublicInbox::$_;"; } if (ref($self)) { + my $pi_cfg = $self->{pi_cfg}; + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { + $pi_cfg->limiter('-cgit'); + } + $pi_cfg->ALL and require PublicInbox::Isearch; + $self->cgit; $self->stylesheets_prepare($_) for ('', '../', '../../'); + $self->news_www; } } @@ -163,23 +187,26 @@ sub r404 { my ($ctx) = @_; if ($ctx && $ctx->{mid}) { require PublicInbox::ExtMsg; - searcher($ctx); return PublicInbox::ExtMsg::ext_msg($ctx); } - r(404, 'Not Found'); + r(404); } -# simple response for errors -sub r { [ $_[0], ['Content-Type' => 'text/plain'], [ join(' ', @_, "\n") ] ] } +sub news_cgit_fallback ($) { + my ($ctx) = @_; + my $www = $ctx->{www}; + my $env = $ctx->{env}; + my $res = $www->news_www->call($env); + $res->[0] == 404 ? $www->cgit->call($env) : $res; +} # returns undef if valid, array ref response if invalid sub invalid_inbox ($$) { my ($ctx, $inbox) = @_; - my $www = $ctx->{www}; - my $obj = $www->{pi_config}->lookup_name($inbox); - if (defined $obj) { - $ctx->{git} = $obj->git; - $ctx->{-inbox} = $obj; + my $ibx = $ctx->{www}->{pi_cfg}->lookup_name($inbox) // + $ctx->{www}->{pi_cfg}->lookup_ei($inbox); + if (defined $ibx) { + $ctx->{ibx} = $ibx; return; } @@ -187,7 +214,7 @@ sub invalid_inbox ($$) { # generation and link things intended for nntp:// to https?://, # so try to infer links and redirect them to the appropriate # list URL. - $www->news_www->call($ctx->{env}); + news_cgit_fallback($ctx); } # returns undef if valid, array ref response if invalid @@ -197,14 +224,13 @@ sub invalid_inbox_mid { return $ret if $ret; my $mid = $ctx->{mid} = uri_unescape($mid_ue); - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; if ($mid =~ m!\A([a-f0-9]{2})([a-f0-9]{38})\z!) { my ($x2, $x38) = ($1, $2); # this is horrifically wasteful for legacy URLs: - my $str = $ctx->{-inbox}->msg_by_path("$x2/$x38") or return; - require Email::Simple; - my $s = Email::Simple->new($str); - $mid = PublicInbox::MID::mid_clean($s->header('Message-ID')); + my $str = $ctx->{ibx}->msg_by_path("$x2/$x38") or return; + my $s = PublicInbox::Eml->new($str); + $mid = PublicInbox::MID::mid_clean($s->header_raw('Message-ID')); return r301($ctx, $inbox, mid_escape($mid)); } undef; @@ -228,7 +254,6 @@ sub get_new { sub get_index { my ($ctx) = @_; require PublicInbox::Feed; - searcher($ctx); if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { require PublicInbox::SearchView; PublicInbox::SearchView::sres_top_html($ctx); @@ -241,21 +266,20 @@ sub get_index { sub get_mid_txt { my ($ctx) = @_; require PublicInbox::Mbox; - PublicInbox::Mbox::emit_raw($ctx) || r404($ctx); + PublicInbox::Mbox::emit_raw($ctx) || r(404); } # /$INBOX/$MESSAGE_ID/ -> HTML content (short quotes) sub get_mid_html { my ($ctx) = @_; require PublicInbox::View; - searcher($ctx); PublicInbox::View::msg_page($ctx) || r404($ctx); } # /$INBOX/$MESSAGE_ID/t/ sub get_thread { my ($ctx, $flat) = @_; - searcher($ctx) or return need_search($ctx); + $ctx->{ibx}->over or return need($ctx, 'Overview'); $ctx->{flat} = $flat; require PublicInbox::View; PublicInbox::View::thread_html($ctx); @@ -274,42 +298,31 @@ sub get_text { } # show git objects (blobs and commits) -# /$INBOX/_/$OBJECT_ID/show -# /$INBOX/_/${OBJECT_ID}_${FILENAME} -# KEY may contain slashes +# /$INBOX/$GIT_OBJECT_ID/s/ +# /$INBOX/$GIT_OBJECT_ID/s/$FILENAME sub get_vcs_object ($$$;$) { my ($ctx, $inbox, $oid, $filename) = @_; my $r404 = invalid_inbox($ctx, $inbox); - return $r404 if $r404; + return $r404 if $r404 || !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx}); require PublicInbox::ViewVCS; PublicInbox::ViewVCS::show($ctx, $oid, $filename); } -sub ctx_get { - my ($ctx, $key) = @_; - my $val = $ctx->{$key}; - (defined $val && $val ne '') or die "BUG: bad ctx, $key unusable"; - $val; -} - -# search support is optional, returns undef if Xapian is not installed -# or not configured for the given GIT_DIR -sub searcher { - my ($ctx) = @_; - eval { - require PublicInbox::Search; - $ctx->{srch} = $ctx->{-inbox}->search; - }; +sub get_altid_dump { + my ($ctx, $inbox, $altid_pfx) =@_; + my $r404 = invalid_inbox($ctx, $inbox); + return $r404 if $r404; + eval { require PublicInbox::WwwAltId } or return need($ctx, 'sqlite3'); + PublicInbox::WwwAltId::sqldump($ctx, $altid_pfx); } -sub need_search { - my ($ctx) = @_; - my $msg = <Search not available for this -public-inbox
Search is not available for this public-inbox
-Return to index
+sub need { + my ($ctx, $extra) = @_; + require PublicInbox::WwwStream; + PublicInbox::WwwStream::html_oneshot($ctx, 501, \<$extra is not available for this public-inbox +Return to index EOF - [ 501, [ 'Content-Type' => 'text/html; charset=UTF-8' ], [ $msg ] ]; } # /$INBOX/$MESSAGE_ID/t.mbox -> thread as mbox @@ -319,16 +332,16 @@ EOF # especially on older systems. Stick to zlib since that's what git uses. sub get_thread_mbox { my ($ctx, $sfx) = @_; - my $srch = searcher($ctx) or return need_search($ctx); + my $over = $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Mbox; - PublicInbox::Mbox::thread_mbox($ctx, $srch, $sfx); + PublicInbox::Mbox::thread_mbox($ctx, $over, $sfx); } # /$INBOX/$MESSAGE_ID/t.atom -> thread as Atom feed sub get_thread_atom { my ($ctx) = @_; - searcher($ctx) or return need_search($ctx); + $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Feed; PublicInbox::Feed::generate_thread_atom($ctx); } @@ -387,19 +400,19 @@ sub legacy_redirects { } elsif ($path_info =~ m!$INBOX_RE/(\S+/\S+)/f\z!o) { r301($ctx, $1, $2); } else { - $ctx->{www}->news_www->call($ctx->{env}); + news_cgit_fallback($ctx); } } sub r301 { my ($ctx, $inbox, $mid_ue, $suffix) = @_; - my $obj = $ctx->{-inbox}; - unless ($obj) { + my $ibx = $ctx->{ibx}; + unless ($ibx) { my $r404 = invalid_inbox($ctx, $inbox); return $r404 if $r404; - $obj = $ctx->{-inbox}; + $ibx = $ctx->{ibx}; } - my $url = $obj->base_url($ctx->{env}); + my $url = $ibx->base_url($ctx->{env}); my $qs = $ctx->{env}->{QUERY_STRING}; if (defined $mid_ue) { # common, and much nicer as '@' than '%40': @@ -432,17 +445,17 @@ sub msg_page { } sub serve_git { - my ($ctx, $part, $path) = @_; + my ($ctx, $epoch, $path) = @_; my $env = $ctx->{env}; - my $ibx = $ctx->{-inbox}; - my $git = defined $part ? $ibx->git_part($part) : $ibx->git; + my $ibx = $ctx->{ibx}; + my $git = defined $epoch ? $ibx->git_epoch($epoch) : $ibx->git; $git ? PublicInbox::GitHTTPBackend::serve($env, $git, $path) : r404(); } sub mbox_results { my ($ctx) = @_; if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { - searcher($ctx) or return need_search($ctx); + $ctx->{ibx}->isrch or return need($ctx, 'search'); require PublicInbox::SearchView; return PublicInbox::SearchView::mbox_results($ctx); } @@ -453,19 +466,42 @@ sub serve_mbox_range { my ($ctx, $inbox, $range) = @_; invalid_inbox($ctx, $inbox) || eval { require PublicInbox::Mbox; - searcher($ctx); PublicInbox::Mbox::emit_range($ctx, $range); } } sub news_www { my ($self) = @_; - $self->{news_www} ||= do { + $self->{news_www} //= do { require PublicInbox::NewsWWW; - PublicInbox::NewsWWW->new($self->{pi_config}); + PublicInbox::NewsWWW->new($self->{pi_cfg}); + } +} + +sub cgit { + my ($self) = @_; + $self->{cgit} //= do { + my $pi_cfg = $self->{pi_cfg}; + + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { + require PublicInbox::Cgit; + PublicInbox::Cgit->new($pi_cfg); + } else { + require Plack::Util; + Plack::Util::inline_object(call => sub { r404() }); + } } } +# GET $INBOX/manifest.js.gz +sub get_inbox_manifest ($$$) { + my ($ctx, $inbox, $key) = @_; + my $r404 = invalid_inbox($ctx, $inbox); + return $r404 if $r404; + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz::per_inbox($ctx); +} + sub get_attach { my ($ctx, $idx, $fn) = @_; require PublicInbox::WwwAttach; @@ -495,7 +531,7 @@ sub stylesheets_prepare ($$) { } || sub { $_[0] }; my $css_map = {}; - my $stylesheets = $self->{pi_config}->{css} || []; + my $stylesheets = $self->{pi_cfg}->{css} || []; my $links = []; my $inline_ok = 1; @@ -511,17 +547,29 @@ sub stylesheets_prepare ($$) { if (defined $attr->{href}) { $inline_ok = 0; } else { - open(my $fh, '<', $_) or do { - warn "failed to open $_: $!\n"; + my $fn = $_; + my ($key) = (m!([^/]+?)(?:\.css)?\z!i); + if ($key !~ /\A[a-zA-Z0-9_\-\.]+\z/) { + warn "ignoring $fn, non-ASCII word character\n"; + next; + } + open(my $fh, '<', $fn) or do { + warn "failed to open $fn: $!\n"; next; }; - my ($key) = (m!([^/]+?)(?:\.css)?\z!i); my $ctime = 0; my $local = do { local $/; <$fh> }; if ($local =~ /\S/) { $ctime = sprintf('%x',(stat($fh))[10]); $local = $mini->($local); } + + # do not let BOFHs override userContent.css: + if ($local =~ /!\s*important\b/i) { + warn "ignoring $fn since it uses `!important'\n"; + next; + } + $css_map->{$key} = $local; $attr->{href} = "$upfx$key.css?$ctime"; if (defined($attr->{title})) { @@ -574,26 +622,46 @@ sub style { }; } -# /$INBOX/$KEY.css endpoint +# /$INBOX/$KEY.css and /+/$KEY.css endpoints # CSS is configured globally for all inboxes, but we access them on # a per-inbox basis. This allows administrators to setup per-inbox # static routes to intercept the request before it hits PSGI +# inbox == undef => top-level WwwListing sub get_css ($$$) { my ($ctx, $inbox, $key) = @_; - my $r404 = invalid_inbox($ctx, $inbox); + my $r404 = defined($inbox) ? invalid_inbox($ctx, $inbox) : undef; return $r404 if $r404; my $self = $ctx->{www}; - my $css_map = $self->{-css_map} || stylesheets_prepare($self, ''); + my $css_map = $self->{-css_map} || + stylesheets_prepare($self, defined($inbox) ? '' : '+/'); my $css = $css_map->{$key}; - if (!defined($css) && $key eq 'userContent') { + if (!defined($css) && defined($inbox) && $key eq 'userContent') { my $env = $ctx->{env}; - $css = PublicInbox::UserContent::sample($ctx->{-inbox}, $env); + $css = PublicInbox::UserContent::sample($ctx->{ibx}, $env); } defined $css or return r404(); - my $h = [ 'Content-Length', bytes::length($css), - 'Content-Type', 'text/css' ]; + my $h = [ 'Content-Length', length($css), 'Content-Type', 'text/css' ]; PublicInbox::GitHTTPBackend::cache_one_year($h); [ 200, $h, [ $css ] ]; } +sub get_description { + my ($ctx, $inbox) = @_; + invalid_inbox($ctx, $inbox) || do { + my $d = $ctx->{ibx}->description . "\n"; + utf8::encode($d); + [ 200, [ 'Content-Length', length($d), + 'Content-Type', 'text/plain' ], [ $d ] ]; + }; +} + +sub event_step { # called via requeue + my ($self) = @_; + # gzf = PublicInbox::GzipFilter == $ctx + my $gzf = shift(@{$self->{-low_prio_q}}) // return; + PublicInbox::DS::requeue($self) if scalar(@{$self->{-low_prio_q}}); + my $http = $gzf->{env}->{'psgix.io'}; # PublicInbox::HTTP + $http->next_step($gzf->can('async_next')); +} + 1;