X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FWWW.pm;h=755d75585df22a18ab495d844705fd075cf57f45;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hp=3ce7cc2a7b6ddb2ef74616898e60accd3f92b0ae;hpb=c5ce0f75020db9b3fa7dae9958bedcdb08452e04;p=public-inbox.git diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 3ce7cc2a..755d7558 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2019 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # # Main web interface for mailing list archives @@ -11,30 +11,27 @@ # - Must not rely on static content # - UTF-8 is only for user-content, 7-bit US-ASCII for us package PublicInbox::WWW; -use 5.010_001; use strict; -use warnings; -use bytes (); # only for bytes::length +use v5.10.1; use PublicInbox::Config; use PublicInbox::Hval; use URI::Escape qw(uri_unescape); use PublicInbox::MID qw(mid_escape); -require PublicInbox::Git; use PublicInbox::GitHTTPBackend; use PublicInbox::UserContent; use PublicInbox::WwwStatic qw(r path_info_raw); +use PublicInbox::Eml; # TODO: consider a routing tree now that we have more endpoints: our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!; our $MID_RE = qr!([^/]+)!; our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!; our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!; -our $OID_RE = qr![a-f0-9]{7,40}!; +our $OID_RE = qr![a-f0-9]{7,}!; sub new { - my ($class, $pi_config) = @_; - $pi_config ||= PublicInbox::Config->new; - bless { pi_config => $pi_config }, $class; + my ($class, $pi_cfg) = @_; + bless { pi_cfg => $pi_cfg // PublicInbox::Config->new }, $class; } # backwards compatibility, do not use @@ -51,10 +48,9 @@ sub call { %{$ctx->{qp}} = map { utf8::decode($_); tr/+/ /; - my ($k, $v) = split('=', $_, 2); - $v = uri_unescape($v // ''); + my ($k, $v) = split(/=/, $_, 2); # none of the keys we care about will need escaping - $k => $v; + ($k // '', uri_unescape($v // '')) } split(/[&;]+/, $env->{QUERY_STRING}); my $path_info = path_info_raw($env); @@ -66,6 +62,12 @@ sub call { my ($epoch, $path) = ($2, $3); return invalid_inbox($ctx, $1) || serve_git($ctx, $epoch, $path); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + return get_altid_dump($ctx, $1, $2); + } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/$ATTACH_RE\z!o) { + my ($idx, $fn) = ($3, $4); + return invalid_inbox_mid($ctx, $1, $2) || + get_attach($ctx, $idx, $fn); } elsif ($path_info =~ m!$INBOX_RE/!o) { return invalid_inbox($ctx, $1) || mbox_results($ctx); } @@ -75,8 +77,12 @@ sub call { } # top-level indices and feeds - if ($path_info eq '/' || $path_info eq '/manifest.js.gz') { - www_listing($self)->call($env); + if ($path_info eq '/') { + require PublicInbox::WwwListing; + PublicInbox::WwwListing->response($ctx); + } elsif ($path_info eq '/manifest.js.gz') { + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz->response($ctx); } elsif ($path_info =~ m!$INBOX_RE\z!o) { invalid_inbox($ctx, $1) || r301($ctx, $1); } elsif ($path_info =~ m!$INBOX_RE(?:/|/index\.html)?\z!o) { @@ -85,6 +91,8 @@ sub call { invalid_inbox($ctx, $1) || get_atom($ctx); } elsif ($path_info =~ m!$INBOX_RE/new\.html\z!o) { invalid_inbox($ctx, $1) || get_new($ctx); + } elsif ($path_info =~ m!$INBOX_RE/description\z!o) { + get_description($ctx, $1); } elsif ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)? ($PublicInbox::GitHTTPBackend::ANY)\z!ox) { my ($epoch, $path) = ($2, $3); @@ -122,37 +130,54 @@ sub call { get_vcs_object($ctx, $1, $2, $3); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s\z!o) { r301($ctx, $1, $2, 's/'); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + get_altid_dump($ctx, $1, $2); # convenience redirects order matters } elsif ($path_info =~ m!$INBOX_RE/([^/]{2,})\z!o) { r301($ctx, $1, $2); - + } elsif ($path_info =~ m!\A/\+/([a-zA-Z0-9_\-\.]+)\.css\z!) { + get_css($ctx, undef, $1); # for WwwListing } else { legacy_redirects($ctx, $path_info); } } -# for CoW-friendliness, MOOOOO! +# for CoW-friendliness, MOOOOO! Even for single-process setups, +# we want to get all immortal allocations done early to avoid heap +# fragmentation since common allocators favor a large contiguous heap. sub preload { my ($self) = @_; + + # populate caches used by Encode internally, since emails + # may show up with any encoding. + require Encode; + Encode::find_encoding($_) for Encode->encodings(':all'); + + require PublicInbox::ExtMsg; require PublicInbox::Feed; require PublicInbox::View; require PublicInbox::SearchThread; - require PublicInbox::MIME; - require Digest::SHA; - require POSIX; + require PublicInbox::Eml; + require PublicInbox::Mbox; + require PublicInbox::ViewVCS; + require PublicInbox::WwwText; + require PublicInbox::WwwAttach; eval { require PublicInbox::Search; PublicInbox::Search::load_xapian(); }; - foreach (qw(PublicInbox::SearchView - PublicInbox::Mbox IO::Compress::Gzip - PublicInbox::NewsWWW)) { - eval "require $_;"; + for (qw(SearchView MboxGz WwwAltId)) { + eval "require PublicInbox::$_;"; } if (ref($self)) { + my $pi_cfg = $self->{pi_cfg}; + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { + $pi_cfg->limiter('-cgit'); + } + $pi_cfg->ALL and require PublicInbox::Isearch; $self->cgit; $self->stylesheets_prepare($_) for ('', '../', '../../'); - $self->www_listing; + $self->news_www; } } @@ -178,9 +203,10 @@ sub news_cgit_fallback ($) { # returns undef if valid, array ref response if invalid sub invalid_inbox ($$) { my ($ctx, $inbox) = @_; - my $ibx = $ctx->{www}->{pi_config}->lookup_name($inbox); + my $ibx = $ctx->{www}->{pi_cfg}->lookup_name($inbox) // + $ctx->{www}->{pi_cfg}->lookup_ei($inbox); if (defined $ibx) { - $ctx->{-inbox} = $ibx; + $ctx->{ibx} = $ibx; return; } @@ -198,14 +224,13 @@ sub invalid_inbox_mid { return $ret if $ret; my $mid = $ctx->{mid} = uri_unescape($mid_ue); - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; if ($mid =~ m!\A([a-f0-9]{2})([a-f0-9]{38})\z!) { my ($x2, $x38) = ($1, $2); # this is horrifically wasteful for legacy URLs: - my $str = $ctx->{-inbox}->msg_by_path("$x2/$x38") or return; - require Email::Simple; - my $s = Email::Simple->new($str); - $mid = PublicInbox::MID::mid_clean($s->header('Message-ID')); + my $str = $ctx->{ibx}->msg_by_path("$x2/$x38") or return; + my $s = PublicInbox::Eml->new($str); + $mid = PublicInbox::MID::mid_clean($s->header_raw('Message-ID')); return r301($ctx, $inbox, mid_escape($mid)); } undef; @@ -241,7 +266,7 @@ sub get_index { sub get_mid_txt { my ($ctx) = @_; require PublicInbox::Mbox; - PublicInbox::Mbox::emit_raw($ctx) || r404($ctx); + PublicInbox::Mbox::emit_raw($ctx) || r(404); } # /$INBOX/$MESSAGE_ID/ -> HTML content (short quotes) @@ -254,7 +279,7 @@ sub get_mid_html { # /$INBOX/$MESSAGE_ID/t/ sub get_thread { my ($ctx, $flat) = @_; - $ctx->{-inbox}->over or return need($ctx, 'Overview'); + $ctx->{ibx}->over or return need($ctx, 'Overview'); $ctx->{flat} = $flat; require PublicInbox::View; PublicInbox::View::thread_html($ctx); @@ -273,25 +298,31 @@ sub get_text { } # show git objects (blobs and commits) -# /$INBOX/_/$OBJECT_ID/show -# /$INBOX/_/${OBJECT_ID}_${FILENAME} -# KEY may contain slashes +# /$INBOX/$GIT_OBJECT_ID/s/ +# /$INBOX/$GIT_OBJECT_ID/s/$FILENAME sub get_vcs_object ($$$;$) { my ($ctx, $inbox, $oid, $filename) = @_; my $r404 = invalid_inbox($ctx, $inbox); - return $r404 if $r404; + return $r404 if $r404 || !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx}); require PublicInbox::ViewVCS; PublicInbox::ViewVCS::show($ctx, $oid, $filename); } +sub get_altid_dump { + my ($ctx, $inbox, $altid_pfx) =@_; + my $r404 = invalid_inbox($ctx, $inbox); + return $r404 if $r404; + eval { require PublicInbox::WwwAltId } or return need($ctx, 'sqlite3'); + PublicInbox::WwwAltId::sqldump($ctx, $altid_pfx); +} + sub need { my ($ctx, $extra) = @_; - my $msg = <$extra not available for this -public-inbox
$extra is not available for this public-inbox
-Return to index
+ require PublicInbox::WwwStream; + PublicInbox::WwwStream::html_oneshot($ctx, 501, \<$extra is not available for this public-inbox +Return to index EOF - [ 501, [ 'Content-Type' => 'text/html; charset=UTF-8' ], [ $msg ] ]; } # /$INBOX/$MESSAGE_ID/t.mbox -> thread as mbox @@ -301,7 +332,7 @@ EOF # especially on older systems. Stick to zlib since that's what git uses. sub get_thread_mbox { my ($ctx, $sfx) = @_; - my $over = $ctx->{-inbox}->over or return need($ctx, 'Overview'); + my $over = $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Mbox; PublicInbox::Mbox::thread_mbox($ctx, $over, $sfx); } @@ -310,7 +341,7 @@ sub get_thread_mbox { # /$INBOX/$MESSAGE_ID/t.atom -> thread as Atom feed sub get_thread_atom { my ($ctx) = @_; - $ctx->{-inbox}->over or return need($ctx, 'Overview'); + $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Feed; PublicInbox::Feed::generate_thread_atom($ctx); } @@ -375,11 +406,11 @@ sub legacy_redirects { sub r301 { my ($ctx, $inbox, $mid_ue, $suffix) = @_; - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; unless ($ibx) { my $r404 = invalid_inbox($ctx, $inbox); return $r404 if $r404; - $ibx = $ctx->{-inbox}; + $ibx = $ctx->{ibx}; } my $url = $ibx->base_url($ctx->{env}); my $qs = $ctx->{env}->{QUERY_STRING}; @@ -416,7 +447,7 @@ sub msg_page { sub serve_git { my ($ctx, $epoch, $path) = @_; my $env = $ctx->{env}; - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; my $git = defined $epoch ? $ibx->git_epoch($epoch) : $ibx->git; $git ? PublicInbox::GitHTTPBackend::serve($env, $git, $path) : r404(); } @@ -424,7 +455,7 @@ sub serve_git { sub mbox_results { my ($ctx) = @_; if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { - $ctx->{-inbox}->search or return need($ctx, 'search'); + $ctx->{ibx}->isrch or return need($ctx, 'search'); require PublicInbox::SearchView; return PublicInbox::SearchView::mbox_results($ctx); } @@ -441,20 +472,20 @@ sub serve_mbox_range { sub news_www { my ($self) = @_; - $self->{news_www} ||= do { + $self->{news_www} //= do { require PublicInbox::NewsWWW; - PublicInbox::NewsWWW->new($self->{pi_config}); + PublicInbox::NewsWWW->new($self->{pi_cfg}); } } sub cgit { my ($self) = @_; - $self->{cgit} ||= do { - my $pi_config = $self->{pi_config}; + $self->{cgit} //= do { + my $pi_cfg = $self->{pi_cfg}; - if (defined($pi_config->{'publicinbox.cgitrc'})) { + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { require PublicInbox::Cgit; - PublicInbox::Cgit->new($pi_config); + PublicInbox::Cgit->new($pi_cfg); } else { require Plack::Util; Plack::Util::inline_object(call => sub { r404() }); @@ -462,21 +493,13 @@ sub cgit { } } -sub www_listing { - my ($self) = @_; - $self->{www_listing} ||= do { - require PublicInbox::WwwListing; - PublicInbox::WwwListing->new($self); - } -} - # GET $INBOX/manifest.js.gz sub get_inbox_manifest ($$$) { my ($ctx, $inbox, $key) = @_; my $r404 = invalid_inbox($ctx, $inbox); return $r404 if $r404; - require PublicInbox::WwwListing; - PublicInbox::WwwListing::js($ctx->{env}, [$ctx->{-inbox}]); + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz::per_inbox($ctx); } sub get_attach { @@ -508,7 +531,7 @@ sub stylesheets_prepare ($$) { } || sub { $_[0] }; my $css_map = {}; - my $stylesheets = $self->{pi_config}->{css} || []; + my $stylesheets = $self->{pi_cfg}->{css} || []; my $links = []; my $inline_ok = 1; @@ -599,26 +622,46 @@ sub style { }; } -# /$INBOX/$KEY.css endpoint +# /$INBOX/$KEY.css and /+/$KEY.css endpoints # CSS is configured globally for all inboxes, but we access them on # a per-inbox basis. This allows administrators to setup per-inbox # static routes to intercept the request before it hits PSGI +# inbox == undef => top-level WwwListing sub get_css ($$$) { my ($ctx, $inbox, $key) = @_; - my $r404 = invalid_inbox($ctx, $inbox); + my $r404 = defined($inbox) ? invalid_inbox($ctx, $inbox) : undef; return $r404 if $r404; my $self = $ctx->{www}; - my $css_map = $self->{-css_map} || stylesheets_prepare($self, ''); + my $css_map = $self->{-css_map} || + stylesheets_prepare($self, defined($inbox) ? '' : '+/'); my $css = $css_map->{$key}; - if (!defined($css) && $key eq 'userContent') { + if (!defined($css) && defined($inbox) && $key eq 'userContent') { my $env = $ctx->{env}; - $css = PublicInbox::UserContent::sample($ctx->{-inbox}, $env); + $css = PublicInbox::UserContent::sample($ctx->{ibx}, $env); } defined $css or return r404(); - my $h = [ 'Content-Length', bytes::length($css), - 'Content-Type', 'text/css' ]; + my $h = [ 'Content-Length', length($css), 'Content-Type', 'text/css' ]; PublicInbox::GitHTTPBackend::cache_one_year($h); [ 200, $h, [ $css ] ]; } +sub get_description { + my ($ctx, $inbox) = @_; + invalid_inbox($ctx, $inbox) || do { + my $d = $ctx->{ibx}->description . "\n"; + utf8::encode($d); + [ 200, [ 'Content-Length', length($d), + 'Content-Type', 'text/plain' ], [ $d ] ]; + }; +} + +sub event_step { # called via requeue + my ($self) = @_; + # gzf = PublicInbox::GzipFilter == $ctx + my $gzf = shift(@{$self->{-low_prio_q}}) // return; + PublicInbox::DS::requeue($self) if scalar(@{$self->{-low_prio_q}}); + my $http = $gzf->{env}->{'psgix.io'}; # PublicInbox::HTTP + $http->next_step($gzf->can('async_next')); +} + 1;