X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FWWW.pm;h=500021d422698eef27c41fea84065453d0eaec43;hb=af0b0fb7a454470a32c452119d0392e0dedb3fe1;hp=1e7d3c1efef1a84833f36c945a85c22dd1a2f29c;hpb=95bdac7f09c69036efed537a4d03d5bdd2ae4eb6;p=public-inbox.git diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 1e7d3c1e..500021d4 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2020 all contributors +# Copyright (C) 2014-2021 all contributors # License: AGPL-3.0+ # # Main web interface for mailing list archives @@ -19,22 +19,21 @@ use PublicInbox::Config; use PublicInbox::Hval; use URI::Escape qw(uri_unescape); use PublicInbox::MID qw(mid_escape); -require PublicInbox::Git; use PublicInbox::GitHTTPBackend; use PublicInbox::UserContent; use PublicInbox::WwwStatic qw(r path_info_raw); +use PublicInbox::Eml; # TODO: consider a routing tree now that we have more endpoints: our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!; our $MID_RE = qr!([^/]+)!; our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!; our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!; -our $OID_RE = qr![a-f0-9]{7,40}!; +our $OID_RE = qr![a-f0-9]{7,}!; sub new { - my ($class, $pi_config) = @_; - $pi_config ||= PublicInbox::Config->new; - bless { pi_config => $pi_config }, $class; + my ($class, $pi_cfg) = @_; + bless { pi_cfg => $pi_cfg // PublicInbox::Config->new }, $class; } # backwards compatibility, do not use @@ -66,6 +65,8 @@ sub call { my ($epoch, $path) = ($2, $3); return invalid_inbox($ctx, $1) || serve_git($ctx, $epoch, $path); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + return get_altid_dump($ctx, $1, $2); } elsif ($path_info =~ m!$INBOX_RE/!o) { return invalid_inbox($ctx, $1) || mbox_results($ctx); } @@ -75,8 +76,12 @@ sub call { } # top-level indices and feeds - if ($path_info eq '/' || $path_info eq '/manifest.js.gz') { - www_listing($self)->call($env); + if ($path_info eq '/') { + require PublicInbox::WwwListing; + PublicInbox::WwwListing->response($ctx); + } elsif ($path_info eq '/manifest.js.gz') { + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz->response($ctx); } elsif ($path_info =~ m!$INBOX_RE\z!o) { invalid_inbox($ctx, $1) || r301($ctx, $1); } elsif ($path_info =~ m!$INBOX_RE(?:/|/index\.html)?\z!o) { @@ -124,6 +129,8 @@ sub call { get_vcs_object($ctx, $1, $2, $3); } elsif ($path_info =~ m!$INBOX_RE/($OID_RE)/s\z!o) { r301($ctx, $1, $2, 's/'); + } elsif ($path_info =~ m!$INBOX_RE/(\w+)\.sql\.gz\z!o) { + get_altid_dump($ctx, $1, $2); # convenience redirects order matters } elsif ($path_info =~ m!$INBOX_RE/([^/]{2,})\z!o) { r301($ctx, $1, $2); @@ -133,31 +140,53 @@ sub call { } } -# for CoW-friendliness, MOOOOO! +# for CoW-friendliness, MOOOOO! Even for single-process setups, +# we want to get all immortal allocations done early to avoid heap +# fragmentation since common allocators favor a large contiguous heap. sub preload { my ($self) = @_; + + # populate caches used by Encode internally, since emails + # may show up with any encoding. + require Encode; + Encode::find_encoding($_) for Encode->encodings(':all'); + + require PublicInbox::ExtMsg; require PublicInbox::Feed; require PublicInbox::View; require PublicInbox::SearchThread; - require PublicInbox::MIME; - require Digest::SHA; - require POSIX; + require PublicInbox::Eml; + require PublicInbox::Mbox; + require PublicInbox::ViewVCS; + require PublicInbox::WwwText; + require PublicInbox::WwwAttach; eval { require PublicInbox::Search; PublicInbox::Search::load_xapian(); }; - foreach (qw(PublicInbox::SearchView - PublicInbox::Mbox IO::Compress::Gzip - PublicInbox::NewsWWW)) { - eval "require $_;"; + for (qw(SearchView MboxGz WwwAltId)) { + eval "require PublicInbox::$_;"; } if (ref($self)) { + my $pi_cfg = $self->{pi_cfg}; + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { + $pi_cfg->limiter('-cgit'); + } $self->cgit; $self->stylesheets_prepare($_) for ('', '../', '../../'); - $self->www_listing; + $self->news_www; + $pi_cfg->each_inbox(\&preload_inbox); } } +sub preload_inbox { + my $ibx = shift; + $ibx->altid_map; + $ibx->cloneurl; + $ibx->description; + $ibx->base_url; +} + # private functions below sub r404 { @@ -180,9 +209,10 @@ sub news_cgit_fallback ($) { # returns undef if valid, array ref response if invalid sub invalid_inbox ($$) { my ($ctx, $inbox) = @_; - my $ibx = $ctx->{www}->{pi_config}->lookup_name($inbox); + my $ibx = $ctx->{www}->{pi_cfg}->lookup_name($inbox) // + $ctx->{www}->{pi_cfg}->lookup_ei($inbox); if (defined $ibx) { - $ctx->{-inbox} = $ibx; + $ctx->{ibx} = $ibx; return; } @@ -200,14 +230,13 @@ sub invalid_inbox_mid { return $ret if $ret; my $mid = $ctx->{mid} = uri_unescape($mid_ue); - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; if ($mid =~ m!\A([a-f0-9]{2})([a-f0-9]{38})\z!) { my ($x2, $x38) = ($1, $2); # this is horrifically wasteful for legacy URLs: - my $str = $ctx->{-inbox}->msg_by_path("$x2/$x38") or return; - require Email::Simple; - my $s = Email::Simple->new($str); - $mid = PublicInbox::MID::mid_clean($s->header('Message-ID')); + my $str = $ctx->{ibx}->msg_by_path("$x2/$x38") or return; + my $s = PublicInbox::Eml->new($str); + $mid = PublicInbox::MID::mid_clean($s->header_raw('Message-ID')); return r301($ctx, $inbox, mid_escape($mid)); } undef; @@ -256,7 +285,7 @@ sub get_mid_html { # /$INBOX/$MESSAGE_ID/t/ sub get_thread { my ($ctx, $flat) = @_; - $ctx->{-inbox}->over or return need($ctx, 'Overview'); + $ctx->{ibx}->over or return need($ctx, 'Overview'); $ctx->{flat} = $flat; require PublicInbox::View; PublicInbox::View::thread_html($ctx); @@ -275,9 +304,8 @@ sub get_text { } # show git objects (blobs and commits) -# /$INBOX/_/$OBJECT_ID/show -# /$INBOX/_/${OBJECT_ID}_${FILENAME} -# KEY may contain slashes +# /$INBOX/$GIT_OBJECT_ID/s/ +# /$INBOX/$GIT_OBJECT_ID/s/$FILENAME sub get_vcs_object ($$$;$) { my ($ctx, $inbox, $oid, $filename) = @_; my $r404 = invalid_inbox($ctx, $inbox); @@ -286,14 +314,21 @@ sub get_vcs_object ($$$;$) { PublicInbox::ViewVCS::show($ctx, $oid, $filename); } +sub get_altid_dump { + my ($ctx, $inbox, $altid_pfx) =@_; + my $r404 = invalid_inbox($ctx, $inbox); + return $r404 if $r404; + eval { require PublicInbox::WwwAltId } or return need($ctx, 'sqlite3'); + PublicInbox::WwwAltId::sqldump($ctx, $altid_pfx); +} + sub need { my ($ctx, $extra) = @_; - my $msg = <$extra not available for this -public-inbox
$extra is not available for this public-inbox
-Return to index
+ require PublicInbox::WwwStream; + PublicInbox::WwwStream::html_oneshot($ctx, 501, \<$extra is not available for this public-inbox +Return to index EOF - [ 501, [ 'Content-Type' => 'text/html; charset=UTF-8' ], [ $msg ] ]; } # /$INBOX/$MESSAGE_ID/t.mbox -> thread as mbox @@ -303,7 +338,7 @@ EOF # especially on older systems. Stick to zlib since that's what git uses. sub get_thread_mbox { my ($ctx, $sfx) = @_; - my $over = $ctx->{-inbox}->over or return need($ctx, 'Overview'); + my $over = $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Mbox; PublicInbox::Mbox::thread_mbox($ctx, $over, $sfx); } @@ -312,7 +347,7 @@ sub get_thread_mbox { # /$INBOX/$MESSAGE_ID/t.atom -> thread as Atom feed sub get_thread_atom { my ($ctx) = @_; - $ctx->{-inbox}->over or return need($ctx, 'Overview'); + $ctx->{ibx}->over or return need($ctx, 'Overview'); require PublicInbox::Feed; PublicInbox::Feed::generate_thread_atom($ctx); } @@ -377,11 +412,11 @@ sub legacy_redirects { sub r301 { my ($ctx, $inbox, $mid_ue, $suffix) = @_; - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; unless ($ibx) { my $r404 = invalid_inbox($ctx, $inbox); return $r404 if $r404; - $ibx = $ctx->{-inbox}; + $ibx = $ctx->{ibx}; } my $url = $ibx->base_url($ctx->{env}); my $qs = $ctx->{env}->{QUERY_STRING}; @@ -418,7 +453,7 @@ sub msg_page { sub serve_git { my ($ctx, $epoch, $path) = @_; my $env = $ctx->{env}; - my $ibx = $ctx->{-inbox}; + my $ibx = $ctx->{ibx}; my $git = defined $epoch ? $ibx->git_epoch($epoch) : $ibx->git; $git ? PublicInbox::GitHTTPBackend::serve($env, $git, $path) : r404(); } @@ -426,7 +461,7 @@ sub serve_git { sub mbox_results { my ($ctx) = @_; if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { - $ctx->{-inbox}->search or return need($ctx, 'search'); + $ctx->{ibx}->isrch or return need($ctx, 'search'); require PublicInbox::SearchView; return PublicInbox::SearchView::mbox_results($ctx); } @@ -445,18 +480,18 @@ sub news_www { my ($self) = @_; $self->{news_www} ||= do { require PublicInbox::NewsWWW; - PublicInbox::NewsWWW->new($self->{pi_config}); + PublicInbox::NewsWWW->new($self->{pi_cfg}); } } sub cgit { my ($self) = @_; $self->{cgit} ||= do { - my $pi_config = $self->{pi_config}; + my $pi_cfg = $self->{pi_cfg}; - if (defined($pi_config->{'publicinbox.cgitrc'})) { + if (defined($pi_cfg->{'publicinbox.cgitrc'})) { require PublicInbox::Cgit; - PublicInbox::Cgit->new($pi_config); + PublicInbox::Cgit->new($pi_cfg); } else { require Plack::Util; Plack::Util::inline_object(call => sub { r404() }); @@ -464,21 +499,13 @@ sub cgit { } } -sub www_listing { - my ($self) = @_; - $self->{www_listing} ||= do { - require PublicInbox::WwwListing; - PublicInbox::WwwListing->new($self); - } -} - # GET $INBOX/manifest.js.gz sub get_inbox_manifest ($$$) { my ($ctx, $inbox, $key) = @_; my $r404 = invalid_inbox($ctx, $inbox); return $r404 if $r404; - require PublicInbox::WwwListing; - PublicInbox::WwwListing::js($ctx->{env}, [$ctx->{-inbox}]); + require PublicInbox::ManifestJsGz; + PublicInbox::ManifestJsGz::per_inbox($ctx); } sub get_attach { @@ -510,7 +537,7 @@ sub stylesheets_prepare ($$) { } || sub { $_[0] }; my $css_map = {}; - my $stylesheets = $self->{pi_config}->{css} || []; + my $stylesheets = $self->{pi_cfg}->{css} || []; my $links = []; my $inline_ok = 1; @@ -614,7 +641,7 @@ sub get_css ($$$) { my $css = $css_map->{$key}; if (!defined($css) && $key eq 'userContent') { my $env = $ctx->{env}; - $css = PublicInbox::UserContent::sample($ctx->{-inbox}, $env); + $css = PublicInbox::UserContent::sample($ctx->{ibx}, $env); } defined $css or return r404(); my $h = [ 'Content-Length', bytes::length($css), @@ -626,7 +653,7 @@ sub get_css ($$$) { sub get_description { my ($ctx, $inbox) = @_; invalid_inbox($ctx, $inbox) || do { - my $d = $ctx->{-inbox}->description . "\n"; + my $d = $ctx->{ibx}->description . "\n"; [ 200, [ 'Content-Length', bytes::length($d), 'Content-Type', 'text/plain' ], [ $d ] ]; };