X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=da81fb67ad22efa8248c208a2d65629512153bbe;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hp=df140cacbec177191839a7ffd310b132e11c566e;hpb=796e7215a167f36d0b950631c3b1e44fa47fec07;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index df140cac..da81fb67 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,84 +1,54 @@ -# Copyright (C) 2016-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) package PublicInbox::Inbox; use strict; +use v5.10.1; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); use PublicInbox::Eml; use List::Util qw(max); use Carp qw(croak); -# Long-running "git-cat-file --batch" processes won't notice -# unlinked packs, so we need to restart those processes occasionally. -# Xapian and SQLite file handles are mostly stable, but sometimes an -# admin will attempt to replace them atomically after compact/vacuum -# and we need to be prepared for that. -my $cleanup_timer; -my $cleanup_avail = -1; # 0, or 1 -my $have_devel_peek; -my $CLEANUP = {}; # string(inbox) -> inbox - -sub git_cleanup ($) { - my ($self) = @_; - my $git = $self->{git} or return; - $git->cleanup; -} - -sub cleanup_task () { - $cleanup_timer = undef; - my $next = {}; - for my $ibx (values %$CLEANUP) { - my $again; - if ($have_devel_peek) { - foreach my $f (qw(search)) { - # we bump refcnt by assigning tmp, here: - my $tmp = $ibx->{$f} or next; - next if Devel::Peek::SvREFCNT($tmp) > 2; - delete $ibx->{$f}; - # refcnt is zero when tmp is out-of-scope - } - } - git_cleanup($ibx); - if (my $gits = $ibx->{-repo_objs}) { - foreach my $git (@$gits) { - $again = 1 if $git->cleanup; - } - } +# returns true if further checking is required +sub check_inodes ($) { + for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} } +} + +sub do_cleanup { + my ($ibx) = @_; + my $live; + if (defined $ibx->{git}) { + $live = $ibx->isa(__PACKAGE__) ? $ibx->{git}->cleanup(1) + : $ibx->{git}->cleanup_if_unlinked; + delete($ibx->{git}) unless $live; + } + if ($live) { check_inodes($ibx); - if ($have_devel_peek) { - $again ||= !!$ibx->{search}; - } - $next->{"$ibx"} = $ibx if $again; + } else { + delete(@$ibx{qw(over mm description cloneurl + -imap_url -nntp_url)}); + } + my $srch = $ibx->{search} // $ibx; + delete @$srch{qw(xdb qp)}; + for my $git (@{$ibx->{-repo_objs} // []}) { + $live = 1 if $git->cleanup(1); } - $CLEANUP = $next; + PublicInbox::DS::add_uniq_timer($ibx+0, 5, \&do_cleanup, $ibx) if $live; } -sub cleanup_possible () { +sub _cleanup_later ($) { # no need to require DS, here, if it were enabled another # module would've require'd it, already - eval { PublicInbox::DS::in_loop() } or return 0; - - eval { - require Devel::Peek; # needs separate package in Fedora - $have_devel_peek = 1; - }; - 1; -} - -sub _cleanup_later ($) { - my ($self) = @_; - $cleanup_avail = cleanup_possible() if $cleanup_avail < 0; - return if $cleanup_avail != 1; - $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task); - $CLEANUP->{"$self"} = $self; + eval { PublicInbox::DS::in_loop() } and + PublicInbox::DS::add_uniq_timer($_[0]+0, 30, \&do_cleanup, @_) } sub _set_limiter ($$$) { my ($self, $pi_cfg, $pfx) = @_; my $lkey = "-${pfx}_limiter"; - $self->{$lkey} ||= do { + $self->{$lkey} //= do { # full key is: publicinbox.$NAME.httpbackendmax my $mkey = $pfx.'max'; my $val = $self->{$mkey} or return; @@ -109,8 +79,6 @@ sub new { } else { delete $opts->{feedmax}; } - $opts->{nntpserver} ||= $pi_cfg->{'publicinbox.nntpserver'}; - # allow any combination of multi-line or comma-delimited hide entries my $hide = {}; if (defined(my $h = $opts->{hide})) { @@ -128,11 +96,12 @@ sub version { sub git_epoch { my ($self, $epoch) = @_; # v2-only, callers always supply $epoch - $self->{"$epoch.git"} ||= do { + $self->{"$epoch.git"} //= do { my $git_dir = "$self->{inboxdir}/git/$epoch.git"; return unless -d $git_dir; my $g = PublicInbox::Git->new($git_dir); - $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + my $lim = $self->{-httpbackend_limiter}; + $g->{-httpbackend_limiter} = $lim if $lim; # caller must manually cleanup when done $g; }; @@ -140,11 +109,12 @@ sub git_epoch { sub git { my ($self) = @_; - $self->{git} ||= do { + $self->{git} //= do { my $git_dir = $self->{inboxdir}; $git_dir .= '/all.git' if $self->version == 2; my $g = PublicInbox::Git->new($git_dir); - $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + my $lim = $self->{-httpbackend_limiter}; + $g->{-httpbackend_limiter} = $lim if $lim; _cleanup_later($self); $g; }; @@ -156,7 +126,7 @@ sub max_git_epoch { my $cur = $self->{-max_git_epoch}; my $changed; if (!defined($cur) || ($changed = git($self)->alternates_changed)) { - git_cleanup($self) if $changed; + $self->{git}->cleanup if $changed; my $gits = "$self->{inboxdir}/git"; if (opendir my $dh, $gits) { my $max = max(map { @@ -168,27 +138,29 @@ sub max_git_epoch { $cur; } +sub mm_file { + my ($self) = @_; + my $d = $self->{inboxdir}; + ($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3'; +} + sub mm { my ($self, $req) = @_; $self->{mm} //= eval { require PublicInbox::Msgmap; - my $dir = $self->{inboxdir}; - if ($self->version >= 2) { - PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); - } else { - PublicInbox::Msgmap->new($dir); - } + _cleanup_later($self); + PublicInbox::Msgmap->new_file($self); } // ($req ? croak("E: $@") : undef); } sub search { my ($self) = @_; - my $srch = $self->{search} //= eval { + $self->{search} // eval { _cleanup_later($self); require PublicInbox::Search; - PublicInbox::Search->new($self); + my $srch = PublicInbox::Search->new($self); + (eval { $srch->xdb }) ? ($self->{search} = $srch) : undef; }; - (eval { $srch->xdb }) ? $srch : undef; } # isrch is preferred for read-only interfaces if available since it @@ -197,15 +169,15 @@ sub isrch { $_[0]->{isrch} // search($_[0]) } sub over { my ($self, $req) = @_; - $self->{over} //= eval { - my $srch = $self->{search} //= do { - _cleanup_later($self); + $self->{over} // eval { + my $srch = $self->{search} // do { require PublicInbox::Search; PublicInbox::Search->new($self); }; + _cleanup_later($self); my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3"); $over->dbh; # may fail - $over; + $self->{over} = $over; } // ($req ? croak("E: $@") : undef); } @@ -233,11 +205,11 @@ sub description { sub cloneurl { my ($self) = @_; - ($self->{cloneurl} //= do { + $self->{cloneurl} // do { my $s = try_cat("$self->{inboxdir}/cloneurl"); my @urls = split(/\s+/s, $s); - scalar(@urls) ? \@urls : undef - }) // []; + scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef; + } // []; } sub base_url { @@ -249,65 +221,67 @@ sub base_url { return $url .= $self->{name} . '/'; } # called from a non-PSGI environment (e.g. NNTP/POP3): - $self->{-base_url} ||= do { - my $url = $self->{url}->[0] or return undef; - # expand protocol-relative URLs to HTTPS if we're - # not inside a web server - $url = "https:$url" if $url =~ m!\A//!; - $url .= '/' if $url !~ m!/\z!; - $url; - }; -} - -sub nntp_url { - my ($self) = @_; - $self->{-nntp_url} ||= do { - # no checking for nntp_usable here, we can point entirely - # to non-local servers or users run by a different user - my $ns = $self->{nntpserver}; - my $group = $self->{newsgroup}; - my @urls; - if ($ns && $group) { - $ns = [ $ns ] if ref($ns) ne 'ARRAY'; - @urls = map { - my $u = m!\Anntps?://! ? $_ : "nntp://$_"; - $u .= '/' if $u !~ m!/\z!; - $u.$group; - } @$ns; - } - - my $mirrors = $self->{nntpmirror}; - if ($mirrors) { - my @m; - foreach (@$mirrors) { - my $u = m!\Anntps?://! ? $_ : "nntp://$_"; - if ($u =~ m!\Anntps?://[^/]+/?\z!) { - if ($group) { - $u .= '/' if $u !~ m!/\z!; - $u .= $group; - } else { - warn -"publicinbox.$self->{name}.nntpmirror=$_ missing newsgroup name\n"; - } + my $url = $self->{url} // return undef; + $url = $url->[0] // return undef; + # expand protocol-relative URLs to HTTPS if we're + # not inside a web server + substr($url, 0, 0, 'https:') if substr($url, 0, 2) eq '//'; + $url .= '/' if substr($url, -1, 1) ne '/'; + $url; +} + +# imapserver, nntpserver, and pop3server configs are used here: +sub _x_url ($$$) { + my ($self, $x, $ctx) = @_; # $x is "imap", "nntp", or "pop3" + # no checking for nntp_usable here, we can point entirely + # to non-local servers or users run by a different user + my $ns = $self->{"${x}server"} // + $ctx->{www}->{pi_cfg}->get_all("publicinbox.${x}server"); + my $group = $self->{newsgroup}; + my @urls; + if ($ns && $group) { + @urls = map { + my $u = m!\A${x}s?://! ? $_ : "$x://$_"; + $u .= '/' if $u !~ m!/\z!; + $u.$group; + } @$ns; + } + if (my $mirrors = $self->{"${x}mirror"}) { + my @m; + for (@$mirrors) { + my $u = m!\A${x}s?://! ? $_ : "$x://$_"; + if ($u =~ m!\A${x}s?://[^/]+/?\z!) { + if ($group) { + $u .= '/' if $u !~ m!/\z!; + $u .= $group; + } else { # n.b. IMAP and POP3 use "newsgroup" + warn <{name}.${x}mirror=$_ missing newsgroup name +EOM } - # else: allow full URLs like: - # nntp://news.example.com/alt.example - push @m, $u; } - - # List::Util::uniq requires Perl 5.26+, maybe we - # can use it by 2030 or so - my %seen; - @urls = grep { !$seen{$_}++ } (@urls, @m); + # else: allow full URLs like: + # nntp://news.example.com/alt.example + push @m, $u; } - \@urls; - }; + + # List::Util::uniq requires Perl 5.26+, maybe we + # can use it by 2030 or so + my %seen; + @urls = grep { !$seen{$_}++ } (@urls, @m); + } + \@urls; } +# my ($self, $ctx) = @_; +sub imap_url { $_[0]->{-imap_url} //= _x_url($_[0], 'imap', $_[1]) } +sub nntp_url { $_[0]->{-nntp_url} //= _x_url($_[0], 'nntp', $_[1]) } +sub pop3_url { $_[0]->{-pop3_url} //= _x_url($_[0], 'pop3', $_[1]) } + sub nntp_usable { my ($self) = @_; my $ret = mm($self) && over($self); - $self->{mm} = $self->{over} = $self->{search} = undef; + delete @$self{qw(mm over search)}; $ret; } @@ -322,17 +296,15 @@ sub msg_by_smsg ($$) { # ghosts may have undef smsg (from SearchThread.node) or # no {blob} field - return unless defined $smsg; - defined(my $blob = $smsg->{blob}) or return; - - $self->git->cat_file($blob); + $smsg // return; + $self->git->cat_file($smsg->{blob} // return); } sub smsg_eml { my ($self, $smsg) = @_; my $bref = msg_by_smsg($self, $smsg) or return; my $eml = PublicInbox::Eml->new($bref); - $smsg->populate($eml) unless exists($smsg->{num}); # v1 w/o SQLite + $smsg->{num} // $smsg->populate($eml); $eml; } @@ -342,7 +314,7 @@ sub smsg_by_mid ($$) { my $smsg; if (my $mm = $self->mm) { # favor the Message-ID we used for the NNTP article number: - defined(my $num = $mm->num_for($mid)) or return; + my $num = $mm->num_for($mid) // return; $smsg = $over->get_art($num); } else { my ($id, $prev); @@ -378,7 +350,7 @@ sub modified { # (pathname is NOT public, but prefix is used for Xapian queries) sub altid_map ($) { my ($self) = @_; - $self->{-altid_map} //= eval { + eval { require PublicInbox::AltId; my $altid = $self->{altid} or return {}; my %h = map {; @@ -400,13 +372,6 @@ sub unsubscribe_unlock { delete $self->{unlock_subs}->{$ident}; } -sub check_inodes ($) { - my ($self) = @_; - for (qw(over mm)) { # TODO: search - $self->{$_}->check_inodes if $self->{$_}; - } -} - # called by inotify sub on_unlock { my ($self) = @_;