X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=bee44f8a92e7ae8861b01064ff10831605b51a1f;hb=af0b0fb7a454470a32c452119d0392e0dedb3fe1;hp=02186dac717fa329b6fb508d8eeab2c18919bca0;hpb=6adbc635ea6ab754462c74c1056a40eb68c6c78e;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 02186dac..bee44f8a 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,13 +1,13 @@ -# Copyright (C) 2016-2020 all contributors +# Copyright (C) 2016-2021 all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) package PublicInbox::Inbox; use strict; -use warnings; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); use PublicInbox::Eml; +use List::Util qw(max); # Long-running "git-cat-file --batch" processes won't notice # unlinked packs, so we need to restart those processes occasionally. @@ -31,7 +31,7 @@ sub cleanup_task () { for my $ibx (values %$CLEANUP) { my $again; if ($have_devel_peek) { - foreach my $f (qw(mm search)) { + foreach my $f (qw(search)) { # we bump refcnt by assigning tmp, here: my $tmp = $ibx->{$f} or next; next if Devel::Peek::SvREFCNT($tmp) > 2; @@ -47,7 +47,7 @@ sub cleanup_task () { } check_inodes($ibx); if ($have_devel_peek) { - $again ||= !!($ibx->{mm} || $ibx->{search}); + $again ||= !!$ibx->{search}; } $next->{"$ibx"} = $ibx if $again; } @@ -70,22 +70,12 @@ sub _cleanup_later ($) { my ($self) = @_; $cleanup_avail = cleanup_possible() if $cleanup_avail < 0; return if $cleanup_avail != 1; - $cleanup_timer ||= PublicInbox::DS::later(*cleanup_task); + $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task); $CLEANUP->{"$self"} = $self; } -sub _set_uint ($$$) { - my ($opts, $field, $default) = @_; - my $val = $opts->{$field}; - if (defined $val) { - $val = $val->[-1] if ref($val) eq 'ARRAY'; - $val = undef if $val !~ /\A[0-9]+\z/; - } - $opts->{$field} = $val || $default; -} - sub _set_limiter ($$$) { - my ($self, $pi_config, $pfx) = @_; + my ($self, $pi_cfg, $pfx) = @_; my $lkey = "-${pfx}_limiter"; $self->{$lkey} ||= do { # full key is: publicinbox.$NAME.httpbackendmax @@ -96,7 +86,7 @@ sub _set_limiter ($$$) { require PublicInbox::Qspawn; $lim = PublicInbox::Qspawn::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { - $lim = $pi_config->limiter($val); + $lim = $pi_cfg->limiter($val); warn "$mkey limiter=$val not found\n" if !$lim; } else { warn "$mkey limiter=$val not understood\n"; @@ -110,14 +100,15 @@ sub new { my $v = $opts->{address} ||= [ 'public-inbox@example.com' ]; my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; - my $pi_config = delete $opts->{-pi_config}; - _set_limiter($opts, $pi_config, 'httpbackend'); - _set_uint($opts, 'feedmax', 25); - $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; - my $dir = $opts->{inboxdir}; - if (defined $dir && -f "$dir/inbox.lock") { - $opts->{version} = 2; + my $pi_cfg = delete $opts->{-pi_cfg}; + _set_limiter($opts, $pi_cfg, 'httpbackend'); + my $fmax = $opts->{feedmax}; + if (defined($fmax) && $fmax =~ /\A[0-9]+\z/) { + $opts->{feedmax} += 0; + } else { + delete $opts->{feedmax}; } + $opts->{nntpserver} ||= $pi_cfg->{'publicinbox.nntpserver'}; # allow any combination of multi-line or comma-delimited hide entries my $hide = {}; @@ -130,16 +121,18 @@ sub new { bless $opts, $class; } -sub version { $_[0]->{version} // 1 } +sub version { + $_[0]->{version} //= -f "$_[0]->{inboxdir}/inbox.lock" ? 2 : 1 +} sub git_epoch { - my ($self, $epoch) = @_; - $self->version == 2 or return; + my ($self, $epoch) = @_; # v2-only, callers always supply $epoch $self->{"$epoch.git"} ||= do { my $git_dir = "$self->{inboxdir}/git/$epoch.git"; + return unless -d $git_dir; my $g = PublicInbox::Git->new($git_dir); $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; - # no cleanup needed, we never cat-file off this, only clone + # caller must manually cleanup when done $g; }; } @@ -160,19 +153,15 @@ sub max_git_epoch { my ($self) = @_; return if $self->version < 2; my $cur = $self->{-max_git_epoch}; - my $changed = git($self)->alternates_changed; - if (!defined($cur) || $changed) { + my $changed; + if (!defined($cur) || ($changed = git($self)->alternates_changed)) { git_cleanup($self) if $changed; my $gits = "$self->{inboxdir}/git"; if (opendir my $dh, $gits) { - my $max = -1; - while (defined(my $git_dir = readdir($dh))) { - $git_dir =~ m!\A([0-9]+)\.git\z! or next; - $max = $1 if $1 > $max; - } - $cur = $self->{-max_git_epoch} = $max if $max >= 0; - } else { - warn "opendir $gits failed: $!\n"; + my $max = max(map { + substr($_, 0, -4) + 0; # drop ".git" suffix + } grep(/\A[0-9]+\.git\z/, readdir($dh))) // return; + $cur = $self->{-max_git_epoch} = $max; } } $cur; @@ -182,7 +171,6 @@ sub mm { my ($self) = @_; $self->{mm} ||= eval { require PublicInbox::Msgmap; - _cleanup_later($self); my $dir = $self->{inboxdir}; if ($self->version >= 2) { PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); @@ -192,46 +180,54 @@ sub mm { }; } -sub search ($;$) { - my ($self, $over_only) = @_; - my $srch = $self->{search} ||= eval { +sub search { + my ($self) = @_; + my $srch = $self->{search} //= eval { _cleanup_later($self); require PublicInbox::Search; PublicInbox::Search->new($self); }; - ($over_only || eval { $srch->xdb }) ? $srch : undef; -} - -sub over ($) { - my ($self) = @_; - my $srch = search($self, 1) or return; - $self->{over} ||= eval { - my $over = $srch->{over_ro}; - $over->dbh_new; # may fail + (eval { $srch->xdb }) ? $srch : undef; +} + +# isrch is preferred for read-only interfaces if available since it +# reduces kernel cache and FD overhead +sub isrch { $_[0]->{isrch} // search($_[0]) } + +sub over { + $_[0]->{over} //= eval { + my $srch = $_[0]->{search} //= eval { + _cleanup_later($_[0]); + require PublicInbox::Search; + PublicInbox::Search->new($_[0]); + }; + my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3"); + $over->dbh; # may fail $over; - } + }; } + sub try_cat { my ($path) = @_; - my $rv = ''; - if (open(my $fh, '<', $path)) { - local $/; - $rv = <$fh>; - } - $rv; + open(my $fh, '<', $path) or return ''; + local $/; + <$fh> // ''; +} + +sub cat_desc ($) { + my $desc = try_cat($_[0]); + local $/ = "\n"; + chomp $desc; + utf8::decode($desc); + $desc =~ s/\s+/ /smg; + $desc eq '' ? undef : $desc; } sub description { my ($self) = @_; - ($self->{description} //= do { - my $desc = try_cat("$self->{inboxdir}/description"); - local $/ = "\n"; - chomp $desc; - utf8::decode($desc); - $desc =~ s/\s+/ /smg; - $desc eq '' ? undef : $desc; - }) // '($INBOX_DIR/description missing)'; + ($self->{description} //= cat_desc("$self->{inboxdir}/description")) // + '($INBOX_DIR/description missing)'; } sub cloneurl { @@ -328,7 +324,7 @@ sub msg_by_smsg ($$) { return unless defined $smsg; defined(my $blob = $smsg->{blob}) or return; - git($self)->cat_file($blob); + $self->git->cat_file($blob); } sub smsg_eml { @@ -339,39 +335,35 @@ sub smsg_eml { $eml; } -sub mid2num($$) { - my ($self, $mid) = @_; - my $mm = mm($self) or return; - $mm->num_for($mid); -} - sub smsg_by_mid ($$) { my ($self, $mid) = @_; - my $over = over($self) or return; - # favor the Message-ID we used for the NNTP article number: - defined(my $num = mid2num($self, $mid)) or return; - my $smsg = $over->get_art($num) or return; - PublicInbox::Smsg::psgi_cull($smsg); + my $over = $self->over or return; + my $smsg; + if (my $mm = $self->mm) { + # favor the Message-ID we used for the NNTP article number: + defined(my $num = $mm->num_for($mid)) or return; + $smsg = $over->get_art($num); + } else { + my ($id, $prev); + $smsg = $over->next_by_mid($mid, \$id, \$prev); + } + $smsg ? PublicInbox::Smsg::psgi_cull($smsg) : undef; } sub msg_by_mid ($$) { my ($self, $mid) = @_; - - over($self) or - return msg_by_path($self, mid2path($mid)); - my $smsg = smsg_by_mid($self, $mid); - $smsg ? msg_by_smsg($self, $smsg) : undef; + $smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid)); } sub recent { my ($self, $opts, $after, $before) = @_; - over($self)->recent($opts, $after, $before); + $self->over->recent($opts, $after, $before); } sub modified { my ($self) = @_; - if (my $over = over($self)) { + if (my $over = $self->over) { my $msgs = $over->recent({limit => 1}); if (my $smsg = $msgs->[0]) { return $smsg->{ts}; @@ -409,7 +401,7 @@ sub unsubscribe_unlock { sub check_inodes ($) { my ($self) = @_; - for (qw(over)) { # TODO: search, mm + for (qw(over mm)) { # TODO: search $self->{$_}->check_inodes if $self->{$_}; } } @@ -419,10 +411,14 @@ sub on_unlock { my ($self) = @_; check_inodes($self); my $subs = $self->{unlock_subs} or return; - for (values %$subs) { - eval { $_->on_inbox_unlock($self) }; + for my $obj (values %$subs) { + eval { $obj->on_inbox_unlock($self) }; warn "E: $@ ($self->{inboxdir})\n" if $@; } } +sub uidvalidity { $_[0]->{uidvalidity} //= eval { $_[0]->mm->created_at } } + +sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } + 1;