X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=bee44f8a92e7ae8861b01064ff10831605b51a1f;hb=af0b0fb7a454470a32c452119d0392e0dedb3fe1;hp=c295b2677e48299cbfbacfc53f36136ad73f479f;hpb=846161e3d1207d59f62b3a6718221d6f5ba2b94f;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index c295b267..bee44f8a 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,13 +1,13 @@ -# Copyright (C) 2016-2020 all contributors +# Copyright (C) 2016-2021 all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) package PublicInbox::Inbox; use strict; -use warnings; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); use PublicInbox::Eml; +use List::Util qw(max); # Long-running "git-cat-file --batch" processes won't notice # unlinked packs, so we need to restart those processes occasionally. @@ -18,13 +18,20 @@ my $cleanup_timer; my $cleanup_avail = -1; # 0, or 1 my $have_devel_peek; my $CLEANUP = {}; # string(inbox) -> inbox + +sub git_cleanup ($) { + my ($self) = @_; + my $git = $self->{git} or return; + $git->cleanup; +} + sub cleanup_task () { $cleanup_timer = undef; my $next = {}; for my $ibx (values %$CLEANUP) { my $again; if ($have_devel_peek) { - foreach my $f (qw(mm search over)) { + foreach my $f (qw(search)) { # we bump refcnt by assigning tmp, here: my $tmp = $ibx->{$f} or next; next if Devel::Peek::SvREFCNT($tmp) > 2; @@ -32,17 +39,15 @@ sub cleanup_task () { # refcnt is zero when tmp is out-of-scope } } - if (my $git = $ibx->{git}) { - $again = $git->cleanup; - } + git_cleanup($ibx); if (my $gits = $ibx->{-repo_objs}) { foreach my $git (@$gits) { $again = 1 if $git->cleanup; } } + check_inodes($ibx); if ($have_devel_peek) { - $again ||= !!($ibx->{over} || $ibx->{mm} || - $ibx->{search}); + $again ||= !!$ibx->{search}; } $next->{"$ibx"} = $ibx if $again; } @@ -65,22 +70,12 @@ sub _cleanup_later ($) { my ($self) = @_; $cleanup_avail = cleanup_possible() if $cleanup_avail < 0; return if $cleanup_avail != 1; - $cleanup_timer ||= PublicInbox::DS::later(*cleanup_task); + $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task); $CLEANUP->{"$self"} = $self; } -sub _set_uint ($$$) { - my ($opts, $field, $default) = @_; - my $val = $opts->{$field}; - if (defined $val) { - $val = $val->[-1] if ref($val) eq 'ARRAY'; - $val = undef if $val !~ /\A[0-9]+\z/; - } - $opts->{$field} = $val || $default; -} - sub _set_limiter ($$$) { - my ($self, $pi_config, $pfx) = @_; + my ($self, $pi_cfg, $pfx) = @_; my $lkey = "-${pfx}_limiter"; $self->{$lkey} ||= do { # full key is: publicinbox.$NAME.httpbackendmax @@ -91,7 +86,7 @@ sub _set_limiter ($$$) { require PublicInbox::Qspawn; $lim = PublicInbox::Qspawn::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { - $lim = $pi_config->limiter($val); + $lim = $pi_cfg->limiter($val); warn "$mkey limiter=$val not found\n" if !$lim; } else { warn "$mkey limiter=$val not understood\n"; @@ -105,14 +100,15 @@ sub new { my $v = $opts->{address} ||= [ 'public-inbox@example.com' ]; my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; - my $pi_config = delete $opts->{-pi_config}; - _set_limiter($opts, $pi_config, 'httpbackend'); - _set_uint($opts, 'feedmax', 25); - $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; - my $dir = $opts->{inboxdir}; - if (defined $dir && -f "$dir/inbox.lock") { - $opts->{version} = 2; + my $pi_cfg = delete $opts->{-pi_cfg}; + _set_limiter($opts, $pi_cfg, 'httpbackend'); + my $fmax = $opts->{feedmax}; + if (defined($fmax) && $fmax =~ /\A[0-9]+\z/) { + $opts->{feedmax} += 0; + } else { + delete $opts->{feedmax}; } + $opts->{nntpserver} ||= $pi_cfg->{'publicinbox.nntpserver'}; # allow any combination of multi-line or comma-delimited hide entries my $hide = {}; @@ -125,16 +121,18 @@ sub new { bless $opts, $class; } -sub version { $_[0]->{version} // 1 } +sub version { + $_[0]->{version} //= -f "$_[0]->{inboxdir}/inbox.lock" ? 2 : 1 +} sub git_epoch { - my ($self, $epoch) = @_; - $self->version == 2 or return; + my ($self, $epoch) = @_; # v2-only, callers always supply $epoch $self->{"$epoch.git"} ||= do { my $git_dir = "$self->{inboxdir}/git/$epoch.git"; + return unless -d $git_dir; my $g = PublicInbox::Git->new($git_dir); $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; - # no cleanup needed, we never cat-file off this, only clone + # caller must manually cleanup when done $g; }; } @@ -155,19 +153,15 @@ sub max_git_epoch { my ($self) = @_; return if $self->version < 2; my $cur = $self->{-max_git_epoch}; - my $changed = git($self)->alternates_changed; - if (!defined($cur) || $changed) { - $self->git->cleanup if $changed; + my $changed; + if (!defined($cur) || ($changed = git($self)->alternates_changed)) { + git_cleanup($self) if $changed; my $gits = "$self->{inboxdir}/git"; if (opendir my $dh, $gits) { - my $max = -1; - while (defined(my $git_dir = readdir($dh))) { - $git_dir =~ m!\A([0-9]+)\.git\z! or next; - $max = $1 if $1 > $max; - } - $cur = $self->{-max_git_epoch} = $max if $max >= 0; - } else { - warn "opendir $gits failed: $!\n"; + my $max = max(map { + substr($_, 0, -4) + 0; # drop ".git" suffix + } grep(/\A[0-9]+\.git\z/, readdir($dh))) // return; + $cur = $self->{-max_git_epoch} = $max; } } $cur; @@ -177,7 +171,6 @@ sub mm { my ($self) = @_; $self->{mm} ||= eval { require PublicInbox::Msgmap; - _cleanup_later($self); my $dir = $self->{inboxdir}; if ($self->version >= 2) { PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); @@ -187,46 +180,54 @@ sub mm { }; } -sub search ($;$) { - my ($self, $over_only) = @_; - my $srch = $self->{search} ||= eval { +sub search { + my ($self) = @_; + my $srch = $self->{search} //= eval { _cleanup_later($self); require PublicInbox::Search; PublicInbox::Search->new($self); }; - ($over_only || eval { $srch->xdb }) ? $srch : undef; + (eval { $srch->xdb }) ? $srch : undef; } -sub over ($) { - my ($self) = @_; - my $srch = search($self, 1) or return; - $self->{over} ||= eval { - my $over = $srch->{over_ro}; - $over->dbh_new; # may fail +# isrch is preferred for read-only interfaces if available since it +# reduces kernel cache and FD overhead +sub isrch { $_[0]->{isrch} // search($_[0]) } + +sub over { + $_[0]->{over} //= eval { + my $srch = $_[0]->{search} //= eval { + _cleanup_later($_[0]); + require PublicInbox::Search; + PublicInbox::Search->new($_[0]); + }; + my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3"); + $over->dbh; # may fail $over; - } + }; } + sub try_cat { my ($path) = @_; - my $rv = ''; - if (open(my $fh, '<', $path)) { - local $/; - $rv = <$fh>; - } - $rv; + open(my $fh, '<', $path) or return ''; + local $/; + <$fh> // ''; +} + +sub cat_desc ($) { + my $desc = try_cat($_[0]); + local $/ = "\n"; + chomp $desc; + utf8::decode($desc); + $desc =~ s/\s+/ /smg; + $desc eq '' ? undef : $desc; } sub description { my ($self) = @_; - ($self->{description} //= do { - my $desc = try_cat("$self->{inboxdir}/description"); - local $/ = "\n"; - chomp $desc; - utf8::decode($desc); - $desc =~ s/\s+/ /smg; - $desc eq '' ? undef : $desc; - }) // '($INBOX_DIR/description missing)'; + ($self->{description} //= cat_desc("$self->{inboxdir}/description")) // + '($INBOX_DIR/description missing)'; } sub cloneurl { @@ -310,63 +311,59 @@ sub nntp_usable { } # for v1 users w/o SQLite only -sub msg_by_path ($$;$) { - my ($self, $path, $ref) = @_; - git($self)->cat_file('HEAD:'.$path, $ref); +sub msg_by_path ($$) { + my ($self, $path) = @_; + git($self)->cat_file('HEAD:'.$path); } -sub msg_by_smsg ($$;$) { - my ($self, $smsg, $ref) = @_; +sub msg_by_smsg ($$) { + my ($self, $smsg) = @_; # ghosts may have undef smsg (from SearchThread.node) or # no {blob} field return unless defined $smsg; defined(my $blob = $smsg->{blob}) or return; - git($self)->cat_file($blob, $ref); -} - -sub smsg_mime { - my ($self, $smsg, $ref) = @_; - if (my $s = msg_by_smsg($self, $smsg, $ref)) { - $smsg->{mime} = PublicInbox::Eml->new($s); - return $smsg; - } + $self->git->cat_file($blob); } -sub mid2num($$) { - my ($self, $mid) = @_; - my $mm = mm($self) or return; - $mm->num_for($mid); +sub smsg_eml { + my ($self, $smsg) = @_; + my $bref = msg_by_smsg($self, $smsg) or return; + my $eml = PublicInbox::Eml->new($bref); + $smsg->populate($eml) unless exists($smsg->{num}); # v1 w/o SQLite + $eml; } sub smsg_by_mid ($$) { my ($self, $mid) = @_; - my $over = over($self) or return; - # favor the Message-ID we used for the NNTP article number: - defined(my $num = mid2num($self, $mid)) or return; - my $smsg = $over->get_art($num) or return; - PublicInbox::Smsg::psgi_cull($smsg); + my $over = $self->over or return; + my $smsg; + if (my $mm = $self->mm) { + # favor the Message-ID we used for the NNTP article number: + defined(my $num = $mm->num_for($mid)) or return; + $smsg = $over->get_art($num); + } else { + my ($id, $prev); + $smsg = $over->next_by_mid($mid, \$id, \$prev); + } + $smsg ? PublicInbox::Smsg::psgi_cull($smsg) : undef; } -sub msg_by_mid ($$;$) { - my ($self, $mid, $ref) = @_; - - over($self) or - return msg_by_path($self, mid2path($mid), $ref); - +sub msg_by_mid ($$) { + my ($self, $mid) = @_; my $smsg = smsg_by_mid($self, $mid); - $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; + $smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid)); } sub recent { my ($self, $opts, $after, $before) = @_; - over($self)->recent($opts, $after, $before); + $self->over->recent($opts, $after, $before); } sub modified { my ($self) = @_; - if (my $over = over($self)) { + if (my $over = $self->over) { my $msgs = $over->recent({limit => 1}); if (my $smsg = $msgs->[0]) { return $smsg->{ts}; @@ -391,4 +388,37 @@ sub altid_map ($) { } // {}; } +# $obj must respond to ->on_inbox_unlock, which takes Inbox ($self) as an arg +sub subscribe_unlock { + my ($self, $ident, $obj) = @_; + $self->{unlock_subs}->{$ident} = $obj; +} + +sub unsubscribe_unlock { + my ($self, $ident) = @_; + delete $self->{unlock_subs}->{$ident}; +} + +sub check_inodes ($) { + my ($self) = @_; + for (qw(over mm)) { # TODO: search + $self->{$_}->check_inodes if $self->{$_}; + } +} + +# called by inotify +sub on_unlock { + my ($self) = @_; + check_inodes($self); + my $subs = $self->{unlock_subs} or return; + for my $obj (values %$subs) { + eval { $obj->on_inbox_unlock($self) }; + warn "E: $@ ($self->{inboxdir})\n" if $@; + } +} + +sub uidvalidity { $_[0]->{uidvalidity} //= eval { $_[0]->mm->created_at } } + +sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } + 1;