X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=3b5ac970d07cf43af6d86b656d92606f108cefc9;hb=53c1b42f1f8bf5b50acbf25cbe77e70ca15d5c77;hp=ff80096528eb2301aae63c9389dba01421051868;hpb=cb1c874520153f5c28a932959446977062c7db69;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index ff800965..3b5ac970 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2019 all contributors +# Copyright (C) 2016-2020 all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) @@ -7,7 +7,7 @@ use strict; use warnings; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); -use PublicInbox::MIME; +use PublicInbox::Eml; # Long-running "git-cat-file --batch" processes won't notice # unlinked packs, so we need to restart those processes occasionally. @@ -18,13 +18,20 @@ my $cleanup_timer; my $cleanup_avail = -1; # 0, or 1 my $have_devel_peek; my $CLEANUP = {}; # string(inbox) -> inbox + +sub git_cleanup ($) { + my ($self) = @_; + my $git = $self->{git} or return; + $git->cleanup; +} + sub cleanup_task () { $cleanup_timer = undef; my $next = {}; for my $ibx (values %$CLEANUP) { my $again; if ($have_devel_peek) { - foreach my $f (qw(mm search over)) { + foreach my $f (qw(search)) { # we bump refcnt by assigning tmp, here: my $tmp = $ibx->{$f} or next; next if Devel::Peek::SvREFCNT($tmp) > 2; @@ -32,17 +39,15 @@ sub cleanup_task () { # refcnt is zero when tmp is out-of-scope } } - if (my $git = $ibx->{git}) { - $again = $git->cleanup; - } + git_cleanup($ibx); if (my $gits = $ibx->{-repo_objs}) { foreach my $git (@$gits) { $again = 1 if $git->cleanup; } } + check_inodes($ibx); if ($have_devel_peek) { - $again ||= !!($ibx->{over} || $ibx->{mm} || - $ibx->{search}); + $again ||= !!$ibx->{search}; } $next->{"$ibx"} = $ibx if $again; } @@ -52,7 +57,7 @@ sub cleanup_task () { sub cleanup_possible () { # no need to require DS, here, if it were enabled another # module would've require'd it, already - eval { $PublicInbox::DS::in_loop } or return 0; + eval { PublicInbox::DS::in_loop() } or return 0; eval { require Devel::Peek; # needs separate package in Fedora @@ -82,7 +87,7 @@ sub _set_uint ($$$) { sub _set_limiter ($$$) { my ($self, $pi_config, $pfx) = @_; my $lkey = "-${pfx}_limiter"; - $self->{$lkey} ||= eval { + $self->{$lkey} ||= do { # full key is: publicinbox.$NAME.httpbackendmax my $mkey = $pfx.'max'; my $val = $self->{$mkey} or return; @@ -102,7 +107,7 @@ sub _set_limiter ($$$) { sub new { my ($class, $opts) = @_; - my $v = $opts->{address} ||= 'public-inbox@example.com'; + my $v = $opts->{address} ||= [ 'public-inbox@example.com' ]; my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; my $pi_config = delete $opts->{-pi_config}; @@ -125,10 +130,12 @@ sub new { bless $opts, $class; } +sub version { $_[0]->{version} // 1 } + sub git_epoch { my ($self, $epoch) = @_; - ($self->{version} || 1) == 2 or return; - $self->{"$epoch.git"} ||= eval { + $self->version == 2 or return; + $self->{"$epoch.git"} ||= do { my $git_dir = "$self->{inboxdir}/git/$epoch.git"; my $g = PublicInbox::Git->new($git_dir); $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; @@ -139,9 +146,9 @@ sub git_epoch { sub git { my ($self) = @_; - $self->{git} ||= eval { + $self->{git} ||= do { my $git_dir = $self->{inboxdir}; - $git_dir .= '/all.git' if (($self->{version} || 1) == 2); + $git_dir .= '/all.git' if $self->version == 2; my $g = PublicInbox::Git->new($git_dir); $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; _cleanup_later($self); @@ -151,12 +158,11 @@ sub git { sub max_git_epoch { my ($self) = @_; - my $v = $self->{version}; - return unless defined($v) && $v == 2; + return if $self->version < 2; my $cur = $self->{-max_git_epoch}; my $changed = git($self)->alternates_changed; if (!defined($cur) || $changed) { - $self->git->cleanup if $changed; + git_cleanup($self) if $changed; my $gits = "$self->{inboxdir}/git"; if (opendir my $dh, $gits) { my $max = -1; @@ -176,9 +182,8 @@ sub mm { my ($self) = @_; $self->{mm} ||= eval { require PublicInbox::Msgmap; - _cleanup_later($self); my $dir = $self->{inboxdir}; - if (($self->{version} || 1) >= 2) { + if ($self->version >= 2) { PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); } else { PublicInbox::Msgmap->new($dir); @@ -186,24 +191,28 @@ sub mm { }; } -sub search ($;$) { - my ($self, $over_only) = @_; +sub search ($;$$) { + my ($self, $over_only, $ctx) = @_; my $srch = $self->{search} ||= eval { _cleanup_later($self); require PublicInbox::Search; PublicInbox::Search->new($self); }; - ($over_only || eval { $srch->xdb }) ? $srch : undef; + ($over_only || eval { $srch->xdb }) ? $srch : do { + $ctx and $ctx->{env}->{'psgi.errors'}->print(<{name}' search went away unexpectedly +EOF + undef; + }; } -sub over ($) { - my ($self) = @_; - my $srch = search($self, 1) or return; - $self->{over} ||= eval { - my $over = $srch->{over_ro}; - $over->dbh_new; # may fail +sub over { + $_[0]->{over} //= eval { + my $srch = search($_[0], 1) or return; + my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3"); + $over->dbh; # may fail $over; - } + }; } sub try_cat { @@ -218,25 +227,23 @@ sub try_cat { sub description { my ($self) = @_; - my $desc = $self->{description}; - return $desc if defined $desc; - $desc = try_cat("$self->{inboxdir}/description"); - local $/ = "\n"; - chomp $desc; - $desc =~ s/\s+/ /smg; - $desc = '($INBOX_DIR/description missing)' if $desc eq ''; - $self->{description} = $desc; + ($self->{description} //= do { + my $desc = try_cat("$self->{inboxdir}/description"); + local $/ = "\n"; + chomp $desc; + utf8::decode($desc); + $desc =~ s/\s+/ /smg; + $desc eq '' ? undef : $desc; + }) // '($INBOX_DIR/description missing)'; } sub cloneurl { my ($self) = @_; - my $url = $self->{cloneurl}; - return $url if $url; - $url = try_cat("$self->{inboxdir}/cloneurl"); - my @url = split(/\s+/s, $url); - local $/ = "\n"; - chomp @url; - $self->{cloneurl} = \@url; + ($self->{cloneurl} //= do { + my $s = try_cat("$self->{inboxdir}/cloneurl"); + my @urls = split(/\s+/s, $s); + scalar(@urls) ? \@urls : undef + }) // []; } sub base_url { @@ -293,12 +300,11 @@ sub nntp_url { # nntp://news.example.com/alt.example push @m, $u; } - my %seen = map { $_ => 1 } @urls; - foreach my $u (@m) { - next if $seen{$u}; - $seen{$u} = 1; - push @urls, $u; - } + + # List::Util::uniq requires Perl 5.26+, maybe we + # can use it by 2030 or so + my %seen; + @urls = grep { !$seen{$_}++ } (@urls, @m); } \@urls; }; @@ -311,33 +317,29 @@ sub nntp_usable { $ret; } -sub msg_by_path ($$;$) { - my ($self, $path, $ref) = @_; - # TODO: allow other refs: - my $str = git($self)->cat_file('HEAD:'.$path, $ref); - $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; - $str; +# for v1 users w/o SQLite only +sub msg_by_path ($$) { + my ($self, $path) = @_; + git($self)->cat_file('HEAD:'.$path); } -sub msg_by_smsg ($$;$) { - my ($self, $smsg, $ref) = @_; +sub msg_by_smsg ($$) { + my ($self, $smsg) = @_; # ghosts may have undef smsg (from SearchThread.node) or # no {blob} field return unless defined $smsg; defined(my $blob = $smsg->{blob}) or return; - my $str = git($self)->cat_file($blob, $ref); - $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; - $str; + git($self)->cat_file($blob); } -sub smsg_mime { - my ($self, $smsg, $ref) = @_; - if (my $s = msg_by_smsg($self, $smsg, $ref)) { - $smsg->{mime} = PublicInbox::MIME->new($s); - return $smsg; - } +sub smsg_eml { + my ($self, $smsg) = @_; + my $bref = msg_by_smsg($self, $smsg) or return; + my $eml = PublicInbox::Eml->new($bref); + $smsg->populate($eml) unless exists($smsg->{num}); # v1 w/o SQLite + $eml; } sub mid2num($$) { @@ -352,17 +354,17 @@ sub smsg_by_mid ($$) { # favor the Message-ID we used for the NNTP article number: defined(my $num = mid2num($self, $mid)) or return; my $smsg = $over->get_art($num) or return; - PublicInbox::SearchMsg::psgi_cull($smsg); + PublicInbox::Smsg::psgi_cull($smsg); } -sub msg_by_mid ($$;$) { - my ($self, $mid, $ref) = @_; +sub msg_by_mid ($$) { + my ($self, $mid) = @_; over($self) or - return msg_by_path($self, mid2path($mid), $ref); + return msg_by_path($self, mid2path($mid)); my $smsg = smsg_by_mid($self, $mid); - $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; + $smsg ? msg_by_smsg($self, $smsg) : undef; } sub recent { @@ -382,4 +384,48 @@ sub modified { git($self)->modified; # v1 } +# returns prefix => pathname mapping +# (pathname is NOT public, but prefix is used for Xapian queries) +sub altid_map ($) { + my ($self) = @_; + $self->{-altid_map} //= eval { + require PublicInbox::AltId; + my $altid = $self->{altid} or return {}; + my %h = map {; + my $x = PublicInbox::AltId->new($self, $_); + "$x->{prefix}" => $x->{filename} + } @$altid; + \%h; + } // {}; +} + +# $obj must respond to ->on_inbox_unlock, which takes Inbox ($self) as an arg +sub subscribe_unlock { + my ($self, $ident, $obj) = @_; + $self->{unlock_subs}->{$ident} = $obj; +} + +sub unsubscribe_unlock { + my ($self, $ident) = @_; + delete $self->{unlock_subs}->{$ident}; +} + +sub check_inodes ($) { + my ($self) = @_; + for (qw(over mm)) { # TODO: search + $self->{$_}->check_inodes if $self->{$_}; + } +} + +# called by inotify +sub on_unlock { + my ($self) = @_; + check_inodes($self); + my $subs = $self->{unlock_subs} or return; + for (values %$subs) { + eval { $_->on_inbox_unlock($self) }; + warn "E: $@ ($self->{inboxdir})\n" if $@; + } +} + 1;