X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=07e8b5b754071de5c8264d9e95034469e45b1128;hb=ff2db829115a5db9cc4d046b4a9018cfba4ca6c0;hp=aa4e141f6cfcd36a67bca71893a6363b8b0005c7;hpb=492910d43c179a7e1919b2d0f417aea9473f9730;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index aa4e141f..07e8b5b7 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016 all contributors +# Copyright (C) 2016-2019 all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) @@ -7,24 +7,65 @@ use strict; use warnings; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); +use PublicInbox::MIME; +# Long-running "git-cat-file --batch" processes won't notice +# unlinked packs, so we need to restart those processes occasionally. +# Xapian and SQLite file handles are mostly stable, but sometimes an +# admin will attempt to replace them atomically after compact/vacuum +# and we need to be prepared for that. my $cleanup_timer; -eval { - $cleanup_timer = 'disabled'; - require PublicInbox::EvCleanup; - $cleanup_timer = undef; # OK if we get here -}; - +my $cleanup_avail = -1; # 0, or 1 +my $have_devel_peek; my $CLEANUP = {}; # string(inbox) -> inbox sub cleanup_task () { $cleanup_timer = undef; - delete $_->{git} for values %$CLEANUP; - $CLEANUP = {}; + my $next = {}; + for my $ibx (values %$CLEANUP) { + my $again; + if ($have_devel_peek) { + foreach my $f (qw(mm search over)) { + # we bump refcnt by assigning tmp, here: + my $tmp = $ibx->{$f} or next; + next if Devel::Peek::SvREFCNT($tmp) > 2; + delete $ibx->{$f}; + # refcnt is zero when tmp is out-of-scope + } + } + if (my $git = $ibx->{git}) { + $again = $git->cleanup; + } + if (my $gits = $ibx->{-repo_objs}) { + foreach my $git (@$gits) { + $again = 1 if $git->cleanup; + } + } + if ($have_devel_peek) { + $again ||= !!($ibx->{over} || $ibx->{mm} || + $ibx->{search}); + } + $next->{"$ibx"} = $ibx if $again; + } + $CLEANUP = $next; +} + +sub cleanup_possible () { + # no need to require DS, here, if it were enabled another + # module would've require'd it, already + eval { PublicInbox::DS::in_loop() } or return 0; + + eval { + require Devel::Peek; # needs separate package in Fedora + $have_devel_peek = 1; + }; + 1; } sub _cleanup_later ($) { my ($self) = @_; - $cleanup_timer ||= PublicInbox::EvCleanup::later(*cleanup_task); + $cleanup_avail = cleanup_possible() if $cleanup_avail < 0; + return if $cleanup_avail != 1; + $cleanup_timer ||= PublicInbox::DS::later(*cleanup_task); $CLEANUP->{"$self"} = $self; } @@ -33,7 +74,7 @@ sub _set_uint ($$$) { my $val = $opts->{$field}; if (defined $val) { $val = $val->[-1] if ref($val) eq 'ARRAY'; - $val = undef if $val !~ /\A\d+\z/; + $val = undef if $val !~ /\A[0-9]+\z/; } $opts->{$field} = $val || $default; } @@ -46,7 +87,7 @@ sub _set_limiter ($$$) { my $mkey = $pfx.'max'; my $val = $self->{$mkey} or return; my $lim; - if ($val =~ /\A\d+\z/) { + if ($val =~ /\A[0-9]+\z/) { require PublicInbox::Qspawn; $lim = PublicInbox::Qspawn::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { @@ -68,28 +109,101 @@ sub new { _set_limiter($opts, $pi_config, 'httpbackend'); _set_uint($opts, 'feedmax', 25); $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; + my $dir = $opts->{inboxdir}; + if (defined $dir && -f "$dir/inbox.lock") { + $opts->{version} = 2; + } + + # allow any combination of multi-line or comma-delimited hide entries + my $hide = {}; + if (defined(my $h = $opts->{hide})) { + foreach my $v (@$h) { + $hide->{$_} = 1 foreach (split(/\s*,\s*/, $v)); + } + $opts->{-hide} = $hide; + } bless $opts, $class; } +sub git_epoch { + my ($self, $epoch) = @_; + ($self->{version} || 1) == 2 or return; + $self->{"$epoch.git"} ||= eval { + my $git_dir = "$self->{inboxdir}/git/$epoch.git"; + my $g = PublicInbox::Git->new($git_dir); + $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + # no cleanup needed, we never cat-file off this, only clone + $g; + }; +} + sub git { my ($self) = @_; $self->{git} ||= eval { - my $g = PublicInbox::Git->new($self->{mainrepo}); + my $git_dir = $self->{inboxdir}; + $git_dir .= '/all.git' if (($self->{version} || 1) == 2); + my $g = PublicInbox::Git->new($git_dir); $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + _cleanup_later($self); $g; }; } -sub mm { +sub max_git_epoch { my ($self) = @_; - $self->{mm} ||= eval { PublicInbox::Msgmap->new($self->{mainrepo}) }; + my $v = $self->{version}; + return unless defined($v) && $v == 2; + my $cur = $self->{-max_git_epoch}; + my $changed = git($self)->alternates_changed; + if (!defined($cur) || $changed) { + $self->git->cleanup if $changed; + my $gits = "$self->{inboxdir}/git"; + if (opendir my $dh, $gits) { + my $max = -1; + while (defined(my $git_dir = readdir($dh))) { + $git_dir =~ m!\A([0-9]+)\.git\z! or next; + $max = $1 if $1 > $max; + } + $cur = $self->{-max_git_epoch} = $max if $max >= 0; + } else { + warn "opendir $gits failed: $!\n"; + } + } + $cur; } -sub search { +sub mm { my ($self) = @_; - $self->{search} ||= eval { - PublicInbox::Search->new($self->{mainrepo}, $self->{altid}); + $self->{mm} ||= eval { + require PublicInbox::Msgmap; + _cleanup_later($self); + my $dir = $self->{inboxdir}; + if (($self->{version} || 1) >= 2) { + PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); + } else { + PublicInbox::Msgmap->new($dir); + } + }; +} + +sub search ($;$) { + my ($self, $over_only) = @_; + my $srch = $self->{search} ||= eval { + _cleanup_later($self); + require PublicInbox::Search; + PublicInbox::Search->new($self); }; + ($over_only || eval { $srch->xdb }) ? $srch : undef; +} + +sub over ($) { + my ($self) = @_; + my $srch = search($self, 1) or return; + $self->{over} ||= eval { + my $over = $srch->{over_ro}; + $over->dbh_new; # may fail + $over; + } } sub try_cat { @@ -106,11 +220,11 @@ sub description { my ($self) = @_; my $desc = $self->{description}; return $desc if defined $desc; - $desc = try_cat("$self->{mainrepo}/description"); + $desc = try_cat("$self->{inboxdir}/description"); local $/ = "\n"; chomp $desc; $desc =~ s/\s+/ /smg; - $desc = '($GIT_DIR/description missing)' if $desc eq ''; + $desc = '($INBOX_DIR/description missing)' if $desc eq ''; $self->{description} = $desc; } @@ -118,7 +232,7 @@ sub cloneurl { my ($self) = @_; my $url = $self->{cloneurl}; return $url if $url; - $url = try_cat("$self->{mainrepo}/cloneurl"); + $url = try_cat("$self->{inboxdir}/cloneurl"); my @url = split(/\s+/s, $url); local $/ = "\n"; chomp @url; @@ -126,26 +240,22 @@ sub cloneurl { } sub base_url { - my ($self, $env) = @_; - if ($env) { # PSGI env - my $scheme = $env->{'psgi.url_scheme'}; - my $host_port = $env->{HTTP_HOST} || - "$env->{SERVER_NAME}:$env->{SERVER_PORT}"; - my $url = "$scheme://$host_port". ($env->{SCRIPT_NAME} || '/'); + my ($self, $env) = @_; # env - PSGI env + if ($env) { + my $url = PublicInbox::Git::host_prefix_url($env, ''); # for mount in Plack::Builder $url .= '/' if $url !~ m!/\z!; - $url .= $self->{name} . '/'; - } else { - # either called from a non-PSGI environment (e.g. NNTP/POP3) - $self->{-base_url} ||= do { - my $url = $self->{url} or return undef; - # expand protocol-relative URLs to HTTPS if we're - # not inside a web server - $url = "https:$url" if $url =~ m!\A//!; - $url .= '/' if $url !~ m!/\z!; - $url; - }; + return $url .= $self->{name} . '/'; } + # called from a non-PSGI environment (e.g. NNTP/POP3): + $self->{-base_url} ||= do { + my $url = $self->{url}->[0] or return undef; + # expand protocol-relative URLs to HTTPS if we're + # not inside a web server + $url = "https:$url" if $url =~ m!\A//!; + $url .= '/' if $url !~ m!/\z!; + $url; + }; } sub nntp_url { @@ -183,12 +293,11 @@ sub nntp_url { # nntp://news.example.com/alt.example push @m, $u; } - my %seen = map { $_ => 1 } @urls; - foreach my $u (@m) { - next if $seen{$u}; - $seen{$u} = 1; - push @urls, $u; - } + + # List::Util::uniq requires Perl 5.26+, maybe we + # can use it by 2030 or so + my %seen; + @urls = grep { !$seen{$_}++ } (@urls, @m); } \@urls; }; @@ -196,8 +305,8 @@ sub nntp_url { sub nntp_usable { my ($self) = @_; - my $ret = $self->mm && $self->search; - $self->{mm} = $self->{search} = undef; + my $ret = mm($self) && over($self); + $self->{mm} = $self->{over} = $self->{search} = undef; $ret; } @@ -212,26 +321,64 @@ sub msg_by_path ($$;$) { sub msg_by_smsg ($$;$) { my ($self, $smsg, $ref) = @_; - return unless defined $smsg; # ghost - - # backwards compat to fallback to msg_by_mid - # TODO: remove if we bump SCHEMA_VERSION in Search.pm: - defined(my $blob = $smsg->{blob}) or - return msg_by_mid($self, $smsg->mid); + # ghosts may have undef smsg (from SearchThread.node) or + # no {blob} field + return unless defined $smsg; + defined(my $blob = $smsg->{blob}) or return; my $str = git($self)->cat_file($blob, $ref); $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; $str; } -sub path_check { - my ($self, $path) = @_; - git($self)->check('HEAD:'.$path); +sub smsg_mime { + my ($self, $smsg, $ref) = @_; + if (my $s = msg_by_smsg($self, $smsg, $ref)) { + $smsg->{mime} = PublicInbox::MIME->new($s); + return $smsg; + } +} + +sub mid2num($$) { + my ($self, $mid) = @_; + my $mm = mm($self) or return; + $mm->num_for($mid); +} + +sub smsg_by_mid ($$) { + my ($self, $mid) = @_; + my $over = over($self) or return; + # favor the Message-ID we used for the NNTP article number: + defined(my $num = mid2num($self, $mid)) or return; + my $smsg = $over->get_art($num) or return; + PublicInbox::SearchMsg::psgi_cull($smsg); } sub msg_by_mid ($$;$) { my ($self, $mid, $ref) = @_; - msg_by_path($self, mid2path($mid), $ref); + + over($self) or + return msg_by_path($self, mid2path($mid), $ref); + + my $smsg = smsg_by_mid($self, $mid); + $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; +} + +sub recent { + my ($self, $opts, $after, $before) = @_; + over($self)->recent($opts, $after, $before); +} + +sub modified { + my ($self) = @_; + if (my $over = over($self)) { + my $msgs = $over->recent({limit => 1}); + if (my $smsg = $msgs->[0]) { + return $smsg->{ts}; + } + return time; + } + git($self)->modified; # v1 } 1;