X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=286555f65cb3b4c8f4a7820b0489e4f6f6005e72;hb=7a3946ef122e8218c6ce3355d7f968562212d53b;hp=b9cd4c40a6dc7541f799f0da8b520e426af7df4d;hpb=c062644c64bf44b48bd6469e1d4e77d8b9cfdc87;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index b9cd4c40..286555f6 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -8,6 +8,8 @@ use warnings; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); use Devel::Peek qw(SvREFCNT); +use PublicInbox::MIME; +use POSIX qw(strftime); my $cleanup_timer; eval { @@ -15,21 +17,36 @@ eval { require PublicInbox::EvCleanup; $cleanup_timer = undef; # OK if we get here }; +my $cleanup_broken = $@; my $CLEANUP = {}; # string(inbox) -> inbox sub cleanup_task () { $cleanup_timer = undef; + my $next = {}; for my $ibx (values %$CLEANUP) { - foreach my $f (qw(git mm search)) { + my $again; + foreach my $f (qw(mm search)) { delete $ibx->{$f} if SvREFCNT($ibx->{$f}) == 1; } + my $expire = time - 60; + if (my $git = $ibx->{git}) { + $again = $git->cleanup($expire); + } + if (my $gits = $ibx->{-repo_objs}) { + foreach my $git (@$gits) { + $again = 1 if $git->cleanup($expire); + } + } + $again ||= !!($ibx->{mm} || $ibx->{search}); + $next->{"$ibx"} = $ibx if $again; } - $CLEANUP = {}; + $CLEANUP = $next; } sub _cleanup_later ($) { my ($self) = @_; - return unless $PublicInbox::EvCleanup::ENABLED; + return if $cleanup_broken; + return unless PublicInbox::EvCleanup::enabled(); $cleanup_timer ||= PublicInbox::EvCleanup::later(*cleanup_task); $CLEANUP->{"$self"} = $self; } @@ -75,12 +92,33 @@ sub new { _set_uint($opts, 'feedmax', 25); $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; my $dir = $opts->{mainrepo}; - if (defined $dir && -f "$dir/msgmap.sqlite3") { # XXX DIRTY + if (defined $dir && -f "$dir/inbox.lock") { $opts->{version} = 2; } + + # allow any combination of multi-line or comma-delimited hide entries + my $hide = {}; + if (defined(my $h = $opts->{hide})) { + foreach my $v (@$h) { + $hide->{$_} = 1 foreach (split(/\s*,\s*/, $v)); + } + $opts->{-hide} = $hide; + } bless $opts, $class; } +sub git_part { + my ($self, $part) = @_; + ($self->{version} || 1) == 2 or return; + $self->{"$part.git"} ||= eval { + my $git_dir = "$self->{mainrepo}/git/$part.git"; + my $g = PublicInbox::Git->new($git_dir); + $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + # no cleanup needed, we never cat-file off this, only clone + $g; + }; +} + sub git { my ($self) = @_; $self->{git} ||= eval { @@ -93,9 +131,33 @@ sub git { }; } +sub max_git_part { + my ($self) = @_; + my $v = $self->{version}; + return unless defined($v) && $v == 2; + my $part = $self->{-max_git_part}; + my $changed = git($self)->alternates_changed; + if (!defined($part) || $changed) { + $self->git->cleanup if $changed; + my $gits = "$self->{mainrepo}/git"; + if (opendir my $dh, $gits) { + my $max = -1; + while (defined(my $git_dir = readdir($dh))) { + $git_dir =~ m!\A(\d+)\.git\z! or next; + $max = $1 if $1 > $max; + } + $part = $self->{-max_git_part} = $max if $max >= 0; + } else { + warn "opendir $gits failed: $!\n"; + } + } + $part; +} + sub mm { my ($self) = @_; $self->{mm} ||= eval { + require PublicInbox::Msgmap; _cleanup_later($self); my $dir = $self->{mainrepo}; if (($self->{version} || 1) >= 2) { @@ -132,7 +194,7 @@ sub description { local $/ = "\n"; chomp $desc; $desc =~ s/\s+/ /smg; - $desc = '($GIT_DIR/description missing)' if $desc eq ''; + $desc = '($REPO_DIR/description missing)' if $desc eq ''; $self->{description} = $desc; } @@ -149,8 +211,8 @@ sub cloneurl { sub base_url { my ($self, $env) = @_; - if ($env) { # PSGI env - my $scheme = $env->{'psgi.url_scheme'}; + my $scheme; + if ($env && ($scheme = $env->{'psgi.url_scheme'})) { # PSGI env my $host_port = $env->{HTTP_HOST} || "$env->{SERVER_NAME}:$env->{SERVER_PORT}"; my $url = "$scheme://$host_port". ($env->{SCRIPT_NAME} || '/'); @@ -234,32 +296,62 @@ sub msg_by_path ($$;$) { sub msg_by_smsg ($$;$) { my ($self, $smsg, $ref) = @_; - return unless defined $smsg; # ghost - - # backwards compat to fallback to msg_by_mid - # TODO: remove if we bump SCHEMA_VERSION in Search.pm: - defined(my $blob = $smsg->{blob}) or - return msg_by_path($self, mid2path($smsg->mid), $ref); + # ghosts may have undef smsg (from SearchThread.node) or + # no {blob} field + return unless defined $smsg; + defined(my $blob = $smsg->{blob}) or return; my $str = git($self)->cat_file($blob, $ref); $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; $str; } -sub path_check { - my ($self, $path) = @_; - git($self)->check('HEAD:'.$path); +sub smsg_mime { + my ($self, $smsg, $ref) = @_; + if (my $s = msg_by_smsg($self, $smsg, $ref)) { + $smsg->{mime} = PublicInbox::MIME->new($s); + return $smsg; + } +} + +sub mid2num($$) { + my ($self, $mid) = @_; + my $mm = mm($self) or return; + $mm->num_for($mid); +} + +sub smsg_by_mid ($$) { + my ($self, $mid) = @_; + my $srch = search($self) or return; + # favor the Message-ID we used for the NNTP article number: + defined(my $num = mid2num($self, $mid)) or return; + my $smsg = $srch->lookup_article($num) or return; + PublicInbox::SearchMsg::psgi_cull($smsg); } sub msg_by_mid ($$;$) { my ($self, $mid, $ref) = @_; my $srch = search($self) or - return msg_by_path($self, mid2path($mid), $ref); - my $smsg; - $srch->retry_reopen(sub { - $smsg = $srch->lookup_skeleton($mid) and $smsg->load_expand; - }); + return msg_by_path($self, mid2path($mid), $ref); + my $smsg = smsg_by_mid($self, $mid); $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; } +sub recent { + my ($self, $opts, $after, $before) = @_; + search($self)->{over_ro}->recent($opts, $after, $before); +} + +sub modified { + my ($self) = @_; + if (my $srch = search($self)) { + my $msgs = $srch->{over_ro}->recent({limit => 1}); + if (my $smsg = $msgs->[0]) { + return $smsg->{ts}; + } + return time; + } + git($self)->modified; # v1 +} + 1;