X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FInbox.pm;h=90ac9ebb7c31628a292ec69647399969b1adf2fe;hb=dfef0d2c34fbf21d665c40b5ad5069e9113c35c8;hp=e552cd4f2ff38696bee4d2370f5b33aa10011561;hpb=95acd5901491e4f333f5d2bbeed6fb5e6b53e07c;p=public-inbox.git diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index e552cd4f..90ac9ebb 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,54 +1,55 @@ -# Copyright (C) 2016 all contributors +# Copyright (C) 2016-2018 all contributors # License: AGPL-3.0+ # # Represents a public-inbox (which may have multiple mailing addresses) package PublicInbox::Inbox; use strict; use warnings; -use Scalar::Util qw(weaken isweak); use PublicInbox::Git; use PublicInbox::MID qw(mid2path); +use Devel::Peek qw(SvREFCNT); +use PublicInbox::MIME; -my $weakt; +my $cleanup_timer; eval { - $weakt = 'disabled'; + $cleanup_timer = 'disabled'; require PublicInbox::EvCleanup; - $weakt = undef; # OK if we get here + $cleanup_timer = undef; # OK if we get here }; -my $WEAKEN = {}; # string(inbox) -> inbox -sub weaken_task () { - $weakt = undef; - _weaken_fields($_) for values %$WEAKEN; - $WEAKEN = {}; +my $CLEANUP = {}; # string(inbox) -> inbox +sub cleanup_task () { + $cleanup_timer = undef; + for my $ibx (values %$CLEANUP) { + foreach my $f (qw(git mm search)) { + delete $ibx->{$f} if SvREFCNT($ibx->{$f}) == 1; + } + } + $CLEANUP = {}; } -sub _weaken_later ($) { +sub _cleanup_later ($) { my ($self) = @_; - $weakt ||= PublicInbox::EvCleanup::later(*weaken_task); - $WEAKEN->{"$self"} = $self; -} - -sub new { - my ($class, $opts) = @_; - my $v = $opts->{address} ||= 'public-inbox@example.com'; - my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; - $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; - weaken($opts->{-pi_config}); - bless $opts, $class; + return unless PublicInbox::EvCleanup::enabled(); + $cleanup_timer ||= PublicInbox::EvCleanup::later(*cleanup_task); + $CLEANUP->{"$self"} = $self; } -sub _weaken_fields { - my ($self) = @_; - foreach my $f (qw(git mm search)) { - isweak($self->{$f}) or weaken($self->{$f}); +sub _set_uint ($$$) { + my ($opts, $field, $default) = @_; + my $val = $opts->{$field}; + if (defined $val) { + $val = $val->[-1] if ref($val) eq 'ARRAY'; + $val = undef if $val !~ /\A\d+\z/; } + $opts->{$field} = $val || $default; } sub _set_limiter ($$$) { - my ($self, $git, $pfx) = @_; + my ($self, $pi_config, $pfx) = @_; my $lkey = "-${pfx}_limiter"; - $git->{$lkey} = $self->{$lkey} ||= eval { + $self->{$lkey} ||= eval { + # full key is: publicinbox.$NAME.httpbackendmax my $mkey = $pfx.'max'; my $val = $self->{$mkey} or return; my $lim; @@ -56,7 +57,7 @@ sub _set_limiter ($$$) { require PublicInbox::Qspawn; $lim = PublicInbox::Qspawn::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { - $lim = $self->{-pi_config}->limiter($val); + $lim = $pi_config->limiter($val); warn "$mkey limiter=$val not found\n" if !$lim; } else { warn "$mkey limiter=$val not understood\n"; @@ -65,29 +66,88 @@ sub _set_limiter ($$$) { } } +sub new { + my ($class, $opts) = @_; + my $v = $opts->{address} ||= 'public-inbox@example.com'; + my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; + $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; + my $pi_config = delete $opts->{-pi_config}; + _set_limiter($opts, $pi_config, 'httpbackend'); + _set_uint($opts, 'feedmax', 25); + $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; + my $dir = $opts->{mainrepo}; + if (defined $dir && -f "$dir/inbox.lock") { + $opts->{version} = 2; + } + bless $opts, $class; +} + +sub git_part { + my ($self, $part) = @_; + ($self->{version} || 1) == 2 or return; + $self->{"$part.git"} ||= eval { + my $git_dir = "$self->{mainrepo}/git/$part.git"; + my $g = PublicInbox::Git->new($git_dir); + $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + # no cleanup needed, we never cat-file off this, only clone + $g; + }; +} + sub git { my ($self) = @_; $self->{git} ||= eval { - _weaken_later($self); - my $g = PublicInbox::Git->new($self->{mainrepo}); - _set_limiter($self, $g, 'httpbackend'); + my $git_dir = $self->{mainrepo}; + $git_dir .= '/all.git' if (($self->{version} || 1) == 2); + my $g = PublicInbox::Git->new($git_dir); + $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter}; + _cleanup_later($self); $g; }; } +sub max_git_part { + my ($self) = @_; + my $v = $self->{version}; + return unless defined($v) && $v == 2; + my $part = $self->{-max_git_part}; + my $changed = git($self)->alternates_changed; + if (!defined($part) || $changed) { + $self->git->cleanup if $changed; + my $gits = "$self->{mainrepo}/git"; + if (opendir my $dh, $gits) { + my $max = -1; + while (defined(my $git_dir = readdir($dh))) { + $git_dir =~ m!\A(\d+)\.git\z! or next; + $max = $1 if $1 > $max; + } + $part = $self->{-max_git_part} = $max if $max >= 0; + } else { + warn "opendir $gits failed: $!\n"; + } + } + $part; +} + sub mm { my ($self) = @_; $self->{mm} ||= eval { - _weaken_later($self); - PublicInbox::Msgmap->new($self->{mainrepo}); + require PublicInbox::Msgmap; + _cleanup_later($self); + my $dir = $self->{mainrepo}; + if (($self->{version} || 1) >= 2) { + PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); + } else { + PublicInbox::Msgmap->new($dir); + } }; } sub search { my ($self) = @_; $self->{search} ||= eval { - _weaken_later($self); - PublicInbox::Search->new($self->{mainrepo}); + _cleanup_later($self); + PublicInbox::Search->new($self, $self->{altid}); }; } @@ -109,7 +169,7 @@ sub description { local $/ = "\n"; chomp $desc; $desc =~ s/\s+/ /smg; - $desc = '($GIT_DIR/description missing)' if $desc eq ''; + $desc = '($REPO_DIR/description missing)' if $desc eq ''; $self->{description} = $desc; } @@ -147,6 +207,52 @@ sub base_url { } } +sub nntp_url { + my ($self) = @_; + $self->{-nntp_url} ||= do { + # no checking for nntp_usable here, we can point entirely + # to non-local servers or users run by a different user + my $ns = $self->{nntpserver}; + my $group = $self->{newsgroup}; + my @urls; + if ($ns && $group) { + $ns = [ $ns ] if ref($ns) ne 'ARRAY'; + @urls = map { + my $u = m!\Anntps?://! ? $_ : "nntp://$_"; + $u .= '/' if $u !~ m!/\z!; + $u.$group; + } @$ns; + } + + my $mirrors = $self->{nntpmirror}; + if ($mirrors) { + my @m; + foreach (@$mirrors) { + my $u = m!\Anntps?://! ? $_ : "nntp://$_"; + if ($u =~ m!\Anntps?://[^/]+/?\z!) { + if ($group) { + $u .= '/' if $u !~ m!/\z!; + $u .= $group; + } else { + warn +"publicinbox.$self->{name}.nntpmirror=$_ missing newsgroup name\n"; + } + } + # else: allow full URLs like: + # nntp://news.example.com/alt.example + push @m, $u; + } + my %seen = map { $_ => 1 } @urls; + foreach my $u (@m) { + next if $seen{$u}; + $seen{$u} = 1; + push @urls, $u; + } + } + \@urls; + }; +} + sub nntp_usable { my ($self) = @_; my $ret = $self->mm && $self->search; @@ -165,23 +271,49 @@ sub msg_by_path ($$;$) { sub msg_by_smsg ($$;$) { my ($self, $smsg, $ref) = @_; - # backwards compat to fallback to msg_by_mid - # TODO: remove if we bump SCHEMA_VERSION in Search.pm: - defined(my $blob = $smsg->blob) or return msg_by_mid($self, $smsg->mid); + # ghosts may have undef smsg (from SearchThread.node) or + # no {blob} field (from each_smsg_by_mid) + return unless defined $smsg; + defined(my $blob = $smsg->{blob}) or return; my $str = git($self)->cat_file($blob, $ref); $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; $str; } +sub smsg_mime { + my ($self, $smsg, $ref) = @_; + if (my $s = msg_by_smsg($self, $smsg, $ref)) { + $smsg->{mime} = PublicInbox::MIME->new($s); + return $smsg; + } +} + sub path_check { my ($self, $path) = @_; git($self)->check('HEAD:'.$path); } +sub mid2num($$) { + my ($self, $mid) = @_; + my $mm = mm($self) or return; + $mm->num_for($mid); +} + +sub smsg_by_mid ($$) { + my ($self, $mid) = @_; + my $srch = search($self) or return; + # favor the Message-ID we used for the NNTP article number: + my $num = mid2num($self, $mid); + defined $num ? $srch->lookup_article($num) : undef; +} + sub msg_by_mid ($$;$) { my ($self, $mid, $ref) = @_; - msg_by_path($self, mid2path($mid), $ref); + my $srch = search($self) or + return msg_by_path($self, mid2path($mid), $ref); + my $smsg = smsg_by_mid($self, $mid); + $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; } 1;