Documentation/technical/ds.txt | 2 +- lib/PublicInbox/GitAsyncCat.pm | 21 +++++++++++++-------- lib/PublicInbox/GzipFilter.pm | 3 +-- lib/PublicInbox/HTTPD.pm | 2 +- lib/PublicInbox/IMAP.pm | 10 +++++----- lib/PublicInbox/NNTP.pm | 4 ++-- lib/PublicInbox/SolverGit.pm | 3 +-- diff --git a/Documentation/technical/ds.txt b/Documentation/technical/ds.txt index a0793ca2337a04e956aaaf0bb5653c0561760a60..7bc1ad79ce0c89f1e87e70f4ef50a0afc82a165d 100644 --- a/Documentation/technical/ds.txt +++ b/Documentation/technical/ds.txt @@ -64,7 +64,7 @@ * ->requeue support. An optimization of the AddTimer(0, ...) idiom for immediately dispatching code at the next event loop iteration. public-inbox uses this for fairly generating large responses - iteratively (see PublicInbox::NNTP::long_response or git_async_cat + iteratively (see PublicInbox::NNTP::long_response or ibx_async_cat for blob retrievals). New features diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm index 7d1a13dbb48bb11fa1b68f97a4f32a1a65ccd6e0..57c194d9e45fb1cc64f178f843bf97d2a09a079e 100644 --- a/lib/PublicInbox/GitAsyncCat.pm +++ b/lib/PublicInbox/GitAsyncCat.pm @@ -8,7 +8,7 @@ use strict; use parent qw(PublicInbox::DS Exporter); use POSIX qw(WNOHANG); use PublicInbox::Syscall qw(EPOLLIN EPOLLET); -our @EXPORT = qw(git_async_cat git_async_prefetch); +our @EXPORT = qw(ibx_async_cat ibx_async_prefetch); use PublicInbox::Git (); our $GCF2C; # singleton PublicInbox::Gcf2Client @@ -45,12 +45,16 @@ $self->close; } } -sub git_async_cat ($$$$) { - my ($git, $oid, $cb, $arg) = @_; - if ($GCF2C //= eval { +sub ibx_async_cat ($$$$) { + my ($ibx, $oid, $cb, $arg) = @_; + my $git = $ibx->git; + # {topdir} means ExtSearch (likely [extindex "all"]) with potentially + # 100K alternates. git(1) has a proposed patch for 100K alternates: + # + if (!defined($ibx->{topdir}) && ($GCF2C //= eval { require PublicInbox::Gcf2Client; PublicInbox::Gcf2Client::new(); - } // 0) { # 0: do not retry if libgit2 or Inline::C are missing + } // 0)) { # 0: do not retry if libgit2 or Inline::C are missing $GCF2C->gcf2_async(\"$oid $git->{git_dir}\n", $cb, $arg); \undef; } else { # read-only end of git-cat-file pipe @@ -66,9 +70,10 @@ } # this is safe to call inside $cb, but not guaranteed to enqueue # returns true if successful, undef if not. -sub git_async_prefetch { - my ($git, $oid, $cb, $arg) = @_; - if ($GCF2C) { +sub ibx_async_prefetch { + my ($ibx, $oid, $cb, $arg) = @_; + my $git = $ibx->git; + if (!defined($ibx->{topdir}) && $GCF2C) { if (!$GCF2C->{wbuf}) { $oid .= " $git->{git_dir}\n"; return $GCF2C->gcf2_async(\$oid, $cb, $arg); # true diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm index 48ed11a5d55cb782c9161a5589e798d63d3a8498..334d658109be102465c79e690c3afed164b54fc0 100644 --- a/lib/PublicInbox/GzipFilter.pm +++ b/lib/PublicInbox/GzipFilter.pm @@ -180,8 +180,7 @@ } sub smsg_blob { my ($self, $smsg) = @_; - git_async_cat($self->{ibx}->git, $smsg->{blob}, - \&async_blob_cb, $self); + ibx_async_cat($self->{ibx}, $smsg->{blob}, \&async_blob_cb, $self); } 1; diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm index b193c9aeeff190abae0c5bee4b363855b61a77fc..fb683f746a9c988875dd2ee0e141df8d23021d1c 100644 --- a/lib/PublicInbox/HTTPD.pm +++ b/lib/PublicInbox/HTTPD.pm @@ -37,7 +37,7 @@ # XXX unstable API!, only GitHTTPBackend needs # this to limit git-http-backend(1) parallelism. # We also check for the truthiness of this to - # detect when to use git_async_cat for slow blobs + # detect when to use async paths for slow blobs 'pi-httpd.async' => \&pi_httpd_async ); bless { diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index af8ce72be48a61df36bec17183fa061a0f2bffae..9402aa41fe4fe94ddbb722ca534c4df4ee56c90f 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -612,7 +612,7 @@ partial_emit($self, $partial, $eml) if $partial; $self->msg_more(")\r\n"); } -sub fetch_blob_cb { # called by git->cat_async via git_async_cat +sub fetch_blob_cb { # called by git->cat_async via ibx_async_cat my ($bref, $oid, $type, $size, $fetch_arg) = @_; my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg; my $ibx = $self->{ibx} or return $self->close; # client disconnected @@ -627,8 +627,8 @@ $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; } my $pre; if (!$self->{wbuf} && (my $nxt = $msgs->[0])) { - $pre = git_async_prefetch($ibx->git, $nxt->{blob}, - \&fetch_blob_cb, $fetch_arg); + $pre = ibx_async_prefetch($ibx, $nxt->{blob}, + \&fetch_blob_cb, $fetch_arg); } fetch_run_ops($self, $smsg, $bref, $ops, $partial); $pre ? $self->zflush : requeue_once($self); @@ -760,7 +760,7 @@ return; } } uo2m_extend($self, $msgs->[-1]->{num}); - git_async_cat($self->{ibx}->git, $msgs->[0]->{blob}, + ibx_async_cat($self->{ibx}, $msgs->[0]->{blob}, \&fetch_blob_cb, \@_); } @@ -1228,7 +1228,7 @@ $self->close; } elsif ($more) { # $self->{wbuf}: $self->update_idle_time; - # control passed to git_async_cat if $more == \undef + # control passed to ibx_async_cat if $more == \undef requeue_once($self) if !ref($more); } else { # all done! delete $self->{long_cb}; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index f7d99913d350e314dcedc9c79b3989d46f9ff29f..9df47133f0c99973544ba37957f31e5ce665e160 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -515,7 +515,7 @@ $smsg->{nntp} = $self; $smsg->{nntp_code} = $code; set_art($self, $art); # this dereferences to `undef' - ${git_async_cat($ibx->git, $smsg->{blob}, \&blob_cb, $smsg)}; + ${ibx_async_cat($ibx, $smsg->{blob}, \&blob_cb, $smsg)}; } } @@ -549,7 +549,7 @@ $$hdr =~ s/^(Message-ID:)[ \t]*\r\n[ \t]+([^\r]+)\r\n/$1 $2\r\n/igsm; $smsg->{nntp}->msg_more($$hdr); } -sub blob_cb { # called by git->cat_async via git_async_cat +sub blob_cb { # called by git->cat_async via ibx_async_cat my ($bref, $oid, $type, $size, $smsg) = @_; my $self = $smsg->{nntp}; my $code = $smsg->{nntp_code}; diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm index 92106e75ee97d3b1f9e543d57cc10b21282b19af..b0cd0f2c20be67c596b130d923941918a1adcf0d 100644 --- a/lib/PublicInbox/SolverGit.pm +++ b/lib/PublicInbox/SolverGit.pm @@ -593,8 +593,7 @@ if (my $msgs = $want->{try_smsgs}) { my $smsg = shift @$msgs; if ($self->{psgi_env}->{'pi-httpd.async'}) { - return git_async_cat($want->{cur_ibx}->git, - $smsg->{blob}, + return ibx_async_cat($want->{cur_ibx}, $smsg->{blob}, \&extract_diff_async, [$self, $want, $smsg]); } else {