search: use one git-rev-parse process for all dates

This is necessary to avoid slowdowns with pathological cases
with many dates in the query, since each rev-parse invocation
takes ~5ms.

This is immeasurably slower with one open-ended range, but
already faster with any closed range featuring two dates which
require parsing via git.

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 9207962bb7df6384eb2f611915dfcddfa555b4a6..ac7ff267a1033cb8bbc8234f8ad0b381ff23c94f 100644 (file)
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -377,8 +377,10 @@ sub qx {
 }
 
 sub date_parse {
-       my $d = $_[0]->qx('rev-parse', "--since=$_[1]");
-       substr($d, length('--max-age='), -1)
+       my $self = shift;
+       map {
+               substr($_, length('--max-age='), -1)
+       } $self->qx('rev-parse', map { "--since=$_" } @_);
 }
 
 # check_async and cat_async may trigger the other, so ensure they're

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f42d70e3c6c8fda84bd3be15e5d4c2a30b5bbbde..aa737d63b5f64b7d30df764b6b34ceaf2e3044b0 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -265,8 +265,10 @@ sub reopen {
 # and neither the SWIG nor XS bindings allow us to use custom code
 # to parse dates (and libgit2 doesn't expose git__date_parse, either,
 # so we're running git-rev-parse(1)).
-sub date_range {
-       my ($git, $pfx, $range) = @_;
+# This replaces things we need to send to $git->date_parse with
+# "\0".$strftime_format.['+'|$idx]."\0" placeholders
+sub date_parse_prepare {
+       my ($to_parse, $pfx, $range) = @_;
        # are we inside a parenthesized statement?
        my $end = $range =~ s/([\)\s]*)\z// ? $1 : '';
        my @r = split(/\.\./, $range, 2);
@@ -275,55 +277,72 @@ sub date_range {
        # n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD
        if ($pfx eq 'd') {
                if (!defined($r[1])) {
-                       $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/;
-                       $r[0] = $git->date_parse($r[0]);
-                       $r[1] = $r[0] + 86400;
-                       for my $x (@r) {
-                               $x = strftime('%Y%m%d', gmtime($x));
+                       if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})\z/) {
+                               push @$to_parse, "$1-$2-$3";
+                               # we could've handled as-is, but we need
+                               # to parse anyways for "d+" below
+                       } else {
+                               push @$to_parse, $r[0];
                        }
+                       $r[0] = "\0%Y%m%d$#$to_parse\0";
+                       $r[1] = "\0%Y%m%d+\0";
                } else {
                        for my $x (@r) {
                                next if $x eq '' || $x =~ /\A[0-9]{8}\z/;
-                               $x = strftime('%Y%m%d',
-                                               gmtime($git->date_parse($x)));
+                               push @$to_parse, $x;
+                               $x = "\0%Y%m%d$#$to_parse\0";
                        }
                }
        } elsif ($pfx eq 'dt') {
                if (!defined($r[1])) { # git needs gaps and not /\d{14}/
-                       $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})
-                                       ([0-9]{2})([0-9]{2})([0-9]{2})\z
-                               /$1-$2-$3 $4:$5:$6/x;
-                       $r[0] = $git->date_parse($r[0]);
-                       $r[1] = $r[0] + 86400;
-                       for my $x (@r) {
-                               $x = strftime('%Y%m%d%H%M%S', gmtime($x));
+                       if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})
+                                       ([0-9]{2})([0-9]{2})([0-9]{2})\z/x) {
+                               push @$to_parse, "$1-$2-$3 $4:$5:$6";
+                       } else {
+                               push @$to_parse, $r[0];
                        }
+                       $r[0] = "\0%Y%m%d%H%M%S$#$to_parse\0";
+                       $r[1] = "\0%Y%m%d%H%M%S+\0";
                } else {
                        for my $x (@r) {
                                next if $x eq '' || $x =~ /\A[0-9]{14}\z/;
-                               $x = strftime('%Y%m%d%H%M%S',
-                                               gmtime($git->date_parse($x)));
+                               push @$to_parse, $x;
+                               $x = "\0%Y%m%d%H%M%S$#$to_parse\0";
                        }
                }
        } else { # "rt", let git interpret "YYYY", deal with Y10K later :P
                for my $x (@r) {
                        next if $x eq '' || $x =~ /\A[0-9]{5,}\z/;
-                       $x = $git->date_parse($x);
+                       push @$to_parse, $x;
+                       $x = "\0%s$#$to_parse\0";
                }
-               $r[1] //= $r[0] + 86400;
+               $r[1] //= "\0%s+\0";
        }
        "$pfx:".join('..', @r).$end;
 }
 
+# n.b. argv never has NUL, though we'll need to filter it out
+# if this $argv isn't from a command execution
 sub query_argv_to_string {
        my (undef, $git, $argv) = @_;
-       join(' ', map {;
-               if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) {
+       my $to_parse;
+       my $tmp = join(' ', map {;
+               if (s!\b(d|rt|dt):([[:print:]]+)\z!date_parse_prepare(
+                                               $to_parse //= [], $1, $2)!sge) {
                        $_;
                } else {
                        /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_
                }
        } @$argv);
+       # git-rev-parse can handle any number of args up to system
+       # limits (around (4096*32) bytes on Linux).
+       if ($to_parse) {
+               my @r = $git->date_parse(@$to_parse);
+               my $i;
+               $tmp =~ s/\0(%[%YmdHMSs]+)([0-9\+]+)\0/strftime($1,
+                       gmtime($2 eq '+' ? ($r[$i]+86400) : $r[$i=$2+0]))/sge;
+       }
+       $tmp
 }
 
 # read-only

diff --git a/t/git.t b/t/git.t
index 7b950d88f4cc5c83f26393183b8918ce452ea34d..4a45bbaf830d0342b17ad9b53a621d2de279ea39 100644 (file)
--- a/t/git.t
+++ b/t/git.t
@@ -20,14 +20,16 @@ use_ok 'PublicInbox::Git';
 }
 {
        my $git = PublicInbox::Git->new($dir);
-       my $s = $git->date_parse('1970-01-01T00:00:00Z');
-       is($s, 0, 'parsed epoch');
+       my @s = $git->date_parse('1970-01-01T00:00:00Z');
+       is($s[0], 0, 'parsed epoch');
        local $ENV{TZ} = 'UTC';
-       $s = $git->date_parse('1993-10-02 01:02:09');
-       is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s)), '1993-10-02T01:02:09Z',
-               'round trips');
-       $s = $git->date_parse('1993-10-02');
-       is(strftime('%Y-%m-%d', gmtime($s)), '1993-10-02',
+       @s = $git->date_parse('1993-10-02 01:02:09', '2010-10-02 01:03:04');
+       is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[0])),
+               '1993-10-02T01:02:09Z', 'round trips');
+       is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[1])),
+               '2010-10-02T01:03:04Z', '2nd arg round trips');
+       @s = $git->date_parse('1993-10-02');
+       is(strftime('%Y-%m-%d', gmtime($s[0])), '1993-10-02',
                'round trips date-only');
 }