X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=bdb84fc73dacf7bbfb1d463d386cdf466f1749ce;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hp=85fae4adf8b7f3f81c014776af5cbcd9fe29ef6d;hpb=3eec2f7792040f75f3988c520f308e2445baf645;p=public-inbox.git

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 85fae4ad..bdb84fc7 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -18,6 +18,7 @@ use PublicInbox::MsgIter;
 use PublicInbox::IdxStack;
 use Carp qw(croak carp);
 use POSIX qw(strftime);
+use Fcntl qw(SEEK_SET);
 use Time::Local qw(timegm);
 use PublicInbox::OverIdx;
 use PublicInbox::Spawn qw(spawn);
@@ -35,9 +36,8 @@ our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff :
 	# assume a typical 64-bit system has 8x more RAM than a
 	# typical 32-bit system:
 	(($Config{ptrsize} >= 8 ? 8192 : 1024) * 1024);
-
 use constant DEBUG => !!$ENV{DEBUG};
-
+my $BASE85 = qr/\A[a-zA-Z0-9\!\#\$\%\&\(\)\*\+\-;<=>\?\@\^_`\{\|\}\~]+\z/;
 my $xapianlevels = qr/\A(?:full|medium)\z/;
 my $hex = '[a-f0-9]';
 my $OID = $hex .'{40,}';
@@ -236,8 +236,8 @@ sub index_old_diff_fn {
 
 	# no renames or space support for traditional diffs,
 	# find the number of leading common paths to strip:
-	my @fa = split('/', $fa);
-	my @fb = split('/', $fb);
+	my @fa = split(m'/', $fa);
+	my @fb = split(m'/', $fb);
 	while (scalar(@fa) && scalar(@fb)) {
 		$fa = join('/', @fa);
 		$fb = join('/', @fb);
@@ -257,38 +257,59 @@ sub index_diff ($$$) {
 	my ($self, $txt, $doc) = @_;
 	my %seen;
 	my $in_diff;
-	my @xnq;
-	my $xnq = \@xnq;
-	foreach (split(/\n/, $txt)) {
-		if ($in_diff && s/^ //) { # diff context
+	my $xnq = [];
+	my @l = split(/\n/, $$txt);
+	undef $$txt;
+	while (defined($_ = shift @l)) {
+		if ($in_diff && /^GIT binary patch/) {
+			push @$xnq, $_;
+			while (@l && $l[0] =~ /^(?:literal|delta) /) {
+				# TODO allow searching by size range?
+				# allows searching by exact size via:
+				# "literal $SIZE" or "delta $SIZE"
+				push @$xnq, shift(@l);
+
+				# skip base85 and empty lines
+				while (@l && ($l[0] =~ /$BASE85/o ||
+						$l[0] !~ /\S/)) {
+					shift @l;
+				}
+				# loop hits trailing "literal 0\nHcmV?d00001\n"
+			}
+		} elsif ($in_diff && s/^ //) { # diff context
 			index_diff_inc($self, $_, 'XDFCTX', $xnq);
 		} elsif (/^-- $/) { # email signature begins
 			$in_diff = undef;
-		} elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) {
-			# wait until "---" and "+++" to capture filenames
+		} elsif (m!^diff --git ("?[^/]+/.+) ("?[^/]+/.+)\z!) {
+			# capture filenames here for binary diffs:
+			my ($fa, $fb) = ($1, $2);
+			push @$xnq, $_;
 			$in_diff = 1;
-			push @xnq, $_;
+			$fa = (split(m'/', git_unquote($fa), 2))[1];
+			$fb = (split(m'/', git_unquote($fb), 2))[1];
+			$seen{$fa}++ or index_diff_inc($self, $fa, 'XDFN', $xnq);
+			$seen{$fb}++ or index_diff_inc($self, $fb, 'XDFN', $xnq);
 		# traditional diff:
 		} elsif (m/^diff -(.+) (\S+) (\S+)$/) {
 			my ($opt, $fa, $fb) = ($1, $2, $3);
-			push @xnq, $_;
+			push @$xnq, $_;
 			# only support unified:
 			next unless $opt =~ /[uU]/;
 			$in_diff = index_old_diff_fn($self, \%seen, $fa, $fb,
 							$xnq);
 		} elsif (m!^--- ("?[^/]+/.+)!) {
 			my $fn = $1;
-			$fn = (split('/', git_unquote($fn), 2))[1];
+			$fn = (split(m'/', git_unquote($fn), 2))[1];
 			$seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq);
 			$in_diff = 1;
 		} elsif (m!^\+\+\+ ("?[^/]+/.+)!)  {
 			my $fn = $1;
-			$fn = (split('/', git_unquote($fn), 2))[1];
+			$fn = (split(m'/', git_unquote($fn), 2))[1];
 			$seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq);
 			$in_diff = 1;
 		} elsif (/^--- (\S+)/) {
-			$in_diff = $1;
-			push @xnq, $_;
+			$in_diff = $1; # old diff filename
+			push @$xnq, $_;
 		} elsif (defined $in_diff && /^\+\+\+ (\S+)/) {
 			$in_diff = index_old_diff_fn($self, \%seen, $in_diff,
 							$1, $xnq);
@@ -314,19 +335,19 @@ sub index_diff ($$$) {
 				/^(?:dis)?similarity index / ||
 				/^\\ No newline at end of file/ ||
 				/^Binary files .* differ/) {
-			push @xnq, $_;
+			push @$xnq, $_;
 		} elsif ($_ eq '') {
 			# possible to be in diff context, some mail may be
 			# stripped by MUA or even GNU diff(1).  "git apply"
 			# treats a bare "\n" as diff context, too
 		} else {
-			push @xnq, $_;
+			push @$xnq, $_;
 			warn "non-diff line: $_\n" if DEBUG && $_ ne '';
 			$in_diff = undef;
 		}
 	}
 
-	index_text($self, join("\n", @xnq), 1, 'XNQ');
+	index_text($self, join("\n", @$xnq), 1, 'XNQ');
 }
 
 sub index_xapian { # msg_iter callback
@@ -349,6 +370,20 @@ sub index_xapian { # msg_iter callback
 	defined $s or return;
 	$_[0]->[0] = $part = undef; # free memory
 
+	if ($s =~ /^(?:diff|---|\+\+\+) /ms) {
+		open(my $fh, '+>:utf8', undef) or die "open: $!";
+		open(my $eh, '+>', undef) or die "open: $!";
+		$fh->autoflush(1);
+		print $fh $s or die "print: $!";
+		sysseek($fh, 0, SEEK_SET) or die "sysseek: $!";
+		my $id = ($self->{ibx} // $self->{eidx})->git->qx(
+						[qw(patch-id --stable)],
+						{}, { 0 => $fh, 2 => $eh });
+		$id =~ /\A([a-f0-9]{40,})/ and $doc->add_term('XDFID'.$1);
+		seek($eh, 0, SEEK_SET) or die "seek: $!";
+		while (<$eh>) { warn $_ }
+	}
+
 	# split off quoted and unquoted blocks:
 	my @sections = PublicInbox::MsgIter::split_quotes($s);
 	undef $s; # free memory
@@ -358,7 +393,7 @@ sub index_xapian { # msg_iter callback
 		} else {
 			# does it look like a diff?
 			if ($txt =~ /^(?:diff|---|\+\+\+) /ms) {
-				index_diff($self, $txt, $doc);
+				index_diff($self, \$txt, $doc);
 			} else {
 				index_text($self, $txt, 1, 'XNQ');
 			}