-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# based on notmuch, but with no concept of folders, files
#
use PublicInbox::IdxStack;
use Carp qw(croak carp);
use POSIX qw(strftime);
+use Fcntl qw(SEEK_SET);
use Time::Local qw(timegm);
use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(spawn nodatacow_dir);
+use PublicInbox::Spawn qw(spawn);
use PublicInbox::Git qw(git_unquote);
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
use PublicInbox::Address;
my $X = \%PublicInbox::Search::X;
our ($DB_CREATE_OR_OPEN, $DB_OPEN);
our $DB_NO_SYNC = 0;
+our $DB_DANGEROUS = 0;
our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff :
# assume a typical 64-bit system has 8x more RAM than a
# typical 32-bit system:
(($Config{ptrsize} >= 8 ? 8192 : 1024) * 1024);
-
use constant DEBUG => !!$ENV{DEBUG};
-
+my $BASE85 = qr/\A[a-zA-Z0-9\!\#\$\%\&\(\)\*\+\-;<=>\?\@\^_`\{\|\}\~]+\z/;
my $xapianlevels = qr/\A(?:full|medium)\z/;
my $hex = '[a-f0-9]';
my $OID = $hex .'{40,}';
my $ver = (eval($xap.'::major_version()') << 16) |
(eval($xap.'::minor_version()') << 8) |
eval($xap.'::revision()');
- $DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+ if ($ver >= 0x10400) {
+ $DB_NO_SYNC = 0x4;
+ $DB_DANGEROUS = 0x10;
+ }
# Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
$X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
1;
if (!-d $dir && (!$is_shard ||
($is_shard && need_xapian($self)))) {
File::Path::mkpath($dir);
- nodatacow_dir($dir);
+ require PublicInbox::Syscall;
+ PublicInbox::Syscall::nodatacow_dir($dir);
$self->{-set_has_threadid_once} = 1;
+ if (($self->{ibx} // $self->{eidx})->{-dangerous}) {
+ $flag |= $DB_DANGEROUS;
+ }
}
}
return unless defined $flag;
# no renames or space support for traditional diffs,
# find the number of leading common paths to strip:
- my @fa = split('/', $fa);
- my @fb = split('/', $fb);
+ my @fa = split(m'/', $fa);
+ my @fb = split(m'/', $fb);
while (scalar(@fa) && scalar(@fb)) {
$fa = join('/', @fa);
$fb = join('/', @fb);
my ($self, $txt, $doc) = @_;
my %seen;
my $in_diff;
- my @xnq;
- my $xnq = \@xnq;
- foreach (split(/\n/, $txt)) {
- if ($in_diff && s/^ //) { # diff context
+ my $xnq = [];
+ my @l = split(/\n/, $$txt);
+ undef $$txt;
+ while (defined($_ = shift @l)) {
+ if ($in_diff && /^GIT binary patch/) {
+ push @$xnq, $_;
+ while (@l && $l[0] =~ /^(?:literal|delta) /) {
+ # TODO allow searching by size range?
+ # allows searching by exact size via:
+ # "literal $SIZE" or "delta $SIZE"
+ push @$xnq, shift(@l);
+
+ # skip base85 and empty lines
+ while (@l && ($l[0] =~ /$BASE85/o ||
+ $l[0] !~ /\S/)) {
+ shift @l;
+ }
+ # loop hits trailing "literal 0\nHcmV?d00001\n"
+ }
+ } elsif ($in_diff && s/^ //) { # diff context
index_diff_inc($self, $_, 'XDFCTX', $xnq);
} elsif (/^-- $/) { # email signature begins
$in_diff = undef;
- } elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) {
- # wait until "---" and "+++" to capture filenames
+ } elsif (m!^diff --git ("?[^/]+/.+) ("?[^/]+/.+)\z!) {
+ # capture filenames here for binary diffs:
+ my ($fa, $fb) = ($1, $2);
+ push @$xnq, $_;
$in_diff = 1;
- push @xnq, $_;
+ $fa = (split(m'/', git_unquote($fa), 2))[1];
+ $fb = (split(m'/', git_unquote($fb), 2))[1];
+ $seen{$fa}++ or index_diff_inc($self, $fa, 'XDFN', $xnq);
+ $seen{$fb}++ or index_diff_inc($self, $fb, 'XDFN', $xnq);
# traditional diff:
} elsif (m/^diff -(.+) (\S+) (\S+)$/) {
my ($opt, $fa, $fb) = ($1, $2, $3);
- push @xnq, $_;
+ push @$xnq, $_;
# only support unified:
next unless $opt =~ /[uU]/;
$in_diff = index_old_diff_fn($self, \%seen, $fa, $fb,
$xnq);
} elsif (m!^--- ("?[^/]+/.+)!) {
my $fn = $1;
- $fn = (split('/', git_unquote($fn), 2))[1];
+ $fn = (split(m'/', git_unquote($fn), 2))[1];
$seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq);
$in_diff = 1;
} elsif (m!^\+\+\+ ("?[^/]+/.+)!) {
my $fn = $1;
- $fn = (split('/', git_unquote($fn), 2))[1];
+ $fn = (split(m'/', git_unquote($fn), 2))[1];
$seen{$fn}++ or index_diff_inc($self, $fn, 'XDFN', $xnq);
$in_diff = 1;
} elsif (/^--- (\S+)/) {
- $in_diff = $1;
- push @xnq, $_;
+ $in_diff = $1; # old diff filename
+ push @$xnq, $_;
} elsif (defined $in_diff && /^\+\+\+ (\S+)/) {
$in_diff = index_old_diff_fn($self, \%seen, $in_diff,
$1, $xnq);
/^(?:dis)?similarity index / ||
/^\\ No newline at end of file/ ||
/^Binary files .* differ/) {
- push @xnq, $_;
+ push @$xnq, $_;
} elsif ($_ eq '') {
# possible to be in diff context, some mail may be
# stripped by MUA or even GNU diff(1). "git apply"
# treats a bare "\n" as diff context, too
} else {
- push @xnq, $_;
+ push @$xnq, $_;
warn "non-diff line: $_\n" if DEBUG && $_ ne '';
$in_diff = undef;
}
}
- index_text($self, join("\n", @xnq), 1, 'XNQ');
+ index_text($self, join("\n", @$xnq), 1, 'XNQ');
}
sub index_xapian { # msg_iter callback
defined $s or return;
$_[0]->[0] = $part = undef; # free memory
+ if ($s =~ /^(?:diff|---|\+\+\+) /ms) {
+ open(my $fh, '+>:utf8', undef) or die "open: $!";
+ open(my $eh, '+>', undef) or die "open: $!";
+ $fh->autoflush(1);
+ print $fh $s or die "print: $!";
+ sysseek($fh, 0, SEEK_SET) or die "sysseek: $!";
+ my $id = ($self->{ibx} // $self->{eidx})->git->qx(
+ [qw(patch-id --stable)],
+ {}, { 0 => $fh, 2 => $eh });
+ $id =~ /\A([a-f0-9]{40,})/ and $doc->add_term('XDFID'.$1);
+ seek($eh, 0, SEEK_SET) or die "seek: $!";
+ while (<$eh>) { warn $_ }
+ }
+
# split off quoted and unquoted blocks:
my @sections = PublicInbox::MsgIter::split_quotes($s);
undef $s; # free memory
} else {
# does it look like a diff?
if ($txt =~ /^(?:diff|---|\+\+\+) /ms) {
- index_diff($self, $txt, $doc);
+ index_diff($self, \$txt, $doc);
} else {
index_text($self, $txt, 1, 'XNQ');
}
push @cmd, "--$k=$v";
}
my $fh = $git->popen(@cmd, $range);
- my ($at, $ct, $stk, $cmt);
- while (<$fh>) {
+ my ($at, $ct, $stk, $cmt, $l);
+ while (defined($l = <$fh>)) {
return if $sync->{quit};
- if (/\A([0-9]+)-([0-9]+)-($OID)$/o) {
+ if ($l =~ /\A([0-9]+)-([0-9]+)-($OID)$/o) {
($at, $ct, $cmt) = ($1 + 0, $2 + 0, $3);
$stk //= PublicInbox::IdxStack->new($cmt);
- } elsif (/$del/) {
+ } elsif ($l =~ /$del/) {
my $oid = $1;
if ($D) { # reindex case
$D->{pack('H*', $oid)}++;
} else { # non-reindex case:
$stk->push_rec('d', $at, $ct, $oid, $cmt);
}
- } elsif (/$add/) {
+ } elsif ($l =~ /$add/) {
my $oid = $1;
if ($D) {
my $oid_bin = pack('H*', $oid);