xap_rw => undef, # PublicInbox::V2SearchIdx
xap_ro => undef,
partitions => $nparts,
+ parallel => 1,
transact_bytes => 0,
lock_path => "$dir/inbox.lock",
# limit each repo to 1GB or so
my $nparts = $self->{partitions};
my $part = $num % $nparts;
my $idx = $self->idx_part($part);
- $idx->index_raw($len, $msgref, $num, $oid, $mid0);
+ $idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime);
my $n = $self->{transact_bytes} += $len;
if ($n > (PublicInbox::SearchIdx::BATCH_BYTES * $nparts)) {
$self->checkpoint;
foreach my $m (@$mids) {
# read-only lookup now safe to do after above barrier
my $existing = $self->lookup_content($mime, $m);
- if ($existing) {
- warn "<$m> resent\n";
- return; # easy, don't store duplicates
- }
+ # easy, don't store duplicates
+ # note: do not add more diagnostic info here since
+ # it gets noisy on public-inbox-watch restarts
+ return if $existing;
}
# very unlikely:
# Now deal with Xapian
$skel->barrier_init(scalar(@$parts));
# each partition needs to issue a barrier command to skel:
- $_->barrier foreach @$parts;
+ $_->remote_barrier foreach @$parts;
$skel->barrier_wait; # wait for each Xapian partition
$self->{im} = $im;
}
+# XXX experimental
+sub diff ($$$) {
+ my ($mid, $cur, $new) = @_;
+ use File::Temp qw(tempfile);
+ use PublicInbox::Spawn qw(spawn);
+
+ my ($ah, $an) = tempfile('email-cur-XXXXXXXX');
+ print $ah $cur->as_string or die "print: $!";
+ close $ah or die "close: $!";
+ my ($bh, $bn) = tempfile('email-new-XXXXXXXX');
+ print $bh $new->as_string or die "print: $!";
+ close $bh or die "close: $!";
+ my $cmd = [ qw(diff -u), $an, $bn ];
+ print STDERR "# MID conflict <$mid>\n";
+ my $pid = spawn($cmd, undef, { 1 => 2 });
+ defined $pid or die "diff failed to spawn $!";
+ waitpid($pid, 0) == $pid or die "diff did not finish";
+ unlink($an, $bn);
+}
+
sub lookup_content {
my ($self, $mime, $mid) = @_;
my $ibx = $self->{-inbox};
$found = $smsg;
return 0; # break out of loop
}
+
+ # XXX DEBUG_DIFF is experimental and may be removed
+ diff($mid, $cur, $mime) if $ENV{DEBUG_DIFF};
+
1; # continue
});
$found;