summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
738c4a6)
When importing several sources in parallel via http(s) mboxrd,
we need to be able to get keywords of uncommitted documents
directly from shard workers. Otherwise, Xapian DocNotFound
errors happen because the read-only LeiSearch won't see
documents from uncomitted transactions. Keep in mind that it's
possible the keywords can be changed on-the-fly even for
uncommitted documents because of inotify watches from LeiNoteEvent.
sto_export_kw($self, $docid, $vmd);
}
sto_export_kw($self, $docid, $vmd);
}
+sub _docids_and_maybe_kw ($$) {
+ my ($self, $docids) = @_;
+ return $docids unless wantarray;
+ my $kw = {};
+ for my $num (@$docids) { # likely only 1, unless ContentHash changes
+ # can't use ->search->msg_keywords on uncommitted docs
+ my $idx = $self->{priv_eidx}->idx_shard($num);
+ my $tmp = eval { $idx->ipc_do('get_terms', 'K', $num) };
+ if ($@) { warn "#$num get_terms: $@" }
+ else { @$kw{keys %$tmp} = values(%$tmp) };
+ }
+ ($docids, [ sort keys %$kw ]);
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
if ($vmd && $vmd->{sync_info}) {
set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
}
if ($vmd && $vmd->{sync_info}) {
set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
}
- $im_mark or return; # duplicate blob returns undef
+ unless ($im_mark) { # duplicate blob returns undef
+ return unless wantarray;
+ my @docids = $oidx->blob_exists($smsg->{blob});
+ return _docids_and_maybe_kw $self, \@docids;
+ }
local $self->{current_info} = $smsg->{blob};
my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches
local $self->{current_info} = $smsg->{blob};
my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches
}
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
}
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
+ _docids_and_maybe_kw $self, $vivify_xvmd;
} elsif (my @docids = _docids_for($self, $eml)) {
# fuzzy match from within lei/store
for my $docid (@docids) {
} elsif (my @docids = _docids_for($self, $eml)) {
# fuzzy match from within lei/store
for my $docid (@docids) {
$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
- \@docids;
- } else { # totally new message
+ _docids_and_maybe_kw $self, \@docids;
+ } else { # totally new message, no keywords
delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_eml($eml, $smsg);
_add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd;
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_eml($eml, $smsg);
_add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd;
+ wantarray ? ($smsg, []) : $smsg;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- my $res = $self->{import_sto}->wq_do('add_eml', $eml);
- if (ref($res) eq ref($smsg)) { # totally new message
- $smsg = $res;
- $smsg->{kw} = []; # short-circuit xsmsg_vmd
- }
+ my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml);
+ $smsg = $res if ref($res) eq ref($smsg); # totally new message
+ $smsg->{kw} = $kw; # short-circuit xsmsg_vmd
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: $lei->git_oid($eml)->hexdigest;
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: $lei->git_oid($eml)->hexdigest;
$self->{xdb}->replace_document($docid, $doc);
}
$self->{xdb}->replace_document($docid, $doc);
}
+sub get_terms {
+ my ($self, $pfx, $docid) = @_;
+ begin_txn_lazy($self);
+ xap_terms($pfx, $self->{xdb}, $docid);
+}
+
sub remove_eidx_info {
my ($self, $docid, $eidx_key, $eml) = @_;
begin_txn_lazy($self);
sub remove_eidx_info {
my ($self, $docid, $eidx_key, $eml) = @_;
begin_txn_lazy($self);
WHAT IS TIME ANYMORE?
EOM
WHAT IS TIME ANYMORE?
EOM
- ok($sto->add_eml($eml), 'recently received message');
+ my $smsg = $sto->add_eml($eml);
+ ok($smsg && $smsg->{blob}, 'recently received message');
$sto->done;
local $ENV{TZ} = 'GMT+5';
my $lse = $sto->search;
$sto->done;
local $ENV{TZ} = 'GMT+5';
my $lse = $sto->search;