use v5.10.1;
use parent qw(PublicInbox::LeiSearch PublicInbox::IPC);
use PublicInbox::DS qw(now);
-use PublicInbox::PktOp qw(pkt_do);
use File::Temp 0.19 (); # 0.19 for ->newdir
use File::Spec ();
use PublicInbox::Search qw(xap_terms);
use PublicInbox::Eml;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
+use POSIX qw(strftime);
sub new {
my ($class) = @_;
my ($self, $ibxish) = @_; # ibxish = ExtSearch or Inbox
my $desc = $ibxish->{inboxdir} // $ibxish->{topdir};
my $srch = $ibxish->search or
- return warn("$desc not indexed for Xapian\n");
+ return warn("$desc not indexed for Xapian ($@ $!)\n");
my @shards = $srch->xdb_shards_flat or
return warn("$desc has no Xapian shards\n");
my $doc = $mitem->get_document;
my $kw = xap_terms('K', $doc);
$kw->{flagged} = 1 if $flagged;
- my $L = xap_terms('L', $doc);
+ my @L = xap_terms('L', $doc);
# we keep the empty {kw} array here to prevent expensive work in
# ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
$smsg->{kw} = [ sort keys %$kw ];
- $smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
+ $smsg->{L} = \@L if scalar(@L);
}
sub mitem_kw ($$$;$) {
my $lei = shift;
return if $lei->{early_mua} || !$lei->{-progress};
if ($lei->{pkt_op_p}) {
- pkt_do($lei->{pkt_op_p}, 'mset_progress', @_);
+ $lei->{pkt_op_p}->pkt_do('mset_progress', @_);
} else { # single lei-daemon consumer
my ($desc, $mset_size, $mset_total_est) = @_;
$lei->{-mset_total} += $mset_size if $mset_total_est ne '?';
my ($srch, $over) = ($ibxish->search, $ibxish->over);
my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+ bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
my $mo = { %{$lei->{mset_opt}} }; # copy
my $mset;
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
my $can_kw = !!$ibxish->can('msg_keywords');
my $threads = $lei->{opt}->{threads} // 0;
my $fl = $threads > 1 ? 1 : undef;
- my $lss = $lei->{dedupe};
- $lss = undef unless $lss && $lss->can('cfg_set'); # saved search
+ my $lss = $lei->{lss};
my $maxk = "external.$dir.maxuid";
my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef;
if (defined $stop_at) {
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ if (ref($res) eq ref($smsg)) { # totally new message
+ $smsg = $res;
+ $smsg->{kw} = []; # short-circuit xsmsg_vmd
+ }
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
+ : $lei->git_oid($eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
$each_smsg->($smsg, undef, $eml);
}
+sub fudge_qstr_time ($$$) {
+ my ($lei, $uri, $qstr) = @_;
+ return ($qstr, undef) unless $lei->{lss};
+ my $cfg = $lei->{lss}->{-cfg} // die 'BUG: no lss->{-cfg}';
+ my $cfg_key = "external.$uri.lastresult";
+ my $lr = $cfg->{$cfg_key} or return ($qstr, $cfg_key);
+ if ($lr !~ /\A\-?[0-9]+\z/) {
+ $lei->child_error(0,
+ "$cfg->{-f}: $cfg_key=$lr not an integer, ignoring");
+ return ($qstr, $cfg_key);
+ }
+ my $rft = $lei->{opt}->{'remote-fudge-time'};
+ if ($rft && $rft !~ /\A-?[0-9]+\z/) {
+ my @t = $lei->{lss}->git->date_parse($rft);
+ my $diff = time - $t[0];
+ $lei->qerr("# $rft => $diff seconds");
+ $rft = $diff;
+ }
+ $lr -= ($rft || (48 * 60 * 60));
+ $lei->qerr("# $uri limiting to ".
+ strftime('%Y-%m-%d %k:%M', gmtime($lr)). ' and newer');
+ # this should really be rt: (received-time), but no stable
+ # public-inbox releases support it, yet.
+ my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
+ if ($qstr =~ /\S/) {
+ substr($qstr, 0, 0, '(');
+ $qstr .= ') AND ';
+ }
+ ($qstr .= $dt, $cfg_key);
+}
+
sub query_remote_mboxrd {
my ($self, $uris) = @_;
local $0 = "$0 query_remote_mboxrd";
local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
my $lei = $self->{lei};
my $opt = $lei->{opt};
- my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+ my $qstr = $lei->{mset_opt}->{qstr};
+ $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+ my @qform = (x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
for my $uri (@$uris) {
$lei->{-current_url} = $uri->as_string;
$lei->{-nr_remote_eml} = 0;
- $uri->query_form(@qform);
+ my $start = time;
+ my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
+ $uri->query_form(@qform, q => $q);
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
my $nr = $lei->{-nr_remote_eml};
- if ($nr && $lei->{sto}) {
- my $wait = $lei->{sto}->ipc_do('done');
- }
+ my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
if ($? == 0) {
+ # don't update if no results, maybe MTA is down
+ $key && $nr and
+ $lei->{lss}->cfg_set($key, $start);
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
$lei->err("truncate($cmd stderr): $!");
}
next if (($? >> 8) == 22 && $err =~ /\b404\b/);
- $uri->query_form(q => $lei->{mset_opt}->{qstr});
+ $uri->query_form(q => $qstr);
$lei->child_error($?, "E: <$uri> $err");
}
undef $each_smsg;
if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
warn "BUG: {sto} missing with --mail-sync";
}
- my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
- $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+ $lei->sto_done_request if $lei->{sto};
+ my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
if ($lei->{-progress}) {
my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
- $lei->qerr($l2m ?
- "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
- "# $tot matches");
+ if ($l2m) {
+ my $m = "# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)";
+ $nr ? $lei->qfin($m) : $lei->qerr($m);
+ } else {
+ $lei->qerr("# $tot matches");
+ }
}
$lei->start_mua if $start_mua;
$lei->dclose;
my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
if ($l2m->{-wq_nr_workers} > 1 &&
$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
- # setup two barriers to coordinate dedupe_nr
+ # setup two barriers to coordinate ->has_entries
# between l2m workers
pipe(my ($a_r, $a_w)) or die "pipe: $!";
fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
start_query($self);
}
$lei->event_step_init; # wait for shutdowns
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub add_uri {
die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
require PublicInbox::Inbox; # v2, v1
$loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
+ } elsif (!-e $loc) {
+ warn "W: $loc gone, perhaps run: lei forget-external $loc\n";
+ return;
} else {
- warn "W: ignoring $loc, unable to determine type\n";
+ warn "W: $loc ignored, unable to determine external type\n";
return;
}
push @{$self->{locals}}, $loc;