use PublicInbox::ProcessPipe;
use PublicInbox::Spawn qw(which spawn popen_rd);
use PublicInbox::LeiDedupe;
-use PublicInbox::OnDestroy;
use PublicInbox::Git;
use PublicInbox::GitAsyncCat;
+use PublicInbox::PktOp qw(pkt_do);
use Symbol qw(gensym);
use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
use Errno qw(EEXIST ESPIPE ENOENT EPIPE);
-my ($maildir_each_file);
+use Digest::SHA qw(sha256_hex);
# struggles with short-lived repos, Gcf2Client makes little sense with lei;
# but we may use in-process libgit2 in the future.
# { foo => '' } means "--foo" is passed to the command-line,
# otherwise { foo => '--bar' } passes "--bar"
our %zsfx2cmd = (
- gz => [ qw(GZIP pigz gzip), { rsyncable => '', threads => '-p' } ],
+ gz => [ qw(GZIP pigz gzip), { rsyncable => '' } ],
bz2 => [ 'bzip2', {} ],
- xz => [ 'xz', { threads => '-T' } ],
+ xz => [ 'xz', {} ],
# XXX does anybody care for these? I prefer zstd on entire FSes,
# so it's probably not necessary on a per-file basis
# zst => [ 'zstd', { -default => [ qw(-q) ], # it's noisy by default
my $switch = $cmd_opt->{rsyncable} // next;
push @cmd, '--'.($switch || $bool);
}
- for my $key (qw(threads)) { # support compression level?
+ for my $key (qw(rsyncable)) { # support compression level?
my $switch = $cmd_opt->{$key} // next;
my $val = $lei->{opt}->{$key} // next;
push @cmd, $switch, $val;
my $cmd = zsfx2cmd($zsfx, undef, $lei);
my ($r, $w) = @{delete $lei->{zpipe}};
my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
- my $pid = spawn($cmd, $lei->{env}, $rdr);
+ my $pid = spawn($cmd, undef, $rdr);
my $pp = gensym;
my $dup = bless { "pid.$pid" => $cmd }, ref($lei);
$dup->{$_} = $lei->{$_} for qw(2 sock);
tie *$pp, 'PublicInbox::ProcessPipe', $pid, $w, \&reap_compress, $dup;
$lei->{1} = $pp;
- die 'BUG: unexpected {ovv}->{lock_path}' if $lei->{ovv}->{lock_path};
- $lei->{ovv}->ovv_out_lk_init;
}
sub decompress_src ($$$) {
my ($in, $zsfx, $lei) = @_;
my $cmd = zsfx2cmd($zsfx, 1, $lei);
- popen_rd($cmd, $lei->{env}, { 0 => $in, 2 => $lei->{2} });
+ popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
}
sub dup_src ($) {
$lei->{dedupe}->is_dup($eml);
}
+sub _mbox_augment_kw_maybe {
+ my ($eml, $lei, $lse, $augment) = @_;
+ my $kw = PublicInbox::MboxReader::mbox_keywords($eml);
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ _augment($eml, $lei) if $augment;
+}
+
sub _mbox_write_cb ($$) {
my ($self, $lei) = @_;
my $ovv = $lei->{ovv};
}
}
-sub _augment_file { # maildir_each_file cb
- my ($f, $lei) = @_;
- my $eml = PublicInbox::InboxWritable::eml_from_path($f) or return;
- _augment($eml, $lei);
+sub update_kw_maybe ($$$$) {
+ my ($lei, $lse, $eml, $kw) = @_;
+ return unless $lse;
+ my $x = $lse->kw_changed($eml, $kw);
+ if ($x) {
+ $lei->{sto}->ipc_do('set_eml', $eml, @$kw);
+ } elsif (!defined($x)) {
+ $lei->{sto}->ipc_do('set_xkw', $eml, $kw);
+ }
+}
+
+sub _augment_or_unlink { # maildir_each_eml cb
+ my ($f, $kw, $eml, $lei, $lse, $mod, $shard, $unlink) = @_;
+ if ($mod) {
+ # can't get dirent.d_ino w/ pure Perl, so we extract the OID
+ # if it looks like one:
+ my $hex = $f =~ m!\b([a-f0-9]{40,})[^/]*\z! ?
+ $1 : sha256_hex($f);
+ my $recno = hex(substr($hex, 0, 8));
+ return if ($recno % $mod) != $shard;
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ }
+ $unlink ? unlink($f) : _augment($eml, $lei);
}
# maildir_each_file callback, \&CORE::unlink doesn't work with it
my ($self, $lei) = @_;
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
- my $imap_append = $lei->{nwr}->can('imap_append');
- my $mic = $lei->{nwr}->mic_get($lei->{ovv}->{dst});
+ my $imap_append = $lei->{net}->can('imap_append');
+ my $mic = $lei->{net}->mic_get($self->{uri});
my $folder = $self->{uri}->mailbox;
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
my $dst = $lei->{ovv}->{dst};
my $self = bless {}, $cls;
if ($fmt eq 'maildir') {
- $maildir_each_file //= do {
- require PublicInbox::MdirReader;
- PublicInbox::MdirReader->can('maildir_each_file');
- };
- $lei->{opt}->{augment} and
- require PublicInbox::InboxWritable; # eml_from_path
+ require PublicInbox::MdirReader;
$self->{base_type} = 'maildir';
-e $dst && !-d _ and die
"$dst exists and is not a directory\n";
$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
} elsif (substr($fmt, 0, 4) eq 'mbox') {
- require PublicInbox::MboxReader if $lei->{opt}->{augment};
+ require PublicInbox::MboxReader;
(-d $dst || (-e _ && !-w _)) and die
"$dst exists and is not a writable file\n";
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
$self->{base_type} = 'mbox';
} elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
require PublicInbox::NetWriter;
- my $nwr = PublicInbox::NetWriter->new;
- $nwr->add_url($dst);
- $nwr->{quiet} = $lei->{opt}->{quiet};
- my $err = $nwr->errors($dst);
+ my $net = PublicInbox::NetWriter->new;
+ $net->add_url($dst);
+ $net->{quiet} = $lei->{opt}->{quiet};
+ my $err = $net->errors($dst);
return $lei->fail($err) if $err;
require PublicInbox::URIimap; # TODO: URI cast early
$self->{uri} = PublicInbox::URIimap->new($dst);
$self->{uri}->mailbox or die "No mailbox: $dst";
- $lei->{nwr} = $nwr;
+ $lei->{net} = $net;
$self->{base_type} = 'imap';
} else {
die "bad mail --format=$fmt\n";
sub _do_augment_maildir {
my ($self, $lei) = @_;
my $dst = $lei->{ovv}->{dst};
+ my $lse = $lei->{sto}->search if $lei->{opt}->{'import-before'};
+ my ($mod, $shard) = @{$self->{shard_info} // []};
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- $maildir_each_file->($dst, \&_augment_file, $lei);
+ PublicInbox::MdirReader::maildir_each_eml($dst,
+ \&_augment_or_unlink,
+ $lei, $lse, $mod, $shard);
$dedupe->pause_dedupe;
}
- } else { # clobber existing Maildir
- $maildir_each_file->($dst, \&_unlink);
+ } elsif ($lse) {
+ PublicInbox::MdirReader::maildir_each_eml($dst,
+ \&_augment_or_unlink,
+ $lei, $lse, $mod, $shard, 1);
+ } else {# clobber existing Maildir
+ PublicInbox::MdirReader::maildir_each_file($dst, \&_unlink);
}
}
-sub _augment_imap { # PublicInbox::NetReader::imap_each cb
- my ($url, $uid, $kw, $eml, $lei) = @_;
- _augment($eml, $lei);
+sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
+ my ($url, $uid, $kw, $eml, $lei, $lse, $delete_mic) = @_;
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ if ($delete_mic) {
+ $lei->{net}->imap_delete_1($url, $uid, $delete_mic);
+ } else {
+ _augment($eml, $lei);
+ }
}
sub _do_augment_imap {
my ($self, $lei) = @_;
- my $dst = $lei->{ovv}->{dst};
- my $nwr = $lei->{nwr};
+ my $net = $lei->{net};
+ my $lse = $lei->{sto}->search if $lei->{opt}->{'import-before'};
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- $nwr->imap_each($dst, \&_augment_imap, $lei);
+ $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+ $lei, $lse);
$dedupe->pause_dedupe;
}
- } else { # clobber existing IMAP folder
- $nwr->imap_delete_all($dst);
+ } elsif ($lse) {
+ my $delete_mic;
+ $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+ $lei, $lse, \$delete_mic);
+ $delete_mic->expunge if $delete_mic;
+ } elsif (!$self->{-wq_worker_nr}) { # undef or 0
+ # clobber existing IMAP folder
+ $net->imap_delete_all($self->{uri});
}
}
sub _pre_augment_mbox {
my ($self, $lei) = @_;
my $dst = $lei->{ovv}->{dst};
+ my $out = $lei->{1};
if ($dst ne '/dev/stdout') {
- my $mode = -p $dst ? '>' : '+>>';
- if (-f _ && !$lei->{opt}->{augment} and !unlink($dst)) {
- $! == ENOENT or die "unlink($dst): $!";
+ if (-p $dst) {
+ $out = undef;
+ open $out, '>', $dst or die "open($dst): $!";
+ } elsif (-f _ || !-e _) {
+ require PublicInbox::MboxLock;
+ my $m = $lei->{opt}->{'lock'} //
+ PublicInbox::MboxLock->defaults;
+ $self->{mbl} = PublicInbox::MboxLock->acq($dst, 1, $m);
+ $out = $self->{mbl}->{fh};
}
- open my $out, $mode, $dst or die "open($dst): $!";
$lei->{old_1} = $lei->{1}; # keep for spawning MUA
- $lei->{1} = $out;
}
# Perl does SEEK_END even with O_APPEND :<
- $self->{seekable} = seek($lei->{1}, 0, SEEK_SET);
+ $self->{seekable} = seek($out, 0, SEEK_SET);
if (!$self->{seekable} && $! != ESPIPE && $dst ne '/dev/stdout') {
die "seek($dst): $!\n";
}
+ if (!$self->{seekable}) {
+ my $ia = $lei->{opt}->{'import-before'};
+ die "--import-before specified but $dst is not seekable\n"
+ if $ia && !ref($ia);
+ die "--augment specified but $dst is not seekable\n" if
+ $lei->{opt}->{augment};
+ }
state $zsfx_allow = join('|', keys %zsfx2cmd);
- ($self->{zsfx}) = ($dst =~ /\.($zsfx_allow)\z/) or return;
- pipe(my ($r, $w)) or die "pipe: $!";
- $lei->{zpipe} = [ $r, $w ];
+ if (($self->{zsfx}) = ($dst =~ /\.($zsfx_allow)\z/)) {
+ pipe(my ($r, $w)) or die "pipe: $!";
+ $lei->{zpipe} = [ $r, $w ];
+ $lei->{ovv}->{lock_path} and
+ die 'BUG: unexpected {ovv}->{lock_path}';
+ $lei->{ovv}->ovv_out_lk_init;
+ } elsif (!$self->{seekable} && !$lei->{ovv}->{lock_path}) {
+ $lei->{ovv}->ovv_out_lk_init;
+ }
+ $lei->{1} = $out;
+ undef;
}
sub _do_augment_mbox {
my ($self, $lei) = @_;
- return if !$lei->{opt}->{augment};
- my $dedupe = $lei->{dedupe};
- my $dst = $lei->{ovv}->{dst};
- die "cannot augment $dst, not seekable\n" if !$self->{seekable};
+ return unless $self->{seekable};
+ my $opt = $lei->{opt};
my $out = $lei->{1};
- if (-s $out && $dedupe && $dedupe->prepare_dedupe) {
- my $zsfx = $self->{zsfx};
- my $rd = $zsfx ? decompress_src($out, $zsfx, $lei) :
- dup_src($out);
- my $fmt = $lei->{ovv}->{fmt};
+ my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)};
+ return unless -s $out;
+ unless ($opt->{augment} || $opt->{'import-before'}) {
+ truncate($out, 0) or die "truncate($dst): $!";
+ return;
+ }
+ my $zsfx = $self->{zsfx};
+ my $rd = $zsfx ? decompress_src($out, $zsfx, $lei) : dup_src($out);
+ my $dedupe;
+ if ($opt->{augment}) {
+ $dedupe = $lei->{dedupe};
+ $dedupe->prepare_dedupe if $dedupe;
+ }
+ if ($opt->{'import-before'}) { # the default
+ my $lse = $lei->{sto}->search;
+ PublicInbox::MboxReader->$fmt($rd, \&_mbox_augment_kw_maybe,
+ $lei, $lse, $opt->{augment});
+ if (!$opt->{augment} and !truncate($out, 0)) {
+ die "truncate($dst): $!";
+ }
+ } else { # --augment --no-import-before
PublicInbox::MboxReader->$fmt($rd, \&_augment, $lei);
}
# maybe some systems don't honor O_APPEND, Perl does this:
sub do_augment { # slow, runs in wq worker
my ($self, $lei) = @_;
- # _do_augment_maildir, _do_augment_mbox
+ # _do_augment_maildir, _do_augment_mbox, or _do_augment_imap
my $m = "_do_augment_$self->{base_type}";
$self->$m($lei);
}
# fast (spawn compressor or mkdir), runs in same process as pre_augment
sub post_augment {
my ($self, $lei, @args) = @_;
+ my $wait = $lei->{opt}->{'import-before'} ?
+ $lei->{sto}->ipc_do('checkpoint', 1) : 0;
# _post_augment_mbox
my $m = $self->can("_post_augment_$self->{base_type}") or return;
$m->($self, $lei, @args);
}
-sub ipc_atfork_child {
+sub do_post_auth {
my ($self) = @_;
- my $lei = delete $self->{lei};
- $lei->lei_atfork_child;
+ my $lei = $self->{lei};
+ # lei_xsearch can start as soon as all l2m workers get here
+ pkt_do($lei->{pkt_op_p}, 'incr_start_query') or
+ die "incr_start_query: $!";
+ my $aug;
+ if (lock_free($self)) {
+ my $mod = $self->{-wq_nr_workers};
+ my $shard = $self->{-wq_worker_nr};
+ if (my $net = $lei->{net}) {
+ $net->{shard_info} = [ $mod, $shard ];
+ } else { # Maildir (MH?)
+ $self->{shard_info} = [ $mod, $shard ];
+ }
+ $aug = '+'; # incr_post_augment
+ } elsif ($self->{-wq_worker_nr} == 0) {
+ $aug = '.'; # do_post_augment
+ }
+ if ($aug) {
+ local $0 = 'do_augment';
+ eval { do_augment($self, $lei) };
+ $lei->fail($@) if $@;
+ pkt_do($lei->{pkt_op_p}, $aug) == 1 or
+ die "do_post_augment trigger: $!";
+ }
if (my $zpipe = delete $lei->{zpipe}) {
$lei->{1} = $zpipe->[1];
close $zpipe->[0];
}
$self->{wcb} = $self->write_cb($lei);
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->lei_atfork_child;
+ $lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
$SIG{__WARN__} = 'DEFAULT';
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_complete {
+ my ($self) = @_;
+ $self->wq_broadcast('do_post_auth');
+ $self->wq_close(1);
+}
+
+no warnings 'once'; # the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
1;