use PublicInbox::ProcessPipe;
use PublicInbox::Spawn qw(which spawn popen_rd);
use PublicInbox::LeiDedupe;
-use PublicInbox::OnDestroy;
use PublicInbox::Git;
use PublicInbox::GitAsyncCat;
use PublicInbox::PktOp qw(pkt_do);
use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
use Errno qw(EEXIST ESPIPE ENOENT EPIPE);
+use Digest::SHA qw(sha256_hex);
my ($maildir_each_file);
# struggles with short-lived repos, Gcf2Client makes little sense with lei;
my $cmd = zsfx2cmd($zsfx, undef, $lei);
my ($r, $w) = @{delete $lei->{zpipe}};
my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
- my $pid = spawn($cmd, $lei->{env}, $rdr);
+ my $pid = spawn($cmd, undef, $rdr);
my $pp = gensym;
my $dup = bless { "pid.$pid" => $cmd }, ref($lei);
$dup->{$_} = $lei->{$_} for qw(2 sock);
sub decompress_src ($$$) {
my ($in, $zsfx, $lei) = @_;
my $cmd = zsfx2cmd($zsfx, 1, $lei);
- popen_rd($cmd, $lei->{env}, { 0 => $in, 2 => $lei->{2} });
+ popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
}
sub dup_src ($) {
}
sub _augment_file { # maildir_each_file cb
- my ($f, $lei) = @_;
+ my ($f, $lei, $mod, $shard) = @_;
+ if ($mod) {
+ # can't get dirent.d_ino w/ pure Perl, so we extract the OID
+ # if it looks like one:
+ my $hex = $f =~ m!\b([a-f0-9]{40,})[^/]*\z! ?
+ $1 : sha256_hex($f);
+ my $recno = hex(substr($hex, 0, 8));
+ return if ($recno % $mod) != $shard;
+ }
my $eml = PublicInbox::InboxWritable::eml_from_path($f) or return;
_augment($eml, $lei);
}
my ($self, $lei) = @_;
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
- my $imap_append = $lei->{nwr}->can('imap_append');
- my $mic = $lei->{nwr}->mic_get($lei->{ovv}->{dst});
+ my $imap_append = $lei->{net}->can('imap_append');
+ my $mic = $lei->{net}->mic_get($self->{uri});
my $folder = $self->{uri}->mailbox;
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$self->{base_type} = 'mbox';
} elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
require PublicInbox::NetWriter;
- my $nwr = PublicInbox::NetWriter->new;
- $nwr->add_url($dst);
- $nwr->{quiet} = $lei->{opt}->{quiet};
- my $err = $nwr->errors($dst);
+ my $net = PublicInbox::NetWriter->new;
+ $net->add_url($dst);
+ $net->{quiet} = $lei->{opt}->{quiet};
+ my $err = $net->errors($dst);
return $lei->fail($err) if $err;
require PublicInbox::URIimap; # TODO: URI cast early
$self->{uri} = PublicInbox::URIimap->new($dst);
$self->{uri}->mailbox or die "No mailbox: $dst";
- $lei->{nwr} = $nwr;
+ $lei->{net} = $net;
$self->{base_type} = 'imap';
} else {
die "bad mail --format=$fmt\n";
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- $maildir_each_file->($dst, \&_augment_file, $lei);
+ my ($mod, $shard) = @{$self->{shard_info} // []};
+ $maildir_each_file->($dst, \&_augment_file,
+ $lei, $mod, $shard);
$dedupe->pause_dedupe;
}
} else { # clobber existing Maildir
sub _do_augment_imap {
my ($self, $lei) = @_;
- my $dst = $lei->{ovv}->{dst};
- my $nwr = $lei->{nwr};
+ my $net = $lei->{net};
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- $nwr->imap_each($dst, \&_augment_imap, $lei);
+ $net->imap_each($self->{uri}, \&_augment_imap, $lei);
$dedupe->pause_dedupe;
}
- } else { # clobber existing IMAP folder
- $nwr->imap_delete_all($dst);
+ } elsif (!$self->{-wq_worker_nr}) { # undef or 0
+ # clobber existing IMAP folder
+ $net->imap_delete_all($self->{uri});
}
}
my ($self, $lei) = @_;
my $dst = $lei->{ovv}->{dst};
if ($dst ne '/dev/stdout') {
- my $mode = -p $dst ? '>' : '+>>';
- if (-f _ && !$lei->{opt}->{augment} and !unlink($dst)) {
- $! == ENOENT or die "unlink($dst): $!";
+ my $out;
+ if (-p $dst) {
+ open $out, '>', $dst or die "open($dst): $!";
+ } elsif (-f _ || !-e _) {
+ require PublicInbox::MboxLock;
+ my $m = $lei->{opt}->{'lock'} //
+ PublicInbox::MboxLock->defaults;
+ $self->{mbl} = PublicInbox::MboxLock->acq($dst, 1, $m);
+ $out = $self->{mbl}->{fh};
+ if (!$lei->{opt}->{augment} and !truncate($out, 0)) {
+ die "truncate($dst): $!";
+ }
}
- open my $out, $mode, $dst or die "open($dst): $!";
$lei->{old_1} = $lei->{1}; # keep for spawning MUA
$lei->{1} = $out;
}
$m->($self, $lei, @args);
}
-sub ipc_atfork_child {
+sub do_post_auth {
my ($self) = @_;
- my $lei = delete $self->{lei};
- $lei->lei_atfork_child;
- if ($self->{-wq_worker_nr} == 0) {
+ my $lei = $self->{lei};
+ # lei_xsearch can start as soon as all l2m workers get here
+ pkt_do($lei->{pkt_op_p}, 'incr_start_query') or
+ die "incr_start_query: $!";
+ my $aug;
+ if (lock_free($self)) {
+ my $mod = $self->{-wq_nr_workers};
+ my $shard = $self->{-wq_worker_nr};
+ if (my $net = $lei->{net}) {
+ $net->{shard_info} = [ $mod, $shard ];
+ } else { # Maildir (MH?)
+ $self->{shard_info} = [ $mod, $shard ];
+ }
+ $aug = '+'; # incr_post_augment
+ } elsif ($self->{-wq_worker_nr} == 0) {
+ $aug = '.'; # do_post_augment
+ }
+ if ($aug) {
local $0 = 'do_augment';
eval { do_augment($self, $lei) };
$lei->fail($@) if $@;
- pkt_do($lei->{pkt_op_p}, '.') == 1 or
- die "do_post_augment trigger: $!";
+ pkt_do($lei->{pkt_op_p}, $aug) == 1 or
+ die "do_post_augment trigger: $!";
}
if (my $zpipe = delete $lei->{zpipe}) {
$lei->{1} = $zpipe->[1];
close $zpipe->[0];
}
$self->{wcb} = $self->write_cb($lei);
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->lei_atfork_child;
+ $lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
$SIG{__WARN__} = 'DEFAULT';
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_complete {
+ my ($self) = @_;
+ $self->wq_broadcast('do_post_auth');
+ $self->wq_close(1);
+}
+
+no warnings 'once'; # the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
1;