use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Eml;
-use PublicInbox::Lock;
use PublicInbox::ProcessPipe;
-use PublicInbox::Spawn qw(which spawn popen_rd);
-use PublicInbox::LeiDedupe;
-use PublicInbox::Git;
-use PublicInbox::GitAsyncCat;
+use PublicInbox::Spawn qw(spawn);
use PublicInbox::PktOp qw(pkt_do);
use Symbol qw(gensym);
use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
-use Errno qw(EEXIST ESPIPE ENOENT EPIPE);
-use Digest::SHA qw(sha256_hex);
-
-# struggles with short-lived repos, Gcf2Client makes little sense with lei;
-# but we may use in-process libgit2 in the future.
-$PublicInbox::GitAsyncCat::GCF2C = 0;
my %kw2char = ( # Maildir characters
draft => 'D',
flagged => 'F',
+ forwarded => 'P', # passed
answered => 'R',
- seen => 'S'
+ seen => 'S',
);
my %kw2status = (
my ($eml, $type, $smsg) = @_;
$eml->header_set($_) for (qw(Lines Bytes Content-Length));
- # Messages are always 'O' (non-\Recent in IMAP), it saves
- # MUAs the trouble of rewriting the mbox if no other
- # changes are made
- my %hdr = (Status => [ 'O' ]); # set Status, X-Status
+ my %hdr = (Status => []); # set Status, X-Status
for my $k (@{$smsg->{kw} // []}) {
if (my $ent = $kw2status{$k}) {
push @{$hdr{$ent->[0]}}, $ent->[1];
warn "TODO: keyword `$k' not supported for mbox\n";
}
}
- while (my ($name, $chars) = each %hdr) {
- $eml->header_set($name, join('', sort @$chars));
+ # Messages are always 'O' (non-\Recent in IMAP), it saves
+ # MUAs the trouble of rewriting the mbox if no other
+ # changes are made. We put 'O' at the end (e.g. "Status: RO")
+ # to match mutt(1) output.
+ $eml->header_set('Status', join('', sort(@{$hdr{Status}})). 'O');
+ if (my $chars = delete $hdr{'X-Status'}) {
+ $eml->header_set('X-Status', join('', sort(@$chars)));
}
my $buf = delete $eml->{hdr};
if (defined(my $w = syswrite($lei->{1} // return, $$buf))) {
return if $w == length($$buf);
$buf = "short atomic write: $w != ".length($$buf);
- } elsif ($! == EPIPE) {
+ } elsif ($!{EPIPE}) {
return $lei->note_sigpipe(1);
} else {
$buf = "atomic write: $!";
sub git_to_mail { # git->cat_async callback
my ($bref, $oid, $type, $size, $arg) = @_;
- if ($type ne 'blob') {
- if ($type eq 'missing') {
- warn "missing $oid\n";
- } else {
- warn "unexpected type=$type for $oid\n";
- }
- }
+ return warn("W: $oid is $type (!= blob)\n") if $type ne 'blob';
+ return warn("E: $oid is empty\n") unless $size;
my ($write_cb, $smsg) = @$arg;
- if ($smsg->{blob} ne $oid) {
- die "BUG: expected=$smsg->{blob} got=$oid";
- }
- $write_cb->($bref, $smsg) if $size > 0;
+ die "BUG: expected=$smsg->{blob} got=$oid" if $smsg->{blob} ne $oid;
+ $write_cb->($bref, $smsg);
}
sub reap_compress { # dwaitpid callback
$lei->fail("@$cmd failed", $? >> 8);
}
-# all of these support -c for stdout and -d for decompression,
-# mutt is commonly distributed with hooks for gz, bz2 and xz, at least
-# { foo => '' } means "--foo" is passed to the command-line,
-# otherwise { foo => '--bar' } passes "--bar"
-our %zsfx2cmd = (
- gz => [ qw(GZIP pigz gzip), { rsyncable => '', threads => '-p' } ],
- bz2 => [ 'bzip2', {} ],
- xz => [ 'xz', { threads => '-T' } ],
- # XXX does anybody care for these? I prefer zstd on entire FSes,
- # so it's probably not necessary on a per-file basis
- # zst => [ 'zstd', { -default => [ qw(-q) ], # it's noisy by default
- # rsyncable => '', threads => '-T' } ],
- # zz => [ 'pigz', { -default => [ '--zlib' ],
- # rsyncable => '', threads => '-p' }],
- # lzo => [ 'lzop', {} ],
- # lzma => [ 'lzma', {} ],
-);
-
-sub zsfx2cmd ($$$) {
- my ($zsfx, $decompress, $lei) = @_;
- my $x = $zsfx2cmd{$zsfx} // die "no support for suffix=.$zsfx";
- my @info = @$x;
- my $cmd_opt = pop @info;
- my @cmd = (undef, $decompress ? qw(-dc) : qw(-c));
- for my $exe (@info) {
- # I think respecting client's ENV{GZIP} is OK, not sure
- # about ENV overrides for other, less-common compressors
- if ($exe eq uc($exe)) {
- $exe = $lei->{env}->{$exe} or next;
- }
- $cmd[0] = which($exe) and last;
- }
- $cmd[0] // die join(' or ', @info)." missing for .$zsfx";
- # push @cmd, @{$cmd_opt->{-default}} if $cmd_opt->{-default};
- for my $bool (qw(rsyncable)) {
- my $switch = $cmd_opt->{rsyncable} // next;
- push @cmd, '--'.($switch || $bool);
- }
- for my $key (qw(threads)) { # support compression level?
- my $switch = $cmd_opt->{$key} // next;
- my $val = $lei->{opt}->{$key} // next;
- push @cmd, $switch, $val;
- }
- \@cmd;
-}
-
-sub _post_augment_mbox { # open a compressor process
+sub _post_augment_mbox { # open a compressor process from top-level process
my ($self, $lei) = @_;
my $zsfx = $self->{zsfx} or return;
- my $cmd = zsfx2cmd($zsfx, undef, $lei);
+ my $cmd = PublicInbox::MboxReader::zsfx2cmd($zsfx, undef, $lei);
my ($r, $w) = @{delete $lei->{zpipe}};
my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
my $pid = spawn($cmd, undef, $rdr);
$dup->{$_} = $lei->{$_} for qw(2 sock);
tie *$pp, 'PublicInbox::ProcessPipe', $pid, $w, \&reap_compress, $dup;
$lei->{1} = $pp;
- die 'BUG: unexpected {ovv}->{lock_path}' if $lei->{ovv}->{lock_path};
- $lei->{ovv}->ovv_out_lk_init;
-}
-
-sub decompress_src ($$$) {
- my ($in, $zsfx, $lei) = @_;
- my $cmd = zsfx2cmd($zsfx, 1, $lei);
- popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
-}
-
-sub dup_src ($) {
- my ($in) = @_;
- open my $dup, '+>>&', $in or die "dup: $!";
- $dup;
}
# --augment existing output destination, with deduplication
$lei->{dedupe}->is_dup($eml);
}
+sub _mbox_augment_kw_maybe {
+ my ($eml, $lei, $lse, $augment) = @_;
+ my $kw = PublicInbox::MboxReader::mbox_keywords($eml);
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ _augment($eml, $lei) if $augment;
+}
+
sub _mbox_write_cb ($$) {
my ($self, $lei) = @_;
my $ovv = $lei->{ovv};
my $atomic_append = !defined($ovv->{lock_path});
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe;
+ my $lse = $lei->{lse}; # may be undef
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
+ $lse->xsmsg_vmd($smsg) if $lse;
$buf = $eml2mbox->($eml, $smsg);
return atomic_append($lei, $buf) if $atomic_append;
my $lk = $ovv->lock_for_scope;
}
}
-sub _augment_file { # maildir_each_eml cb
- my ($f, undef, $eml, $lei, $mod, $shard) = @_;
- if ($mod) {
- # can't get dirent.d_ino w/ pure Perl, so we extract the OID
- # if it looks like one:
- my $hex = $f =~ m!\b([a-f0-9]{40,})[^/]*\z! ?
- $1 : sha256_hex($f);
- my $recno = hex(substr($hex, 0, 8));
- return if ($recno % $mod) != $shard;
+sub update_kw_maybe ($$$$) {
+ my ($lei, $lse, $eml, $kw) = @_;
+ return unless $lse;
+ my $c = $lse->kw_changed($eml, $kw, my $docids = []);
+ my $vmd = { kw => $kw };
+ if (scalar @$docids) { # already in lei/store
+ $lei->{sto}->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ } elsif (my $xoids = $lei->{ale}->xoids_for($eml)) {
+ # it's in an external, only set kw, here
+ $lei->{sto}->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ } else { # never-before-seen, import the whole thing
+ # XXX this is critical in protecting against accidental
+ # data loss without --augment
+ $lei->{sto}->ipc_do('set_eml', $eml, $vmd);
}
- _augment($eml, $lei);
+}
+
+sub _md_update { # maildir_each_eml cb
+ my ($f, $kw, $eml, $lei, $lse, $unlink) = @_;
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ $unlink ? unlink($f) : _augment($eml, $lei);
}
# maildir_each_file callback, \&CORE::unlink doesn't work with it
do {
$tmp = $dst.'tmp/'.$rand.$common;
} while (!($ok = sysopen($fh, $tmp, O_CREAT|O_EXCL|O_WRONLY)) &&
- $! == EEXIST && ($rand = _rand.','));
+ $!{EEXIST} && ($rand = _rand.','));
if ($ok && print $fh $$buf and close($fh)) {
# ignore new/ and write only to cur/, otherwise MUAs
# with R/W access to the Maildir will end up doing
$rand = '';
do {
$final = $dst.$rand.$common.':2,'.$sfx;
- } while (!($ok = link($tmp, $final)) && $! == EEXIST &&
+ } while (!($ok = link($tmp, $final)) && $!{EEXIST} &&
($rand = _rand.','));
die "link($tmp, $final): $!" unless $ok;
unlink($tmp) or warn "W: failed to unlink $tmp: $!\n";
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
my $dst = $lei->{ovv}->{dst};
+ my $lse = $lei->{lse}; # may be undef
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
$buf //= \($eml->as_string);
+ $lse->xsmsg_vmd($smsg) if $lse;
return _buf2maildir($dst, $buf, $smsg) if !$dedupe;
$eml //= PublicInbox::Eml->new($$buf); # copy buf
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
undef $eml;
_buf2maildir($dst, $buf, $smsg);
}
my $imap_append = $lei->{net}->can('imap_append');
my $mic = $lei->{net}->mic_get($self->{uri});
my $folder = $self->{uri}->mailbox;
+ my $lse = $lei->{lse}; # may be undef
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
- $mic // return $lei->fail; # dst may be undef-ed in last run
+ $mic // return $lei->fail; # mic may be undef-ed in last run
if ($dedupe) {
$eml //= PublicInbox::Eml->new($$bref); # copy bref
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
}
+ $lse->xsmsg_vmd($smsg) if $lse;
eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
if (my $err = $@) {
undef $mic;
"$dst exists and is not a directory\n";
$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
} elsif (substr($fmt, 0, 4) eq 'mbox') {
- require PublicInbox::MboxReader if $lei->{opt}->{augment};
+ require PublicInbox::MboxReader;
(-d $dst || (-e _ && !-w _)) and die
"$dst exists and is not a writable file\n";
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
die "bad mail --format=$fmt\n";
}
$self->{dst} = $dst;
- $lei->{dedupe} = PublicInbox::LeiDedupe->new($lei);
+ $lei->{dedupe} = $lei->{lss} // do {
+ my $dd_cls = 'PublicInbox::'.
+ ($lei->{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe');
+ eval "require $dd_cls";
+ die "$dd_cls: $@" if $@;
+ $dd_cls->new($lei);
+ };
$self;
}
sub _do_augment_maildir {
my ($self, $lei) = @_;
+ return if defined($lei->{opt}->{save});
my $dst = $lei->{ovv}->{dst};
+ my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
+ my $mdr = PublicInbox::MdirReader->new;
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- my ($mod, $shard) = @{$self->{shard_info} // []};
- PublicInbox::MdirReader::maildir_each_eml($dst,
- \&_augment_file,
- $lei, $mod, $shard);
+ $mdr->{shard_info} = $self->{shard_info};
+ $mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse);
$dedupe->pause_dedupe;
}
- } else { # clobber existing Maildir
- PublicInbox::MdirReader::maildir_each_file($dst, \&_unlink);
+ } elsif ($lse) {
+ $mdr->{shard_info} = $self->{shard_info};
+ $mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse, 1);
+ } else {# clobber existing Maildir
+ $mdr->maildir_each_file($dst, \&_unlink);
}
}
-sub _augment_imap { # PublicInbox::NetReader::imap_each cb
- my ($url, $uid, $kw, $eml, $lei) = @_;
- _augment($eml, $lei);
+sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
+ my ($url, $uid, $kw, $eml, $lei, $lse, $delete_mic) = @_;
+ update_kw_maybe($lei, $lse, $eml, $kw);
+ if ($delete_mic) {
+ $lei->{net}->imap_delete_1($url, $uid, $delete_mic);
+ } else {
+ _augment($eml, $lei);
+ }
}
sub _do_augment_imap {
my ($self, $lei) = @_;
+ return if defined($lei->{opt}->{save});
my $net = $lei->{net};
+ my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
if ($lei->{opt}->{augment}) {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
- $net->imap_each($self->{uri}, \&_augment_imap, $lei);
+ $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+ $lei, $lse);
$dedupe->pause_dedupe;
}
+ } elsif ($lse) {
+ my $delete_mic;
+ $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+ $lei, $lse, \$delete_mic);
+ $delete_mic->expunge if $delete_mic;
} elsif (!$self->{-wq_worker_nr}) { # undef or 0
# clobber existing IMAP folder
$net->imap_delete_all($self->{uri});
sub _pre_augment_mbox {
my ($self, $lei) = @_;
my $dst = $lei->{ovv}->{dst};
- if ($dst ne '/dev/stdout') {
- my $out;
+ my $out;
+ my $devfd = $lei->path_to_fd($dst) // die "bad $dst";
+ if ($devfd >= 0) {
+ $out = $lei->{$devfd};
+ } else { # normal-looking path
if (-p $dst) {
open $out, '>', $dst or die "open($dst): $!";
} elsif (-f _ || !-e _) {
PublicInbox::MboxLock->defaults;
$self->{mbl} = PublicInbox::MboxLock->acq($dst, 1, $m);
$out = $self->{mbl}->{fh};
- if (!$lei->{opt}->{augment} and !truncate($out, 0)) {
- die "truncate($dst): $!";
- }
+ } else {
+ die "$dst is not a file or FIFO\n";
}
$lei->{old_1} = $lei->{1}; # keep for spawning MUA
- $lei->{1} = $out;
}
# Perl does SEEK_END even with O_APPEND :<
- $self->{seekable} = seek($lei->{1}, 0, SEEK_SET);
- if (!$self->{seekable} && $! != ESPIPE && $dst ne '/dev/stdout') {
+ $self->{seekable} = seek($out, 0, SEEK_SET);
+ if (!$self->{seekable} && !$!{ESPIPE} && !defined($devfd)) {
die "seek($dst): $!\n";
}
- state $zsfx_allow = join('|', keys %zsfx2cmd);
- ($self->{zsfx}) = ($dst =~ /\.($zsfx_allow)\z/) or return;
- pipe(my ($r, $w)) or die "pipe: $!";
- $lei->{zpipe} = [ $r, $w ];
+ if (!$self->{seekable}) {
+ my $imp_before = $lei->{opt}->{'import-before'};
+ die "--import-before specified but $dst is not seekable\n"
+ if $imp_before && !ref($imp_before);
+ die "--augment specified but $dst is not seekable\n" if
+ $lei->{opt}->{augment};
+ }
+ if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) {
+ pipe(my ($r, $w)) or die "pipe: $!";
+ $lei->{zpipe} = [ $r, $w ];
+ $lei->{ovv}->{lock_path} and
+ die 'BUG: unexpected {ovv}->{lock_path}';
+ $lei->{ovv}->ovv_out_lk_init;
+ } elsif (!$self->{seekable} && !$lei->{ovv}->{lock_path}) {
+ $lei->{ovv}->ovv_out_lk_init;
+ }
+ $lei->{1} = $out;
+ undef;
}
sub _do_augment_mbox {
my ($self, $lei) = @_;
- return if !$lei->{opt}->{augment};
- my $dedupe = $lei->{dedupe};
- my $dst = $lei->{ovv}->{dst};
- die "cannot augment $dst, not seekable\n" if !$self->{seekable};
+ return unless $self->{seekable};
+ my $opt = $lei->{opt};
+ return if defined($opt->{save});
my $out = $lei->{1};
- if (-s $out && $dedupe && $dedupe->prepare_dedupe) {
- my $zsfx = $self->{zsfx};
- my $rd = $zsfx ? decompress_src($out, $zsfx, $lei) :
- dup_src($out);
- my $fmt = $lei->{ovv}->{fmt};
+ my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)};
+ return unless -s $out;
+ unless ($opt->{augment} || $opt->{'import-before'}) {
+ truncate($out, 0) or die "truncate($dst): $!";
+ return;
+ }
+ my $rd;
+ if (my $zsfx = $self->{zsfx}) {
+ $rd = PublicInbox::MboxReader::zsfxcat($out, $zsfx, $lei);
+ } else {
+ open($rd, '+>>&', $out) or die "dup: $!";
+ }
+ my $dedupe;
+ if ($opt->{augment}) {
+ $dedupe = $lei->{dedupe};
+ $dedupe->prepare_dedupe if $dedupe;
+ }
+ if ($opt->{'import-before'}) { # the default
+ my $lse = $lei->{lse};
+ PublicInbox::MboxReader->$fmt($rd, \&_mbox_augment_kw_maybe,
+ $lei, $lse, $opt->{augment});
+ if (!$opt->{augment} and !truncate($out, 0)) {
+ die "truncate($dst): $!";
+ }
+ } else { # --augment --no-import-before
PublicInbox::MboxReader->$fmt($rd, \&_augment, $lei);
}
# maybe some systems don't honor O_APPEND, Perl does this:
# fast (spawn compressor or mkdir), runs in same process as pre_augment
sub post_augment {
my ($self, $lei, @args) = @_;
+ my $wait = $lei->{opt}->{'import-before'} ?
+ $lei->{sto}->ipc_do('checkpoint', 1) : 0;
# _post_augment_mbox
my $m = $self->can("_post_augment_$self->{base_type}") or return;
$m->($self, $lei, @args);
}
+# called by every single l2m worker process
sub do_post_auth {
my ($self) = @_;
my $lei = $self->{lei};
pkt_do($lei->{pkt_op_p}, 'incr_start_query') or
die "incr_start_query: $!";
my $aug;
- if (lock_free($self)) {
+ if (lock_free($self)) { # all workers do_augment
my $mod = $self->{-wq_nr_workers};
my $shard = $self->{-wq_worker_nr};
if (my $net = $lei->{net}) {
$net->{shard_info} = [ $mod, $shard ];
- } else { # Maildir (MH?)
+ } else { # Maildir
$self->{shard_info} = [ $mod, $shard ];
}
$aug = '+'; # incr_post_augment
- } elsif ($self->{-wq_worker_nr} == 0) {
+ } elsif ($self->{-wq_worker_nr} == 0) { # 1st worker do_augment
$aug = '.'; # do_post_augment
}
if ($aug) {
pkt_do($lei->{pkt_op_p}, $aug) == 1 or
die "do_post_augment trigger: $!";
}
+ # done augmenting, connect the compressor pipe for each worker
if (my $zpipe = delete $lei->{zpipe}) {
$lei->{1} = $zpipe->[1];
close $zpipe->[0];
sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
- $lei->lei_atfork_child;
+ $lei->_lei_atfork_child;
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
sub lock_free {
- $_[0]->{base_type} =~ /\A(?:maildir|mh|imap|jmap)\z/ ? 1 : 0;
+ $_[0]->{base_type} =~ /\A(?:maildir|imap|jmap)\z/ ? 1 : 0;
}
+# wakes up the MUA when complete so it can refresh messages list
sub poke_dst {
my ($self) = @_;
if ($self->{base_type} eq 'maildir') {
my $t = time + 1;
- utime($t, $t, "$self->{dst}/cur");
+ utime($t, $t, $self->{dst} . 'cur');
}
}
sub write_mail { # via ->wq_io_do
- my ($self, $git_dir, $smsg) = @_;
- my $git = $self->{"$$\0$git_dir"} //= PublicInbox::Git->new($git_dir);
- git_async_cat($git, $smsg->{blob}, \&git_to_mail,
+ my ($self, $smsg, $eml) = @_;
+ return $self->{wcb}->(undef, $smsg, $eml) if $eml;
+ $self->{lei}->{ale}->git->cat_async($smsg->{blob}, \&git_to_mail,
[$self->{wcb}, $smsg]);
}
sub wq_atexit_child {
my ($self) = @_;
delete $self->{wcb};
- for my $git (delete @$self{grep(/\A$$\0/, keys %$self)}) {
- $git->async_wait_all;
- }
- $SIG{__WARN__} = 'DEFAULT';
+ $self->{lei}->{ale}->git->async_wait_all;
}
# called in top-level lei-daemon when LeiAuth is done