use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
_lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
}
+sub lms_forget_folders {
+ my ($self, @folders) = @_;
+ my $lms = _lms_rw($self);
+ for my $f (@folders) { $lms->forget_folder($f) }
+}
+
+sub lms_rename_folder {
+ my ($self, $old, $new) = @_;
+ _lms_rw($self)->rename_folder($old, $new);
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
$self->{im}->remove($bref) if $bref;
}
+sub remove_docids ($;@) {
+ my ($self, @docids) = @_;
+ my $eidx = eidx_init($self);
+ for my $docid (@docids) {
+ $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+ $self->{oidx}->delete_by_num($docid);
+ $self->{oidx}->{dbh}->do(<<EOF, undef, $docid);
+DELETE FROM xref3 WHERE docid = ?
+EOF
+ }
+}
+
# remove the entire message from the index, does not touch mail_sync.sqlite3
sub remove_eml {
my ($self, $eml) = @_;
my $oidhex = unpack('H*', $oidbin);
$git->cat_async($oidhex, \&_remove_if_local, $self);
}
- $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $oidx->delete_by_num($docid);
}
$git->cat_async_wait;
+ remove_docids($self, @docids);
\@docids;
}
+sub oid2docid ($$) {
+ my ($self, $oid) = @_;
+ my $eidx = eidx_init($self);
+ my ($docid, @cull) = $eidx->{oidx}->blob_exists($oid);
+ if (@cull) { # fixup old bugs...
+ warn <<EOF;
+W: $oid indexed as multiple docids: $docid @cull, culling to fixup old bugs
+EOF
+ remove_docids($self, @cull);
+ }
+ $docid;
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
if (scalar keys %$xoids) {
my %docids = map { $_ => 1 } @$vivify_xvmd;
for my $oid (keys %$xoids) {
- my @id = $oidx->blob_exists($oid);
- @docids{@id} = @id;
+ my $docid = oid2docid($self, $oid);
+ $docids{$docid} = $docid if defined($docid);
}
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
}
}
\@docids;
} else { # totally new message
+ delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
my $oidx = $eidx->{oidx};
my %seen;
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('update_vmd', $docid, $vmd_mod);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid};
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('update_vmd', $docid, $vmd_mod);
}
return unless scalar(keys(%$xoids));
# see if we can just update existing docs
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('set_vmd', $docid, $vmd);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid}; # all done with this oid
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('set_vmd', $docid, $vmd);
}
return unless scalar(keys(%$xoids));
}
sub done {
- my ($self) = @_;
+ my ($self, $sock_ref) = @_;
my $err = '';
if (my $im = delete($self->{im})) {
eval { $im->done };
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
+
+ # notify clients ->done has been issued
+ defined($sock_ref) and
+ $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref);
}
sub ipc_atfork_child {
my $lei = $self->{lei};
$lei->_lei_atfork_child(1) if $lei;
xchg_stderr($self);
- if (my $err = delete($self->{err_pipe})) {
- close $err->[0];
- $self->{-err_wr} = $err->[1];
+ if (my $to_close = delete($self->{to_close})) {
+ close($_) for @$to_close;
}
$self->SUPER::ipc_atfork_child;
}
sub write_prepare {
my ($self, $lei) = @_;
+ $lei // die 'BUG: $lei not passed';
unless ($self->{-ipc_req}) {
- my $d = $lei->store_path;
- $self->ipc_lock_init("$d/ipc.lock");
- substr($d, -length('/lei/store'), 10, '');
+ # s2d => store-to-daemon messages
+ require PublicInbox::PktOp;
+ my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
+ my $dir = $lei->store_path;
+ $self->ipc_lock_init("$dir/ipc.lock");
+ substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
- my $err_pipe = [ $r, $w ];
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->ipc_worker_spawn("lei/store $d", $lei->oldset,
- { lei => $lei, err_pipe => $err_pipe });
+ $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, {
+ lei => $lei,
+ -err_wr => $w,
+ to_close => [ $r, $s2d_op_c->{sock} ],
+ s2d_op_p => $s2d_op_p,
+ });
require PublicInbox::LeiStoreErr;
- PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
+ PublicInbox::LeiStoreErr->new($r, $lei);
+ $s2d_op_c->{ops} = {
+ sto_done_complete => [ $lei->can('sto_done_complete') ]
+ };
}
$lei->{sto} = $self;
}
+# TODO: support SHA-256
+sub git_blob_id { # called via LEI->git_blob_id
+ my ($self, $eml) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml)->hexdigest;
+}
+
1;