X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiStore.pm;h=c1abc288bfcb46f539f0a689fac33c53030aa065;hb=7d2e572aca7297ea2015d2b6e7c71b672521ec82;hp=3a215973a4f369ab2776a1f1fab64822c0551d0e;hpb=e3c82310f5df24a42a0dc3c08b97b409a185b3a3;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 3a215973..c1abc288 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -12,10 +12,10 @@ use v5.10.1;
 use parent qw(PublicInbox::Lock PublicInbox::IPC);
 use PublicInbox::ExtSearchIdx;
 use PublicInbox::Import;
-use PublicInbox::InboxWritable;
+use PublicInbox::InboxWritable qw(eml_from_path);
 use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash content_digest);
-use PublicInbox::MID qw(mids mids_in);
+use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::MID qw(mids);
 use PublicInbox::LeiSearch;
 use PublicInbox::MDA;
 use List::Util qw(max);
@@ -104,25 +104,13 @@ sub eidx_init {
 	$eidx;
 }
 
-# when a message has no Message-IDs at all, this is needed for
-# unsent Draft messages, at least
-sub _fake_mid_for ($$) {
-	my ($eml, $dig) = @_;
-	my $mids = mids_in($eml, qw(X-Alt-Message-ID Resent-Message-ID));
-	$eml->{-lei_fake_mid} =
-		$mids->[0] // PublicInbox::Import::digest2mid($dig, $eml);
-}
-
 sub _docids_for ($$) {
 	my ($self, $eml) = @_;
 	my %docids;
-	my $dig = content_digest($eml);
-	my $chash = $dig->clone->digest;
+	my ($chash, $mids) = PublicInbox::LeiSearch::content_key($eml);
 	my $eidx = eidx_init($self);
 	my $oidx = $eidx->{oidx};
 	my $im = $self->{im};
-	my $mids = mids($eml);
-	$mids->[0] //= _fake_mid_for($eml, $dig);
 	for my $mid (@$mids) {
 		my ($id, $prev);
 		while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) {
@@ -141,61 +129,42 @@ sub _docids_for ($$) {
 	sort { $a <=> $b } values %docids;
 }
 
-sub set_eml_keywords {
-	my ($self, $eml, @kw) = @_;
+sub set_eml_vmd {
+	my ($self, $eml, $vmd) = @_;
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->ipc_do('set_keywords', $docid, @kw);
+		$eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd);
 	}
 	\@docids;
 }
 
-sub add_eml_keywords {
-	my ($self, $eml, @kw) = @_;
+sub add_eml_vmd {
+	my ($self, $eml, $vmd) = @_;
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->ipc_do('add_keywords', $docid, @kw);
+		$eidx->idx_shard($docid)->ipc_do('add_vmd', $docid, $vmd);
 	}
 	\@docids;
 }
 
-sub remove_eml_keywords {
-	my ($self, $eml, @kw) = @_;
+sub remove_eml_vmd {
+	my ($self, $eml, $vmd) = @_;
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->ipc_do('remove_keywords', $docid, @kw)
+		$eidx->idx_shard($docid)->ipc_do('remove_vmd', $docid, $vmd);
 	}
 	\@docids;
 }
 
-# cf: https://doc.dovecot.org/configuration_manual/mail_location/mbox/
-my %status2kw = (F => 'flagged', A => 'answered', R => 'seen', T => 'draft');
-# O (old/non-recent), and D (deleted) aren't in JMAP,
-# so probably won't be supported by us.
-sub mbox_keywords {
-	my $eml = $_[-1];
-	my $s = "@{[$eml->header_raw('X-Status'),$eml->header_raw('Status')]}";
-	my %kw;
-	$s =~ s/([FART])/$kw{$status2kw{$1}} = 1/sge;
-	sort(keys %kw);
-}
-
-# cf: https://cr.yp.to/proto/maildir.html
-my %c2kw = ('D' => 'draft', F => 'flagged', R => 'answered', S => 'seen');
-sub maildir_keywords {
-	$_[-1] =~ /:2,([A-Z]+)\z/i ?
-		sort(map { $c2kw{$_} // () } split(//, $1)) : ();
-}
-
 sub add_eml {
-	my ($self, $eml, @kw) = @_;
-	my $eidx = eidx_init($self);
+	my ($self, $eml, $vmd) = @_;
+	my $im = $self->importer; # may create new epoch
+	my $eidx = eidx_init($self); # writes ALL.git/objects/info/alternates
 	my $oidx = $eidx->{oidx};
 	my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
-	my $im = $self->importer;
 	$im->add($eml, undef, $smsg) or return; # duplicate returns undef
 
 	local $self->{current_info} = $smsg->{blob};
@@ -205,7 +174,7 @@ sub add_eml {
 			$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
 			# add_eidx_info for List-Id
 			$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
-			$idx->ipc_do('add_keywords', $docid, @kw) if @kw;
+			$idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
 		}
 		\@docids;
 	} else {
@@ -214,14 +183,40 @@ sub add_eml {
 		$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
 		my $idx = $eidx->idx_shard($smsg->{num});
 		$idx->index_eml($eml, $smsg);
-		$idx->ipc_do('add_keywords', $smsg->{num}, @kw) if @kw;
+		$idx->ipc_do('add_vmd', $smsg->{num}, $vmd ) if $vmd;
 		$smsg;
 	}
 }
 
 sub set_eml {
-	my ($self, $eml, @kw) = @_;
-	add_eml($self, $eml, @kw) // set_eml_keywords($self, $eml, @kw);
+	my ($self, $eml, $vmd) = @_;
+	add_eml($self, $eml, $vmd) // set_eml_vmd($self, $eml, $vmd);
+}
+
+sub add_eml_maybe {
+	my ($self, $eml) = @_;
+	my $lxs = $self->{lxs_all_local} // die 'BUG: no {lxs_all_local}';
+	return if $lxs->xids_for($eml, 1);
+	add_eml($self, $eml);
+}
+
+# set or update keywords for external message, called via ipc_do
+sub set_xkw {
+	my ($self, $eml, $kw) = @_;
+	my $lxs = $self->{lxs_all_local} // die 'BUG: no {lxs_all_local}';
+	if ($lxs->xids_for($eml, 1)) { # is it in a local external?
+		# TODO: index keywords only
+	} else {
+		set_eml($self, $eml, { kw => $kw });
+	}
+}
+
+sub checkpoint {
+	my ($self, $wait) = @_;
+	if (my $im = $self->{im}) {
+		$wait ? $im->barrier : $im->checkpoint;
+	}
+	$self->{priv_eidx}->checkpoint($wait);
 }
 
 sub done {
@@ -240,18 +235,41 @@ sub done {
 
 sub ipc_atfork_child {
 	my ($self) = @_;
-	my $lei = delete $self->{lei};
+	my $lei = $self->{lei};
 	$lei->lei_atfork_child(1) if $lei;
 	$self->SUPER::ipc_atfork_child;
 }
 
+sub refresh_local_externals {
+	my ($self) = @_;
+	my $cfg = $self->{lei}->_lei_cfg or return;
+	my $cur_cfg = $self->{cur_cfg} // -1;
+	my $lxs = $self->{lxs_all_local};
+	if ($cfg != $cur_cfg || !$lxs) {
+		$lxs = PublicInbox::LeiXSearch->new;
+		my @loc = $self->{lei}->externals_each;
+		for my $loc (@loc) { # locals only
+			$lxs->prepare_external($loc) if -d $loc;
+		}
+		$self->{lei}->ale->refresh_externals($lxs);
+		$lxs->{git} = $self->{lei}->ale->git;
+		$self->{lxs_all_local} = $lxs;
+		$self->{cur_cfg} = $cfg;
+	}
+}
+
 sub write_prepare {
 	my ($self, $lei) = @_;
-	$self->ipc_lock_init;
-	# Mail we import into lei are private, so headers filtered out
-	# by -mda for public mail are not appropriate
-	local @PublicInbox::MDA::BAD_HEADERS = ();
-	$self->ipc_worker_spawn('lei_store', $lei->oldset, { lei => $lei });
+	unless ($self->{-ipc_req}) {
+		require PublicInbox::LeiXSearch;
+		$self->ipc_lock_init($lei->store_path . '/ipc.lock');
+		# Mail we import into lei are private, so headers filtered out
+		# by -mda for public mail are not appropriate
+		local @PublicInbox::MDA::BAD_HEADERS = ();
+		$self->ipc_worker_spawn('lei_store', $lei->oldset,
+					{ lei => $lei });
+	}
+	my $wait = $self->ipc_do('refresh_local_externals');
 	$lei->{sto} = $self;
 }