1 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # Local storage (cache/memo) for lei(1), suitable for personal/private
5 # mail iff on encrypted device/FS. Based on v2, but only deduplicates
8 # for xref3, the following are constant: $eidx_key = '.', $xnum = -1
9 package PublicInbox::LeiStore;
12 use parent qw(PublicInbox::Lock PublicInbox::IPC);
13 use PublicInbox::ExtSearchIdx;
14 use PublicInbox::Import;
15 use PublicInbox::InboxWritable;
16 use PublicInbox::V2Writable;
17 use PublicInbox::ContentHash qw(content_hash content_digest);
18 use PublicInbox::MID qw(mids mids_in);
19 use PublicInbox::LeiSearch;
20 use List::Util qw(max);
23 my (undef, $dir, $opt) = @_;
24 my $eidx = PublicInbox::ExtSearchIdx->new($dir, $opt);
25 my $self = bless { priv_eidx => $eidx }, __PACKAGE__;
26 eidx_init($self)->done if $opt->{creat};
30 sub git { $_[0]->{priv_eidx}->git } # read-only
32 sub packing_factor { $PublicInbox::V2Writable::PACKING_FACTOR }
35 $_[0]->{rotate_bytes} // ((1024 * 1024 * 1024) / $_[0]->packing_factor)
38 sub git_pfx { "$_[0]->{priv_eidx}->{topdir}/local" };
42 if (opendir(my $dh, $self->git_pfx)) {
44 substr($_, 0, -4) + 0; # drop ".git" suffix
45 } grep(/\A[0-9]+\.git\z/, readdir($dh))) // 0;
47 $!{ENOENT} ? 0 : die("opendir ${\$self->git_pfx}: $!\n");
53 chomp(my $i = $git->qx(qw(var GIT_COMMITTER_IDENT)));
54 warn "$git->{git_dir} GIT_COMMITTER_IDENT failed\n" if $?;
55 $i =~ /\A(.+) <([^>]+)> [0-9]+ [-\+]?[0-9]+$/ ? ($1, $2) :
56 ('lei user', 'x@example.com')
64 return $im if $im->{bytes_added} < $self->rotate_bytes;
70 $max = $self->git_epoch_max + 1;
72 my $pfx = $self->git_pfx;
73 $max //= $self->git_epoch_max;
75 my $latest = "$pfx/$max.git";
77 PublicInbox::Import::init_bare($latest);
78 my $git = PublicInbox::Git->new($latest);
79 $git->qx(qw(config core.sharedRepository 0600)) if !$old;
80 my $packed_bytes = $git->packed_bytes;
81 my $unpacked_bytes = $packed_bytes / $self->packing_factor;
82 if ($unpacked_bytes >= $self->rotate_bytes) {
86 my ($n, $e) = git_ident($git);
87 $self->{im} = $im = PublicInbox::Import->new($git, $n, $e);
88 $im->{bytes_added} = int($packed_bytes / $self->packing_factor);
89 $im->{lock_path} = undef;
90 $im->{path_type} = 'v2';
96 PublicInbox::LeiSearch->new($_[0]->{priv_eidx}->{topdir});
101 my $eidx = $self->{priv_eidx};
102 $eidx->idx_init({-private => 1});
106 # when a message has no Message-IDs at all, this is needed for
107 # unsent Draft messages, at least
108 sub _fake_mid_for ($$) {
109 my ($eml, $dig) = @_;
110 my $mids = mids_in($eml, qw(X-Alt-Message-ID Resent-Message-ID));
111 $eml->{-lei_fake_mid} =
112 $mids->[0] // PublicInbox::Import::digest2mid($dig, $eml);
115 sub _docids_for ($$) {
116 my ($self, $eml) = @_;
118 my $dig = content_digest($eml);
119 my $chash = $dig->clone->digest;
120 my $eidx = eidx_init($self);
121 my $oidx = $eidx->{oidx};
122 my $im = $self->{im};
123 my $mids = mids($eml);
124 $mids->[0] //= _fake_mid_for($eml, $dig);
125 for my $mid (@$mids) {
127 while (my $cur = $oidx->next_by_mid($mid, \$id, \$prev)) {
128 my $oid = $cur->{blob};
129 my $docid = $cur->{num};
130 my $bref = $im ? $im->cat_blob($oid) : undef;
131 $bref //= $eidx->git->cat_file($oid) // do {
132 warn "W: $oid (#$docid) <$mid> not found\n";
135 local $self->{current_info} = $oid;
136 my $x = PublicInbox::Eml->new($bref);
137 $docids{$docid} = $docid if content_hash($x) eq $chash;
140 sort { $a <=> $b } values %docids;
143 sub set_eml_keywords {
144 my ($self, $eml, @kw) = @_;
145 my $eidx = eidx_init($self);
146 my @docids = _docids_for($self, $eml);
147 for my $docid (@docids) {
148 $eidx->idx_shard($docid)->ipc_do('set_keywords', $docid, @kw);
153 sub add_eml_keywords {
154 my ($self, $eml, @kw) = @_;
155 my $eidx = eidx_init($self);
156 my @docids = _docids_for($self, $eml);
157 for my $docid (@docids) {
158 $eidx->idx_shard($docid)->ipc_do('add_keywords', $docid, @kw);
163 sub remove_eml_keywords {
164 my ($self, $eml, @kw) = @_;
165 my $eidx = eidx_init($self);
166 my @docids = _docids_for($self, $eml);
167 for my $docid (@docids) {
168 $eidx->idx_shard($docid)->ipc_do('remove_keywords', $docid, @kw)
173 # cf: https://doc.dovecot.org/configuration_manual/mail_location/mbox/
174 my %status2kw = (F => 'flagged', A => 'answered', R => 'seen', T => 'draft');
175 # O (old/non-recent), and D (deleted) aren't in JMAP,
176 # so probably won't be supported by us.
179 my $s = "@{[$eml->header_raw('X-Status'),$eml->header_raw('Status')]}";
181 $s =~ s/([FART])/$kw{$status2kw{$1}} = 1/sge;
185 # cf: https://cr.yp.to/proto/maildir.html
186 my %c2kw = ('D' => 'draft', F => 'flagged', R => 'answered', S => 'seen');
187 sub maildir_keywords {
188 $_[-1] =~ /:2,([A-Z]+)\z/i ?
189 sort(map { $c2kw{$_} // () } split(//, $1)) : ();
193 my ($self, $eml, @kw) = @_;
194 my $eidx = eidx_init($self);
195 my $oidx = $eidx->{oidx};
196 my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
197 my $im = $self->importer;
198 $im->add($eml, undef, $smsg) or return; # duplicate returns undef
200 local $self->{current_info} = $smsg->{blob};
201 if (my @docids = _docids_for($self, $eml)) {
202 for my $docid (@docids) {
203 my $idx = $eidx->idx_shard($docid);
204 $oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
205 # add_eidx_info for List-Id
206 $idx->ipc_do('add_eidx_info', $docid, '.', $eml);
207 $idx->ipc_do('add_keywords', $docid, @kw) if @kw;
211 $smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
212 $oidx->add_overview($eml, $smsg);
213 $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
214 my $idx = $eidx->idx_shard($smsg->{num});
215 $idx->index_eml($eml, $smsg);
216 $idx->ipc_do('add_keywords', $smsg->{num}, @kw) if @kw;
222 my ($self, $eml, @kw) = @_;
223 add_eml($self, $eml, @kw) // set_eml_keywords($self, $eml, @kw);
229 if (my $im = delete($self->{im})) {
232 $err .= "import done: $@\n";
236 $self->{priv_eidx}->done;