-# returns begin and end PostingIterator
-sub find_doc_ids {
- my ($self, $termval) = @_;
- my $db = $self->{xdb};
-
- ($db->postlist_begin($termval), $db->postlist_end($termval));
-}
-
-# v1 only
-sub batch_do {
- my ($self, $termval, $cb) = @_;
- my $batch_size = 1000; # don't let @ids grow too large to avoid OOM
- while (1) {
- my ($head, $tail) = $self->find_doc_ids($termval);
- return if $head == $tail;
- my @ids;
- for (; $head != $tail && @ids < $batch_size; $head++) {
- push @ids, $head->get_docid;
- }
- $cb->(\@ids);
- }
-}
-
-# v1 only, where $mid is unique
-sub remove_message {
- my ($self, $mid) = @_;
- $mid = mid_clean($mid);
-
- if (my $over = $self->{over}) {
- my $nr = eval { $over->remove_oid(undef, $mid) };
- if ($@) {
- warn "failed to remove <$mid> from overview: $@\n";
- } elsif ($nr == 0) {
- warn "<$mid> missing for removal from overview\n";
- }
- }
- return unless need_xapian($self);
- my $db = $self->{xdb};
- my $nr = 0;
- eval {
- batch_do($self, 'Q' . $mid, sub {
- my ($ids) = @_;
- $db->delete_document($_) for @$ids;
- $nr += scalar @$ids;
- });
- };
- if ($@) {
- warn "failed to remove <$mid> from Xapian: $@\n";
- } elsif ($nr == 0) {
- warn "<$mid> missing for removal from Xapian\n";
- }
-}
-
-# MID is a hint in V2
-sub remove_by_oid {
- my ($self, $oid, $mid) = @_;
-
- $self->{over}->remove_oid($oid, $mid) if $self->{over};
-
- return unless need_xapian($self);
- my $db = $self->{xdb};
-
- # XXX careful, we cannot use batch_do here since we conditionally
- # delete documents based on other factors, so we cannot call
- # find_doc_ids twice.
- my ($head, $tail) = $self->find_doc_ids('Q' . $mid);
- return if $head == $tail;
-
- # there is only ONE element in @delete unless we
- # have bugs in our v2writable deduplication check
- my @delete;
- for (; $head != $tail; $head++) {
- my $docid = $head->get_docid;
- my $doc = $db->get_document($docid);
- my $smsg = PublicInbox::Smsg->wrap($mid);
- $smsg->load_expand($doc);
- if ($smsg->{blob} eq $oid) {
- push(@delete, $docid);
- }
- }
- $db->delete_document($_) foreach @delete;
- scalar(@delete);
+sub _get_doc ($$) {
+ my ($self, $docid) = @_;
+ my $doc = eval { $self->{xdb}->get_document($docid) };
+ $doc // do {
+ warn "E: $@\n" if $@;
+ warn "E: #$docid missing in Xapian\n";
+ undef;
+ }
+}
+
+sub add_eidx_info {
+ my ($self, $docid, $eidx_key, $eml) = @_;
+ begin_txn_lazy($self);
+ my $doc = _get_doc($self, $docid) or return;
+ term_generator($self)->set_document($doc);
+ $doc->add_boolean_term('O'.$eidx_key);
+ index_list_id($self, $doc, $eml);
+ $self->{xdb}->replace_document($docid, $doc);
+}
+
+sub remove_eidx_info {
+ my ($self, $docid, $eidx_key, $eml) = @_;
+ begin_txn_lazy($self);
+ my $doc = _get_doc($self, $docid) or return;
+ eval { $doc->remove_term('O'.$eidx_key) };
+ warn "W: ->remove_term O$eidx_key: $@\n" if $@;
+ for my $l ($eml ? $eml->header_raw('List-Id') : ()) {
+ $l =~ /<([^>]+)>/ or next;
+ my $lid = lc $1;
+ eval { $doc->remove_term('G' . $lid) };
+ warn "W: ->remove_term G$lid: $@\n" if $@;
+
+ # nb: we don't remove the XL probabilistic terms
+ # since terms may overlap if cross-posted.
+ #
+ # IOW, a message which has both <foo.example.com>
+ # and <bar.example.com> would have overlapping
+ # "XLexample" and "XLcom" as terms and which we
+ # wouldn't know if they're safe to remove if we just
+ # unindex <foo.example.com> while preserving
+ # <bar.example.com>.
+ #
+ # In any case, this entire sub is will likely never
+ # be needed and users using the "l:" prefix are probably
+ # rarer.
+ }
+ $self->{xdb}->replace_document($docid, $doc);
+}
+
+sub get_val ($$) {
+ my ($doc, $col) = @_;
+ sortable_unserialise($doc->get_value($col));
+}
+
+sub smsg_from_doc ($) {
+ my ($doc) = @_;
+ my $data = $doc->get_data or return;
+ my $smsg = bless {}, 'PublicInbox::Smsg';
+ $smsg->{ts} = get_val($doc, PublicInbox::Search::TS());
+ my $dt = get_val($doc, PublicInbox::Search::DT());
+ my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt);
+ $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy);
+ $smsg->load_from_data($data);
+ $smsg;
+}
+
+sub xdb_remove {
+ my ($self, @docids) = @_;
+ my $xdb = $self->{xdb} or return;
+ for my $docid (@docids) {
+ eval { $xdb->delete_document($docid) };
+ warn "E: #$docid not in in Xapian? $@\n" if $@;
+ }
+}
+
+sub remove_by_docid {
+ my ($self, $num) = @_;
+ die "BUG: remove_by_docid is v2-only\n" if $self->{oidx};
+ $self->begin_txn_lazy;
+ xdb_remove($self, $num) if need_xapian($self);