From 2b18ef9ca28e8c4e70a881dbad832d2f7cb838ee Mon Sep 17 00:00:00 2001
From: "Eric W. Biederman" <ebiederm@xmission.com>
Date: Wed, 18 Jul 2018 11:53:24 -0500
Subject: [PATCH] SearchIdx: Add the mechanism for making all Xapian indexing
optional
Create a new method add_xapian that holds all of the code to create
Xapian indexes. The creation of this method simpliy involved
idenitifying the relevant code and moving it from add_message.
A call is added to add_xapian from add_message to keep everything
working as it currently does. The new call is made conditional upon
index levels of 'full' and 'medium'. The index levels that index
positions and terms the two things public-inbox uses Xapian to index.
Signed-off-by: "Eric W. Biederman" <ebiederm@xmission.com>
---
lib/PublicInbox/SearchIdx.pm | 172 ++++++++++++++++++-----------------
1 file changed, 89 insertions(+), 83 deletions(-)
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index b19618c7..8978914a 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -268,10 +268,95 @@ sub index_body ($$$) {
@$lines = ();
}
+sub add_xapian ($$$$$) {
+ my ($self, $mime, $num, $oid, $mids, $mid0) = @_;
+ my $smsg = PublicInbox::SearchMsg->new($mime);
+ my $doc = $smsg->{doc};
+ my $subj = $smsg->subject;
+ add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
+ my @ds = gmtime($smsg->ds);
+ my $yyyymmdd = strftime('%Y%m%d', @ds);
+ add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
+ my $dt = strftime('%Y%m%d%H%M%S', @ds);
+ add_val($doc, PublicInbox::Search::DT(), $dt);
+
+ my $tg = $self->term_generator;
+
+ $tg->set_document($doc);
+ $self->index_text($subj, 1, 'S') if $subj;
+ $self->index_users($smsg);
+
+ msg_iter($mime, sub {
+ my ($part, $depth, @idx) = @{$_[0]};
+ my $ct = $part->content_type || 'text/plain';
+ my $fn = $part->filename;
+ if (defined $fn && $fn ne '') {
+ $self->index_text($fn, 1, 'XFN');
+ }
+
+ return if $ct =~ m!\btext/x?html\b!i;
+
+ my $s = eval { $part->body_str };
+ if ($@) {
+ if ($ct =~ m!\btext/plain\b!i) {
+ # Try to assume UTF-8 because Alpine
+ # seems to do wacky things and set
+ # charset=X-UNKNOWN
+ $part->charset_set('UTF-8');
+ $s = eval { $part->body_str };
+ $s = $part->body if $@;
+ }
+ }
+ defined $s or return;
+
+ my (@orig, @quot);
+ my $body = $part->body;
+ my @lines = split(/\n/, $body);
+ while (defined(my $l = shift @lines)) {
+ if ($l =~ /^>/) {
+ $self->index_body(\@orig, $doc) if @orig;
+ push @quot, $l;
+ } else {
+ $self->index_body(\@quot, 0) if @quot;
+ push @orig, $l;
+ }
+ }
+ $self->index_body(\@quot, 0) if @quot;
+ $self->index_body(\@orig, $doc) if @orig;
+ });
+
+ foreach my $mid (@$mids) {
+ $self->index_text($mid, 1, 'XM');
+
+ # because too many Message-IDs are prefixed with
+ # "Pine.LNX."...
+ if ($mid =~ /\w{12,}/) {
+ my @long = ($mid =~ /(\w{3,}+)/g);
+ $self->index_text(join(' ', @long), 1, 'XM');
+ }
+ }
+ $smsg->{to} = $smsg->{cc} = '';
+ PublicInbox::OverIdx::parse_references($smsg, $mid0, $mids);
+ my $data = $smsg->to_doc_data($oid, $mid0);
+ $doc->set_data($data);
+ if (my $altid = $self->{-altid}) {
+ foreach my $alt (@$altid) {
+ my $pfx = $alt->{xprefix};
+ foreach my $mid (@$mids) {
+ my $id = $alt->mid2alt($mid);
+ next unless defined $id;
+ $doc->add_boolean_term($pfx . $id);
+ }
+ }
+ }
+ $doc->add_boolean_term('Q' . $_) foreach @$mids;
+ $self->{xdb}->replace_document($num, $doc);
+}
+
sub add_message {
# mime = Email::MIME object
my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
- my $doc_id;
+ my $xapianlevels = qr/\A(?:full|medium)\z/;
my $mids = mids($mime->header_obj);
$mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
unless (defined $num) { # v1
@@ -279,98 +364,19 @@ sub add_message {
$num = index_mm($self, $mime);
}
eval {
- my $smsg = PublicInbox::SearchMsg->new($mime);
- my $doc = $smsg->{doc};
- my $subj = $smsg->subject;
- add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
- my @ds = gmtime($smsg->ds);
- my $yyyymmdd = strftime('%Y%m%d', @ds);
- add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
- my $dt = strftime('%Y%m%d%H%M%S', @ds);
- add_val($doc, PublicInbox::Search::DT(), $dt);
-
- my $tg = $self->term_generator;
-
- $tg->set_document($doc);
- $self->index_text($subj, 1, 'S') if $subj;
- $self->index_users($smsg);
-
- msg_iter($mime, sub {
- my ($part, $depth, @idx) = @{$_[0]};
- my $ct = $part->content_type || 'text/plain';
- my $fn = $part->filename;
- if (defined $fn && $fn ne '') {
- $self->index_text($fn, 1, 'XFN');
- }
-
- return if $ct =~ m!\btext/x?html\b!i;
-
- my $s = eval { $part->body_str };
- if ($@) {
- if ($ct =~ m!\btext/plain\b!i) {
- # Try to assume UTF-8 because Alpine
- # seems to do wacky things and set
- # charset=X-UNKNOWN
- $part->charset_set('UTF-8');
- $s = eval { $part->body_str };
- $s = $part->body if $@;
- }
- }
- defined $s or return;
-
- my (@orig, @quot);
- my $body = $part->body;
- my @lines = split(/\n/, $body);
- while (defined(my $l = shift @lines)) {
- if ($l =~ /^>/) {
- $self->index_body(\@orig, $doc) if @orig;
- push @quot, $l;
- } else {
- $self->index_body(\@quot, 0) if @quot;
- push @orig, $l;
- }
- }
- $self->index_body(\@quot, 0) if @quot;
- $self->index_body(\@orig, $doc) if @orig;
- });
-
- foreach my $mid (@$mids) {
- $self->index_text($mid, 1, 'XM');
-
- # because too many Message-IDs are prefixed with
- # "Pine.LNX."...
- if ($mid =~ /\w{12,}/) {
- my @long = ($mid =~ /(\w{3,}+)/g);
- $self->index_text(join(' ', @long), 1, 'XM');
- }
+ if ($self->{indexlevel} =~ $xapianlevels) {
+ $self->add_xapian($mime, $num, $oid, $mids, $mid0)
}
- $smsg->{to} = $smsg->{cc} = '';
- PublicInbox::OverIdx::parse_references($smsg, $mid0, $mids);
- my $data = $smsg->to_doc_data($oid, $mid0);
- $doc->set_data($data);
- if (my $altid = $self->{-altid}) {
- foreach my $alt (@$altid) {
- my $pfx = $alt->{xprefix};
- foreach my $mid (@$mids) {
- my $id = $alt->mid2alt($mid);
- next unless defined $id;
- $doc->add_boolean_term($pfx . $id);
- }
- }
- }
-
if (my $over = $self->{over}) {
$over->add_overview($mime, $bytes, $num, $oid, $mid0);
}
- $doc->add_boolean_term('Q' . $_) foreach @$mids;
- $self->{xdb}->replace_document($doc_id = $num, $doc);
};
if ($@) {
warn "failed to index message <".join('> <',@$mids).">: $@\n";
return undef;
}
- $doc_id;
+ $num;
}
# returns begin and end PostingIterator
--
2.50.0