*idx: pass $smsg in more places instead of many args

We can pass blessed PublicInbox::Smsg objects to internal
indexing APIs instead of having long parameter lists in some
places.  The end goal is to avoid parsing redundant information
each step of the way and hopefully make things more
understandable.

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index f49dfa000431af739eb5e82060c5596fe1af5baf..2d71956d592938bc3f8a9df17e09671b29893a6b 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -245,15 +245,9 @@ sub subject_path ($) {
 }
 
 sub add_overview {
-       my ($self, $mime, $bytes, $num, $oid, $mid0, $times) = @_;
-       my $lines = $mime->body_raw =~ tr!\n!\n!;
-       my $smsg = bless {
-               mime => $mime,
-               mid => $mid0,
-               bytes => $bytes,
-               lines => $lines,
-               blob => $oid,
-       }, 'PublicInbox::Smsg';
+       my ($self, $mime, $smsg, $times) = @_;
+       $smsg->{lines} = $mime->body_raw =~ tr!\n!\n!;
+       $smsg->{mime} = $mime; # XXX temporary?
        my $hdr = $mime->header_obj;
        my $mids = mids_for_index($hdr);
        my $refs = parse_references($smsg, $hdr, $mids);
@@ -268,7 +262,7 @@ sub add_overview {
        $dd = compress($dd);
        my $ds = msg_timestamp($hdr, $times->{autime});
        my $ts = msg_datestamp($hdr, $times->{cotime});
-       my $values = [ $ts, $ds, $num, $mids, $refs, $xpath, $dd ];
+       my $values = [ $ts, $ds, $smsg->{num}, $mids, $refs, $xpath, $dd ];
        add_over($self, $values);
 }
 

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 32be9c3ffbb63963f64c1958038f9b4c1d6899ba..5ca819c383525de90c72eae9bc5c3f31efea76ce 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -306,9 +306,9 @@ sub index_xapian { # msg_iter callback
        index_body($self, $_, /\A>/ ? 0 : $doc) for @sections;
 }
 
-sub add_xapian ($$$$$$) {
-       my ($self, $mime, $num, $oid, $mids, $mid0) = @_;
-       my $smsg = PublicInbox::Smsg->new($mime);
+sub add_xapian ($$$$) {
+       my ($self, $mime, $smsg, $mids) = @_;
+       $smsg->{mime} = $mime; # XXX dangerous
        my $hdr = $mime->header_obj;
        $smsg->{ds} = msg_datestamp($hdr, $self->{autime});
        $smsg->{ts} = msg_timestamp($hdr, $self->{cotime});
@@ -338,9 +338,7 @@ sub add_xapian ($$$$$$) {
                        index_text($self, join(' ', @long), 1, 'XM');
                }
        }
-       $smsg->{to} = $smsg->{cc} = '';
-       $smsg->{blob} = $oid;
-       $smsg->{mid} = $mid0;
+       $smsg->{to} = $smsg->{cc} = ''; # WWW doesn't need these, only NNTP
        PublicInbox::OverIdx::parse_references($smsg, $hdr, $mids);
        my $data = $smsg->to_doc_data;
        $doc->set_data($data);
@@ -355,7 +353,7 @@ sub add_xapian ($$$$$$) {
                }
        }
        $doc->add_boolean_term('Q' . $_) foreach @$mids;
-       $self->{xdb}->replace_document($num, $doc);
+       $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
 
 sub _msgmap_init ($) {
@@ -369,20 +367,25 @@ sub _msgmap_init ($) {
 
 sub add_message {
        # mime = Email::MIME object
-       my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
+       my ($self, $mime, $smsg) = @_;
        my $mids = mids_for_index($mime->header_obj);
-       $mid0 //= $mids->[0]; # v1 compatibility
-       $num //= do { # v1
+       $smsg //= bless { blob => '' }, 'PublicInbox::Smsg'; # test-only compat
+       $smsg->{mid} //= $mids->[0]; # v1 compatibility
+       $smsg->{num} //= do { # v1
                _msgmap_init($self);
                index_mm($self, $mime);
        };
        eval {
-               if (need_xapian($self)) {
-                       add_xapian($self, $mime, $num, $oid, $mids, $mid0);
+               # order matters, overview stores every possible piece of
+               # data in doc_data (deflated).  Xapian only stores a subset
+               # of the fields which exist in over.sqlite3.  We may stop
+               # storing doc_data in Xapian sometime after we get multi-inbox
+               # search working.
+               if (my $over = $self->{over}) { # v1 only
+                       $over->add_overview($mime, $smsg, $self);
                }
-               if (my $over = $self->{over}) {
-                       $over->add_overview($mime, $bytes, $num, $oid, $mid0,
-                                               $self);
+               if (need_xapian($self)) {
+                       add_xapian($self, $mime, $smsg, $mids);
                }
        };
 
@@ -390,7 +393,7 @@ sub add_message {
                warn "failed to index message <".join('> <',@$mids).">: $@\n";
                return undef;
        }
-       $num;
+       $smsg->{num};
 }
 
 # returns begin and end PostingIterator
@@ -530,9 +533,10 @@ sub unindex_mm {
 }
 
 sub index_both {
-       my ($self, $mime, $bytes, $blob) = @_;
+       my ($self, $mime, $smsg) = @_;
        my $num = index_mm($self, $mime);
-       add_message($self, $mime, $bytes, $num, $blob);
+       $smsg->{num} = $num;
+       add_message($self, $mime, $smsg);
 }
 
 sub unindex_both {
@@ -595,8 +599,11 @@ sub read_log {
                                next;
                        }
                        my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+                       my $smsg = bless {}, 'PublicInbox::Smsg';
                        batch_adjust(\$max, $bytes, $batch_cb, $latest, ++$nr);
-                       $add_cb->($self, $mime, $bytes, $blob);
+                       $smsg->{blob} = $blob;
+                       $smsg->{bytes} = $bytes;
+                       $add_cb->($self, $mime, $smsg);
                } elsif ($line =~ /$delmsg/o) {
                        my $blob = $1;
                        $D{$blob} = 1;

diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 74c624a4d3dddcb67ac5938aaa1f5186c34284e1..d29e6090b2d58eb2942a3c4fb16e9e8be3d47af7 100644 (file)
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -76,7 +76,13 @@ sub shard_worker_loop ($$$$$) {
                        $artnum = int($artnum);
                        $self->{autime} = $autime;
                        $self->{cotime} = $cotime;
-                       $self->add_message($mime, $n, $artnum, $oid, $mid0);
+                       my $smsg = bless {
+                               bytes => $len,
+                               num => $artnum,
+                               blob => $oid,
+                               mid => $mid0,
+                       }, 'PublicInbox::Smsg';
+                       $self->add_message($mime, $smsg);
                }
        }
        $self->worker_done;
@@ -95,7 +101,13 @@ sub index_raw {
                $self->begin_txn_lazy;
                $self->{autime} = $at;
                $self->{cotime} = $ct;
-               $self->add_message($mime, $bytes, $artnum, $oid, $mid0);
+               my $smsg = bless {
+                       bytes => $bytes,
+                       num => $artnum,
+                       blob => $oid,
+                       mid => $mid0,
+               }, 'PublicInbox::Smsg';
+               $self->add_message($mime, $smsg);
        }
 }
 

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index d39a6f894539b4dce8de87c7360f7724a38b86d7..34dd139b8ab82495296ddb9abd76847dee54b5fc 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -150,7 +150,13 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$$$$) {
        my ($self, $msgref, $mime, $len, $num, $oid, $mid0) = @_;
-       $self->{over}->add_overview($mime, $len, $num, $oid, $mid0, $self);
+       my $smsg = bless {
+               bytes => $len,
+               num => $num,
+               blob => $oid,
+               mid => $mid0,
+       }, 'PublicInbox::Smsg';
+       $self->{over}->add_overview($mime, $smsg, $self);
        my $idx = idx_shard($self, $num % $self->{shards});
        $idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime, $self);
        my $n = $self->{transact_bytes} += $len;

diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index 6a5fd91901c140463d5eaef20da3ec19f31ba8a3..f073304a2a5bc1869255af9f180095e500a49ef4 100644 (file)
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -9,6 +9,7 @@ use PublicInbox::MID qw(mids);
 use Email::MIME;
 require_mods(qw(DBD::SQLite Search::Xapian));
 require PublicInbox::SearchIdx;
+require PublicInbox::Smsg;
 require PublicInbox::Inbox;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
@@ -45,7 +46,13 @@ foreach (reverse split(/\n\n/, $data)) {
        $mime->header_set('To' => 'git@vger.kernel.org');
        my $bytes = bytes::length($mime->as_string);
        my $mid = mids($mime->header_obj)->[0];
-       my $doc_id = $rw->add_message($mime, $bytes, ++$num, 'ignored', $mid);
+       my $smsg = bless {
+               bytes => $bytes,
+               num => ++$num,
+               mid => $mid,
+               blob => '',
+       }, 'PublicInbox::Smsg';
+       my $doc_id = $rw->add_message($mime, $smsg);
        push @mids, $mid;
        ok($doc_id, 'message added: '. $mid);
 }
@@ -86,7 +93,13 @@ SELECT tid FROM over WHERE num = ? LIMIT 1
 
        my $bytes = bytes::length($mime->as_string);
        my $mid = mids($mime->header_obj)->[0];
-       my $doc_id = $rw->add_message($mime, $bytes, $num, 'ignored', $mid);
+       my $smsg = bless {
+               bytes => $bytes,
+               num => $num,
+               mid => $mid,
+               blob => '',
+       }, 'PublicInbox::Smsg';
+       my $doc_id = $rw->add_message($mime, $smsg);
        ok($doc_id, 'message reindexed'. $mid);
        is($doc_id, $num, "article number unchanged: $num");