store less data in the Xapian document

[public-inbox.git] / lib / PublicInbox / OverIdx.pm

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 0e43aabc044f61d0f16153b70f554cbc11159fc9..62fec0dafb4056118a320ca3fe5ba42030b1fa33 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -2,33 +2,31 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
-# Unlike Msgmap, this is an _UNSTABLE_ database which can be
+# Unlike Msgmap, this is an _UNSTABLE_ cache which can be
 # tweaked/updated over time and rebuilt.
+#
+# Ghost messages (messages which are only referenced in References/In-Reply-To)
+# are denoted by a negative NNTP article number.
 package PublicInbox::OverIdx;
 use strict;
 use warnings;
 use base qw(PublicInbox::Over);
 use IO::Handle;
 use DBI qw(:sql_types); # SQL_BLOB
+use PublicInbox::MID qw/id_compress mids references/;
+use PublicInbox::SearchMsg;
+use Compress::Zlib qw(compress);
+use PublicInbox::Search;
 
 sub dbh_new {
        my ($self) = @_;
        my $dbh = $self->SUPER::dbh_new;
-       $dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead
        $dbh->do('PRAGMA journal_mode = TRUNCATE');
        $dbh->do('PRAGMA cache_size = 80000');
        create_tables($dbh);
        $dbh;
 }
 
-sub commit_fsync {
-       my $fn = $_[0]->{filename};
-       if (open my $fh, '+<', $fn) {
-               $fh->sync;
-               close $fh;
-       }
-}
-
 sub get_counter ($$) {
        my ($dbh, $key) = @_;
        my $sth = $dbh->prepare_cached(<<'', undef, 1);
@@ -209,9 +207,57 @@ sub link_refs {
        $tid;
 }
 
+sub parse_references ($$$) {
+       my ($smsg, $mid0, $mids) = @_;
+       my $mime = $smsg->{mime};
+       my $hdr = $mime->header_obj;
+       my $refs = references($hdr);
+       push(@$refs, @$mids) if scalar(@$mids) > 1;
+       return $refs if scalar(@$refs) == 0;
+
+       # prevent circular references here:
+       my %seen = ( $mid0 => 1 );
+       my @keep;
+       foreach my $ref (@$refs) {
+               if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) {
+                       warn "References: <$ref> too long, ignoring\n";
+                       next;
+               }
+               next if $seen{$ref}++;
+               push @keep, $ref;
+       }
+       $smsg->{references} = '<'.join('> <', @keep).'>' if @keep;
+       \@keep;
+}
+
+sub add_overview {
+       my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
+       my $lines = $mime->body_raw =~ tr!\n!\n!;
+       my $smsg = bless {
+               mime => $mime,
+               mid => $mid0,
+               bytes => $bytes,
+               lines => $lines,
+               blob => $oid,
+       }, 'PublicInbox::SearchMsg';
+       my $mids = mids($mime->header_obj);
+       my $refs = parse_references($smsg, $mid0, $mids);
+       my $subj = $smsg->subject;
+       my $xpath;
+       if ($subj ne '') {
+               $xpath = PublicInbox::Search::subject_path($subj);
+               $xpath = id_compress($xpath);
+       }
+       my $dd = $smsg->to_doc_data($oid, $mid0);
+       utf8::encode($dd);
+       $dd = compress($dd);
+       my $values = [ $smsg->ts, $smsg->ds, $num, $mids, $refs, $xpath, $dd ];
+       add_over($self, $values);
+}
+
 sub add_over {
        my ($self, $values) = @_;
-       my ($ts, $num, $mids, $refs, $xpath, $ddd) = @$values;
+       my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values;
        my $old_tid;
        my $vivified = 0;
 
@@ -241,11 +287,11 @@ sub add_over {
        my $sid = $self->sid($xpath);
        my $dbh = $self->{dbh};
        my $sth = $dbh->prepare_cached(<<'');
-INSERT INTO over (num, tid, sid, ts, ddd)
-VALUES (?,?,?,?,?)
+INSERT INTO over (num, tid, sid, ts, ds, ddd)
+VALUES (?,?,?,?,?,?)
 
        my $n = 0;
-       my @v = ($num, $tid, $sid, $ts);
+       my @v = ($num, $tid, $sid, $ts, $ds);
        foreach (@v) { $sth->bind_param(++$n, $_) }
        $sth->bind_param(++$n, $ddd, SQL_BLOB);
        $sth->execute;
@@ -283,6 +329,7 @@ CREATE TABLE IF NOT EXISTS over (
        tid INTEGER NOT NULL,
        sid INTEGER,
        ts INTEGER,
+       ds INTEGER,
        ddd VARBINARY, /* doc-data-deflated */
        UNIQUE (num)
 )
@@ -290,6 +337,7 @@ CREATE TABLE IF NOT EXISTS over (
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_tid ON over (tid)');
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_sid ON over (sid)');
        $dbh->do('CREATE INDEX IF NOT EXISTS idx_ts ON over (ts)');
+       $dbh->do('CREATE INDEX IF NOT EXISTS idx_ds ON over (ds)');
 
        $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS counter (