X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FOverIdx.pm;h=08f87447bea5a288a6f1c8fa7992bb966e129930;hb=06cd576bb9b73eb177728fead06de8c3fac8d7a3;hp=0e43aabc044f61d0f16153b70f554cbc11159fc9;hpb=35ff6bb106909b1c1232666a9792156dfa398ea8;p=public-inbox.git

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 0e43aabc..08f87447 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -2,33 +2,31 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
-# Unlike Msgmap, this is an _UNSTABLE_ database which can be
+# Unlike Msgmap, this is an _UNSTABLE_ cache which can be
 # tweaked/updated over time and rebuilt.
+#
+# Ghost messages (messages which are only referenced in References/In-Reply-To)
+# are denoted by a negative NNTP article number.
 package PublicInbox::OverIdx;
 use strict;
 use warnings;
 use base qw(PublicInbox::Over);
 use IO::Handle;
 use DBI qw(:sql_types); # SQL_BLOB
+use PublicInbox::MID qw/id_compress mids references/;
+use PublicInbox::SearchMsg;
+use Compress::Zlib qw(compress);
+use PublicInbox::Search;
 
 sub dbh_new {
 	my ($self) = @_;
 	my $dbh = $self->SUPER::dbh_new;
-	$dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead
 	$dbh->do('PRAGMA journal_mode = TRUNCATE');
 	$dbh->do('PRAGMA cache_size = 80000');
 	create_tables($dbh);
 	$dbh;
 }
 
-sub commit_fsync {
-	my $fn = $_[0]->{filename};
-	if (open my $fh, '+<', $fn) {
-		$fh->sync;
-		close $fh;
-	}
-}
-
 sub get_counter ($$) {
 	my ($dbh, $key) = @_;
 	my $sth = $dbh->prepare_cached(<<'', undef, 1);
@@ -209,9 +207,57 @@ sub link_refs {
 	$tid;
 }
 
+sub parse_references ($$$$) {
+	my ($self, $smsg, $mid0, $mids) = @_;
+	my $mime = $smsg->{mime};
+	my $hdr = $mime->header_obj;
+	my $refs = references($hdr);
+	push(@$refs, @$mids) if scalar(@$mids) > 1;
+	return $refs if scalar(@$refs) == 0;
+
+	# prevent circular references here:
+	my %seen = ( $mid0 => 1 );
+	my @keep;
+	foreach my $ref (@$refs) {
+		if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) {
+			warn "References: <$ref> too long, ignoring\n";
+			next;
+		}
+		next if $seen{$ref}++;
+		push @keep, $ref;
+	}
+	$smsg->{references} = '<'.join('> <', @keep).'>' if @keep;
+	\@keep;
+}
+
+sub add_overview {
+	my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
+	my $lines = $mime->body_raw =~ tr!\n!\n!;
+	my $smsg = bless {
+		mime => $mime,
+		mid => $mid0,
+		bytes => $bytes,
+		lines => $lines,
+		blob => $oid,
+	}, 'PublicInbox::SearchMsg';
+	my $mids = mids($mime->header_obj);
+	my $refs = $self->parse_references($smsg, $mid0, $mids);
+	my $subj = $smsg->subject;
+	my $xpath;
+	if ($subj ne '') {
+		$xpath = PublicInbox::Search::subject_path($subj);
+		$xpath = id_compress($xpath);
+	}
+	my $dd = $smsg->to_doc_data($oid, $mid0);
+	utf8::encode($dd);
+	$dd = compress($dd);
+	my $values = [ $smsg->ts, $smsg->ds, $num, $mids, $refs, $xpath, $dd ];
+	add_over($self, $values);
+}
+
 sub add_over {
 	my ($self, $values) = @_;
-	my ($ts, $num, $mids, $refs, $xpath, $ddd) = @$values;
+	my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values;
 	my $old_tid;
 	my $vivified = 0;
 
@@ -241,11 +287,11 @@ sub add_over {
 	my $sid = $self->sid($xpath);
 	my $dbh = $self->{dbh};
 	my $sth = $dbh->prepare_cached(<<'');
-INSERT INTO over (num, tid, sid, ts, ddd)
-VALUES (?,?,?,?,?)
+INSERT INTO over (num, tid, sid, ts, ds, ddd)
+VALUES (?,?,?,?,?,?)
 
 	my $n = 0;
-	my @v = ($num, $tid, $sid, $ts);
+	my @v = ($num, $tid, $sid, $ts, $ds);
 	foreach (@v) { $sth->bind_param(++$n, $_) }
 	$sth->bind_param(++$n, $ddd, SQL_BLOB);
 	$sth->execute;
@@ -283,6 +329,7 @@ CREATE TABLE IF NOT EXISTS over (
 	tid INTEGER NOT NULL,
 	sid INTEGER,
 	ts INTEGER,
+	ds INTEGER,
 	ddd VARBINARY, /* doc-data-deflated */
 	UNIQUE (num)
 )
@@ -290,6 +337,7 @@ CREATE TABLE IF NOT EXISTS over (
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_tid ON over (tid)');
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_sid ON over (sid)');
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_ts ON over (ts)');
+	$dbh->do('CREATE INDEX IF NOT EXISTS idx_ds ON over (ds)');
 
 	$dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS counter (