X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FOverIdx.pm;h=acbf2c8de6092742e10cdda77415812cbf6778d6;hb=b5bc3576af3d0ef0fa884ed32a674c7a703a19b2;hp=0e43aabc044f61d0f16153b70f554cbc11159fc9;hpb=35ff6bb106909b1c1232666a9792156dfa398ea8;p=public-inbox.git diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index 0e43aabc..acbf2c8d 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -1,34 +1,32 @@ -# Copyright (C) 2018 all contributors +# Copyright (C) 2018-2020 all contributors # License: AGPL-3.0+ # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI -# Unlike Msgmap, this is an _UNSTABLE_ database which can be +# Unlike Msgmap, this is an _UNSTABLE_ cache which can be # tweaked/updated over time and rebuilt. +# +# Ghost messages (messages which are only referenced in References/In-Reply-To) +# are denoted by a negative NNTP article number. package PublicInbox::OverIdx; use strict; use warnings; use base qw(PublicInbox::Over); use IO::Handle; use DBI qw(:sql_types); # SQL_BLOB +use PublicInbox::MID qw/id_compress mids_for_index references/; +use PublicInbox::Smsg qw(subject_normalized); +use Compress::Zlib qw(compress); +use PublicInbox::Search; sub dbh_new { my ($self) = @_; - my $dbh = $self->SUPER::dbh_new; - $dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead + my $dbh = $self->SUPER::dbh_new(1); $dbh->do('PRAGMA journal_mode = TRUNCATE'); $dbh->do('PRAGMA cache_size = 80000'); create_tables($dbh); $dbh; } -sub commit_fsync { - my $fn = $_[0]->{filename}; - if (open my $fh, '+<', $fn) { - $fh->sync; - close $fh; - } -} - sub get_counter ($$) { my ($dbh, $key) = @_; my $sth = $dbh->prepare_cached(<<'', undef, 1); @@ -81,8 +79,15 @@ sub mid2id { } sub delete_by_num { - my ($self, $num) = @_; + my ($self, $num, $tid_ref) = @_; my $dbh = $self->{dbh}; + if ($tid_ref) { + my $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT tid FROM over WHERE num = ? LIMIT 1 + + $sth->execute($num); + $$tid_ref = $sth->fetchrow_array; # may be undef + } foreach (qw(over id2num)) { $dbh->prepare_cached(<<"")->execute($num); DELETE FROM $_ WHERE num = ? @@ -209,14 +214,62 @@ sub link_refs { $tid; } +sub parse_references ($$$) { + my ($smsg, $hdr, $mids) = @_; + my $refs = references($hdr); + push(@$refs, @$mids) if scalar(@$mids) > 1; + return $refs if scalar(@$refs) == 0; + + # prevent circular references here: + my %seen = ( $smsg->{mid} => 1 ); + my @keep; + foreach my $ref (@$refs) { + if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) { + warn "References: <$ref> too long, ignoring\n"; + next; + } + push(@keep, $ref) unless $seen{$ref}++; + } + $smsg->{references} = '<'.join('> <', @keep).'>' if @keep; + \@keep; +} + +# normalize subjects so they are suitable as pathnames for URLs +# XXX: consider for removal +sub subject_path ($) { + my ($subj) = @_; + $subj = subject_normalized($subj); + $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g; + lc($subj); +} + +sub add_overview { + my ($self, $mime, $smsg) = @_; + $smsg->{lines} = $mime->body_raw =~ tr!\n!\n!; + $smsg->{mime} = $mime; # XXX temporary? + my $hdr = $mime->header_obj; + my $mids = mids_for_index($hdr); + my $refs = parse_references($smsg, $hdr, $mids); + my $subj = $smsg->subject; + my $xpath; + if ($subj ne '') { + $xpath = subject_path($subj); + $xpath = id_compress($xpath); + } + my $dd = $smsg->to_doc_data; + utf8::encode($dd); + $dd = compress($dd); + add_over($self, [ @$smsg{qw(ts ds num)}, $mids, $refs, $xpath, $dd ]); +} + sub add_over { my ($self, $values) = @_; - my ($ts, $num, $mids, $refs, $xpath, $ddd) = @$values; + my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values; my $old_tid; my $vivified = 0; $self->begin_lazy; - $self->delete_by_num($num); + $self->delete_by_num($num, \$old_tid); foreach my $mid (@$mids) { my $v = 0; each_by_mid($self, $mid, ['tid'], sub { @@ -241,11 +294,11 @@ sub add_over { my $sid = $self->sid($xpath); my $dbh = $self->{dbh}; my $sth = $dbh->prepare_cached(<<''); -INSERT INTO over (num, tid, sid, ts, ddd) -VALUES (?,?,?,?,?) +INSERT INTO over (num, tid, sid, ts, ds, ddd) +VALUES (?,?,?,?,?,?) my $n = 0; - my @v = ($num, $tid, $sid, $ts); + my @v = ($num, $tid, $sid, $ts, $ds); foreach (@v) { $sth->bind_param(++$n, $_) } $sth->bind_param(++$n, $ddd, SQL_BLOB); $sth->execute; @@ -258,20 +311,35 @@ INSERT INTO id2num (id, num) VALUES (?,?) } } -sub delete_articles { - my ($self, $nums) = @_; - my $dbh = $self->connect; - $self->delete_by_num($_) foreach @$nums; -} - +# returns number of removed messages +# $oid may be undef to match only on $mid sub remove_oid { my ($self, $oid, $mid) = @_; + my $nr = 0; $self->begin_lazy; each_by_mid($self, $mid, ['ddd'], sub { my ($smsg) = @_; - $self->delete_by_num($smsg->{num}) if $smsg->{blob} eq $oid; + if (!defined($oid) || $smsg->{blob} eq $oid) { + $self->delete_by_num($smsg->{num}); + $nr++; + } 1; }); + $nr; +} + +sub num_mid0_for_oid { + my ($self, $oid, $mid) = @_; + my ($num, $mid0); + $self->begin_lazy; + each_by_mid($self, $mid, ['ddd'], sub { + my ($smsg) = @_; + my $blob = $smsg->{blob}; + return 1 if (!defined($blob) || $blob ne $oid); # continue; + ($num, $mid0) = ($smsg->{num}, $smsg->{mid}); + 0; # done + }); + ($num, $mid0); } sub create_tables { @@ -283,6 +351,7 @@ CREATE TABLE IF NOT EXISTS over ( tid INTEGER NOT NULL, sid INTEGER, ts INTEGER, + ds INTEGER, ddd VARBINARY, /* doc-data-deflated */ UNIQUE (num) ) @@ -290,6 +359,7 @@ CREATE TABLE IF NOT EXISTS over ( $dbh->do('CREATE INDEX IF NOT EXISTS idx_tid ON over (tid)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_sid ON over (sid)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_ts ON over (ts)'); + $dbh->do('CREATE INDEX IF NOT EXISTS idx_ds ON over (ds)'); $dbh->do(<<''); CREATE TABLE IF NOT EXISTS counter (