X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FOverIdx.pm;h=f49dfa000431af739eb5e82060c5596fe1af5baf;hb=d34a4b80724e3f77a507ad08b91039427b0e09d5;hp=0e43aabc044f61d0f16153b70f554cbc11159fc9;hpb=35ff6bb106909b1c1232666a9792156dfa398ea8;p=public-inbox.git diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index 0e43aabc..f49dfa00 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -1,34 +1,33 @@ -# Copyright (C) 2018 all contributors +# Copyright (C) 2018-2020 all contributors # License: AGPL-3.0+ # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI -# Unlike Msgmap, this is an _UNSTABLE_ database which can be +# Unlike Msgmap, this is an _UNSTABLE_ cache which can be # tweaked/updated over time and rebuilt. +# +# Ghost messages (messages which are only referenced in References/In-Reply-To) +# are denoted by a negative NNTP article number. package PublicInbox::OverIdx; use strict; use warnings; use base qw(PublicInbox::Over); use IO::Handle; use DBI qw(:sql_types); # SQL_BLOB +use PublicInbox::MID qw/id_compress mids_for_index references/; +use PublicInbox::Smsg qw(subject_normalized); +use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); +use Compress::Zlib qw(compress); +use PublicInbox::Search; sub dbh_new { my ($self) = @_; - my $dbh = $self->SUPER::dbh_new; - $dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead + my $dbh = $self->SUPER::dbh_new(1); $dbh->do('PRAGMA journal_mode = TRUNCATE'); $dbh->do('PRAGMA cache_size = 80000'); create_tables($dbh); $dbh; } -sub commit_fsync { - my $fn = $_[0]->{filename}; - if (open my $fh, '+<', $fn) { - $fh->sync; - close $fh; - } -} - sub get_counter ($$) { my ($dbh, $key) = @_; my $sth = $dbh->prepare_cached(<<'', undef, 1); @@ -81,8 +80,15 @@ sub mid2id { } sub delete_by_num { - my ($self, $num) = @_; + my ($self, $num, $tid_ref) = @_; my $dbh = $self->{dbh}; + if ($tid_ref) { + my $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT tid FROM over WHERE num = ? LIMIT 1 + + $sth->execute($num); + $$tid_ref = $sth->fetchrow_array; # may be undef + } foreach (qw(over id2num)) { $dbh->prepare_cached(<<"")->execute($num); DELETE FROM $_ WHERE num = ? @@ -209,14 +215,71 @@ sub link_refs { $tid; } +sub parse_references ($$$) { + my ($smsg, $hdr, $mids) = @_; + my $refs = references($hdr); + push(@$refs, @$mids) if scalar(@$mids) > 1; + return $refs if scalar(@$refs) == 0; + + # prevent circular references here: + my %seen = ( $smsg->{mid} => 1 ); + my @keep; + foreach my $ref (@$refs) { + if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) { + warn "References: <$ref> too long, ignoring\n"; + next; + } + push(@keep, $ref) unless $seen{$ref}++; + } + $smsg->{references} = '<'.join('> <', @keep).'>' if @keep; + \@keep; +} + +# normalize subjects so they are suitable as pathnames for URLs +# XXX: consider for removal +sub subject_path ($) { + my ($subj) = @_; + $subj = subject_normalized($subj); + $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g; + lc($subj); +} + +sub add_overview { + my ($self, $mime, $bytes, $num, $oid, $mid0, $times) = @_; + my $lines = $mime->body_raw =~ tr!\n!\n!; + my $smsg = bless { + mime => $mime, + mid => $mid0, + bytes => $bytes, + lines => $lines, + blob => $oid, + }, 'PublicInbox::Smsg'; + my $hdr = $mime->header_obj; + my $mids = mids_for_index($hdr); + my $refs = parse_references($smsg, $hdr, $mids); + my $subj = $smsg->subject; + my $xpath; + if ($subj ne '') { + $xpath = subject_path($subj); + $xpath = id_compress($xpath); + } + my $dd = $smsg->to_doc_data; + utf8::encode($dd); + $dd = compress($dd); + my $ds = msg_timestamp($hdr, $times->{autime}); + my $ts = msg_datestamp($hdr, $times->{cotime}); + my $values = [ $ts, $ds, $num, $mids, $refs, $xpath, $dd ]; + add_over($self, $values); +} + sub add_over { my ($self, $values) = @_; - my ($ts, $num, $mids, $refs, $xpath, $ddd) = @$values; + my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values; my $old_tid; my $vivified = 0; $self->begin_lazy; - $self->delete_by_num($num); + $self->delete_by_num($num, \$old_tid); foreach my $mid (@$mids) { my $v = 0; each_by_mid($self, $mid, ['tid'], sub { @@ -241,11 +304,11 @@ sub add_over { my $sid = $self->sid($xpath); my $dbh = $self->{dbh}; my $sth = $dbh->prepare_cached(<<''); -INSERT INTO over (num, tid, sid, ts, ddd) -VALUES (?,?,?,?,?) +INSERT INTO over (num, tid, sid, ts, ds, ddd) +VALUES (?,?,?,?,?,?) my $n = 0; - my @v = ($num, $tid, $sid, $ts); + my @v = ($num, $tid, $sid, $ts, $ds); foreach (@v) { $sth->bind_param(++$n, $_) } $sth->bind_param(++$n, $ddd, SQL_BLOB); $sth->execute; @@ -258,20 +321,35 @@ INSERT INTO id2num (id, num) VALUES (?,?) } } -sub delete_articles { - my ($self, $nums) = @_; - my $dbh = $self->connect; - $self->delete_by_num($_) foreach @$nums; -} - +# returns number of removed messages +# $oid may be undef to match only on $mid sub remove_oid { my ($self, $oid, $mid) = @_; + my $nr = 0; $self->begin_lazy; each_by_mid($self, $mid, ['ddd'], sub { my ($smsg) = @_; - $self->delete_by_num($smsg->{num}) if $smsg->{blob} eq $oid; + if (!defined($oid) || $smsg->{blob} eq $oid) { + $self->delete_by_num($smsg->{num}); + $nr++; + } 1; }); + $nr; +} + +sub num_mid0_for_oid { + my ($self, $oid, $mid) = @_; + my ($num, $mid0); + $self->begin_lazy; + each_by_mid($self, $mid, ['ddd'], sub { + my ($smsg) = @_; + my $blob = $smsg->{blob}; + return 1 if (!defined($blob) || $blob ne $oid); # continue; + ($num, $mid0) = ($smsg->{num}, $smsg->{mid}); + 0; # done + }); + ($num, $mid0); } sub create_tables { @@ -283,6 +361,7 @@ CREATE TABLE IF NOT EXISTS over ( tid INTEGER NOT NULL, sid INTEGER, ts INTEGER, + ds INTEGER, ddd VARBINARY, /* doc-data-deflated */ UNIQUE (num) ) @@ -290,6 +369,7 @@ CREATE TABLE IF NOT EXISTS over ( $dbh->do('CREATE INDEX IF NOT EXISTS idx_tid ON over (tid)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_sid ON over (sid)'); $dbh->do('CREATE INDEX IF NOT EXISTS idx_ts ON over (ts)'); + $dbh->do('CREATE INDEX IF NOT EXISTS idx_ds ON over (ds)'); $dbh->do(<<''); CREATE TABLE IF NOT EXISTS counter (