X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FOver.pm;h=786f9d922282ea3358c7664462ceb8b591d05e4d;hb=14fa0abdcc7b6513540e529375e53edd74ce13e8;hp=3e74b7a6f3ccd9372d67078fab01a38ebc92aa05;hpb=5f6a0d2342323541e44ff2f1e7329053d0263800;p=public-inbox.git diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index 3e74b7a6..786f9d92 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2020 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI @@ -6,8 +6,8 @@ # tweaked/updated over time and rebuilt. package PublicInbox::Over; use strict; -use warnings; -use DBI; +use v5.10.1; +use DBI qw(:sql_types); # SQL_BLOB use DBD::SQLite; use PublicInbox::Smsg; use Compress::Zlib qw(uncompress); @@ -16,11 +16,12 @@ use constant DEFAULT_LIMIT => 1000; sub dbh_new { my ($self, $rw) = @_; my $f = delete $self->{filename}; - if (!-f $f) { # SQLite defaults mode to 0644, we want 0666 + if (!-s $f) { # SQLite defaults mode to 0644, we want 0666 if ($rw) { - require PublicInbox::Spawn; + require PublicInbox::Syscall; + my ($dir) = ($f =~ m!(.+)/[^/]+\z!); + PublicInbox::Syscall::nodatacow_dir($dir); open my $fh, '+>>', $f or die "failed to open $f: $!"; - PublicInbox::Spawn::nodatacow_fd(fileno($fh)); } else { $self->{filename} = $f; # die on stat() below: } @@ -72,7 +73,7 @@ sub new { bless { filename => $f }, $class; } -sub disconnect { +sub dbh_close { my ($self) = @_; if (my $dbh = delete $self->{dbh}) { delete $self->{-get_art}; @@ -80,7 +81,7 @@ sub disconnect { } } -sub connect { $_[0]->{dbh} //= $_[0]->dbh_new } +sub dbh ($) { $_[0]->{dbh} //= $_[0]->dbh_new } # dbh_new may be subclassed sub load_from_row ($;$) { my ($smsg, $cull) = @_; @@ -97,18 +98,17 @@ sub load_from_row ($;$) { sub do_get { my ($self, $sql, $opts, @args) = @_; - my $dbh = $self->connect; my $lim = (($opts->{limit} || 0) + 0) || DEFAULT_LIMIT; $sql .= "LIMIT $lim"; - my $msgs = $dbh->selectall_arrayref($sql, { Slice => {} }, @args); + my $msgs = dbh($self)->selectall_arrayref($sql, { Slice => {} }, @args); my $cull = $opts->{cull}; load_from_row($_, $cull) for @$msgs; $msgs } sub query_xover { - my ($self, $beg, $end) = @_; - do_get($self, <<'', {}, $beg, $end); + my ($self, $beg, $end, $opt) = @_; + do_get($self, <<'', $opt, $beg, $end); SELECT num,ts,ds,ddd FROM over WHERE num >= ? AND num <= ? ORDER BY num ASC @@ -127,7 +127,7 @@ sub get_all { my $nr = scalar(@_) or return []; my $in = '?' . (',?' x ($nr - 1)); do_get($self, <<"", { cull => 1, limit => $nr }, @_); -SELECT num,ds,ddd FROM over WHERE num IN ($in) +SELECT num,ts,ds,ddd FROM over WHERE num IN ($in) } @@ -135,7 +135,7 @@ sub nothing () { wantarray ? (0, []) : [] }; sub get_thread { my ($self, $mid, $prev) = @_; - my $dbh = $self->connect; + my $dbh = dbh($self); my $opts = { cull => 1 }; my $id = $dbh->selectrow_array(<<'', undef, $mid); @@ -202,7 +202,7 @@ ORDER BY $sort_col DESC # returns true if we have IDs, undef if not sub expand_thread { my ($self, $ctx) = @_; - my $dbh = $self->connect; + my $dbh = dbh($self); do { defined(my $num = $ctx->{ids}->[0]) or return; my ($tid) = $dbh->selectrow_array(<<'', undef, $num); @@ -245,21 +245,15 @@ sub recent { $s = '+num > 0 ORDER BY ts DESC'; } } - my $msgs = do_get($self, <<"", $opts, @v); + do_get($self, <<"", $opts, @v); SELECT ts,ds,ddd FROM over WHERE $s - return $msgs unless wantarray; - - my $nr = $self->{dbh}->selectrow_array(<<''); -SELECT COUNT(num) FROM over WHERE num > 0 - - ($nr, $msgs); } sub get_art { my ($self, $num) = @_; # caching $sth ourselves is faster than prepare_cached - my $sth = $self->{-get_art} //= $self->connect->prepare(<<''); + my $sth = $self->{-get_art} //= dbh($self)->prepare(<<''); SELECT num,tid,ds,ts,ddd FROM over WHERE num = ? LIMIT 1 $sth->execute($num); @@ -267,9 +261,30 @@ SELECT num,tid,ds,ts,ddd FROM over WHERE num = ? LIMIT 1 $smsg ? load_from_row($smsg) : undef; } +sub get_xref3 { + my ($self, $num, $raw) = @_; + my $dbh = dbh($self); + my $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT ibx_id,xnum,oidbin FROM xref3 WHERE docid = ? ORDER BY ibx_id,xnum ASC + + $sth->execute($num); + my $rows = $sth->fetchall_arrayref; + return $rows if $raw; + my $eidx_key_sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT eidx_key FROM inboxes WHERE ibx_id = ? + + [ map { + my $r = $_; + $eidx_key_sth->execute($r->[0]); + my $eidx_key = $eidx_key_sth->fetchrow_array; + $eidx_key //= "missing://ibx_id=$r->[0]"; + "$eidx_key:$r->[1]:".unpack('H*', $r->[2]); + } @$rows ]; +} + sub next_by_mid { my ($self, $mid, $id, $prev) = @_; - my $dbh = $self->connect; + my $dbh = dbh($self); unless (defined $$id) { my $sth = $dbh->prepare_cached(<<'', undef, 1); @@ -293,7 +308,7 @@ ORDER BY num ASC LIMIT 1 # IMAP search, this is limited by callers to UID_SLICE size (50K) sub uid_range { my ($self, $beg, $end, $sql) = @_; - my $dbh = $self->connect; + my $dbh = dbh($self); my $q = 'SELECT num FROM over WHERE num >= ? AND num <= ?'; # This is read-only, anyways; but caller should verify it's @@ -305,7 +320,7 @@ sub uid_range { sub max { my ($self) = @_; - my $sth = $self->connect->prepare_cached(<<'', undef, 1); + my $sth = dbh($self)->prepare_cached(<<'', undef, 1); SELECT MAX(num) FROM over WHERE num > 0 $sth->execute; @@ -314,7 +329,7 @@ SELECT MAX(num) FROM over WHERE num > 0 sub imap_exists { my ($self, $uid_base, $uid_end) = @_; - my $sth = $self->connect->prepare_cached(<<'', undef, 1); + my $sth = dbh($self)->prepare_cached(<<'', undef, 1); SELECT COUNT(num) FROM over WHERE num > ? AND num <= ? $sth->execute($uid_base, $uid_end); @@ -329,10 +344,43 @@ sub check_inodes { my $st = pack('dd', $st[0], $st[1]); # don't actually reopen, just let {dbh} be recreated later - disconnect($self) if $st ne ($self->{st} // $st); + dbh_close($self) if $st ne ($self->{st} // $st); } else { warn "W: stat $f: $!\n"; } } +sub oidbin_exists { + my ($self, $oidbin) = @_; + if (wantarray) { + my $sth = $self->dbh->prepare_cached(<<'', undef, 1); +SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC + + $sth->bind_param(1, $oidbin, SQL_BLOB); + $sth->execute; + my $tmp = $sth->fetchall_arrayref; + map { $_->[0] } @$tmp; + } else { + my $sth = $self->dbh->prepare_cached(<<'', undef, 1); +SELECT COUNT(*) FROM xref3 WHERE oidbin = ? + + $sth->bind_param(1, $oidbin, SQL_BLOB); + $sth->execute; + $sth->fetchrow_array; + } +} + +sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) } + +# used by NNTP.pm +sub ids_after { + my ($self, $num) = @_; + my $ids = dbh($self)->selectcol_arrayref(<<'', undef, $$num); +SELECT num FROM over WHERE num > ? +ORDER BY num ASC LIMIT 1000 + + $$num = $ids->[-1] if @$ids; + $ids; +} + 1;