X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSharedKV.pm;h=90ccf2b4388c34dc48b21ad647c60b47b1ab98c0;hb=2231c8b183be0be5d8a9738a3e417b5c3a09c7c7;hp=52a7424eb19fe8a733c153b2ca94988942f84165;hpb=fa7bab31c9e5ded5c832b42de56f48dbdf9de972;p=public-inbox.git diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm index 52a7424e..90ccf2b4 100644 --- a/lib/PublicInbox/SharedKV.pm +++ b/lib/PublicInbox/SharedKV.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # fork()-friendly key-value store. Will be used for making @@ -8,15 +8,16 @@ package PublicInbox::SharedKV; use strict; use v5.10.1; use parent qw(PublicInbox::Lock); -use File::Temp 0.19 (); # 0.19 for ->newdir -use DBI (); +use File::Temp qw(tempdir); +use DBI qw(:sql_types); # SQL_BLOB use PublicInbox::Spawn; +use File::Path qw(rmtree make_path); sub dbh { my ($self, $lock) = @_; - $self->{dbh} //= do { + $self->{dbh} // do { my $f = $self->{filename}; - $lock //= $self->lock_for_scope; + $lock //= $self->lock_for_scope_fast; my $dbh = DBI->connect("dbi:SQLite:dbname=$f", '', '', { AutoCommit => 1, RaiseError => 1, @@ -26,7 +27,6 @@ sub dbh { }); my $opt = $self->{opt} // {}; $dbh->do('PRAGMA synchronous = OFF') if !$opt->{fsync}; - $dbh->do('PRAGMA cache_size = '.($opt->{cache_size} || 80000)); $dbh->do('PRAGMA journal_mode = '. ($opt->{journal_mode} // 'WAL')); $dbh->do(<<''); @@ -36,35 +36,35 @@ CREATE TABLE IF NOT EXISTS kv ( UNIQUE (k) ) - $dbh->do('CREATE INDEX IF NOT EXISTS idx_v ON kv (v)'); - $dbh; + $self->{dbh} = $dbh; } } sub new { my ($cls, $dir, $base, $opt) = @_; my $self = bless { opt => $opt }, $cls; - unless (defined $dir) { - $self->{tmp} = File::Temp->newdir('kv-XXXXXX', TMPDIR => 1); - $dir = $self->{tmp}->dirname; - } - -d $dir or mkdir($dir) or die "mkdir($dir): $!"; + make_path($dir) if defined($dir) && !-d $dir; + $dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1); $base //= ''; my $f = $self->{filename} = "$dir/$base.sqlite3"; $self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock"; - unless (-f $f) { + unless (-s $f) { + require PublicInbox::Syscall; + PublicInbox::Syscall::nodatacow_dir($dir); # for journal/shm/wal open my $fh, '+>>', $f or die "failed to open $f: $!"; - PublicInbox::Spawn::nodatacow_fd(fileno($fh)); } $self; } sub set_maybe { my ($self, $key, $val, $lock) = @_; - $lock //= $self->lock_for_scope; - my $e = $self->{dbh}->prepare_cached(<<'')->execute($key, $val); + $lock //= $self->lock_for_scope_fast; + my $sth = $self->{dbh}->prepare_cached(<<''); INSERT OR IGNORE INTO kv (k,v) VALUES (?, ?) + $sth->bind_param(1, $key, SQL_BLOB); + $sth->bind_param(2, $val, SQL_BLOB); + my $e = $sth->execute; $e == 0 ? undef : $e; } @@ -78,33 +78,43 @@ SELECT k,v FROM kv $sth } -sub delete_by_val { - my ($self, $val, $lock) = @_; - $lock //= $self->lock_for_scope; - $self->{dbh}->prepare_cached(<<'')->execute($val) + 0; -DELETE FROM kv WHERE v = ? - -} - -sub replace_values { - my ($self, $oldval, $newval, $lock) = @_; - $lock //= $self->lock_for_scope; - $self->{dbh}->prepare_cached(<<'')->execute($newval, $oldval) + 0; -UPDATE kv SET v = ? WHERE v = ? - +sub keys { + my ($self, @pfx) = @_; + my $sql = 'SELECT k FROM kv'; + if (defined $pfx[0]) { + $sql .= ' WHERE k LIKE ? ESCAPE ?'; + my $anywhere = !!$pfx[1]; + $pfx[1] = '\\'; + $pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars + $pfx[0] .= '%'; + substr($pfx[0], 0, 0, '%') if $anywhere; + } else { + @pfx = (); # [0] may've been undef + } + my $sth = $self->dbh->prepare($sql); + if (@pfx) { + $sth->bind_param(1, $pfx[0], SQL_BLOB); + $sth->bind_param(2, $pfx[1]); + } + $sth->execute; + map { $_->[0] } @{$sth->fetchall_arrayref}; } sub set { my ($self, $key, $val) = @_; if (defined $val) { - my $e = $self->{dbh}->prepare_cached(<<'')->execute($key, $val); + my $sth = $self->{dbh}->prepare_cached(<<''); INSERT OR REPLACE INTO kv (k,v) VALUES (?,?) + $sth->bind_param(1, $key, SQL_BLOB); + $sth->bind_param(2, $val, SQL_BLOB); + my $e = $sth->execute; $e == 0 ? undef : $e; } else { - $self->{dbh}->prepare_cached(<<'')->execute($key); + my $sth = $self->{dbh}->prepare_cached(<<''); DELETE FROM kv WHERE k = ? + $sth->bind_param(1, $key, SQL_BLOB); } } @@ -113,20 +123,23 @@ sub get { my $sth = $self->{dbh}->prepare_cached(<<'', undef, 1); SELECT v FROM kv WHERE k = ? - $sth->execute($key); + $sth->bind_param(1, $key, SQL_BLOB); + $sth->execute; $sth->fetchrow_array; } sub xchg { my ($self, $key, $newval, $lock) = @_; - $lock //= $self->lock_for_scope; + $lock //= $self->lock_for_scope_fast; my $oldval = get($self, $key); if (defined $newval) { set($self, $key, $newval); } else { - $self->{dbh}->prepare_cached(<<'')->execute($key); + my $sth = $self->{dbh}->prepare_cached(<<''); DELETE FROM kv WHERE k = ? + $sth->bind_param(1, $key, SQL_BLOB); + $sth->execute; } $oldval; } @@ -140,4 +153,32 @@ SELECT COUNT(k) FROM kv $sth->fetchrow_array; } +# faster than ->count due to how SQLite works +sub has_entries { + my ($self) = @_; + my @n = $self->{dbh}->selectrow_array('SELECT k FROM kv LIMIT 1'); + scalar(@n) ? 1 : undef; +} + +sub dbh_release { + my ($self, $lock) = @_; + my $dbh = delete $self->{dbh} or return; + $lock //= $self->lock_for_scope_fast; # may be needed for WAL + %{$dbh->{CachedKids}} = (); # cleanup prepare_cached + $dbh->disconnect; +} + +sub DESTROY { + my ($self) = @_; + dbh_release($self); + my $dir = delete $self->{"tmp$$.$self"} or return; + my $tries = 0; + do { + $! = 0; + eval { rmtree($dir) }; + } while ($@ && $!{ENOENT} && $tries++ < 5); + warn "error removing $dir: $@" if $@; + warn "Took $tries tries to remove $dir\n" if $tries; +} + 1;