X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiMailSync.pm;h=e70cb5de2b6b5046736ba72feeeb51a05a6bdf7b;hp=6dfa03be2feaf2063d3681626a869271423f36fa;hb=9f8e28a80374e905c831d2d5f3a45c6a9d708fa3;hpb=a29946ad3bc49704a512ae3ab31a45bbd66165e2 diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm index 6dfa03be..e70cb5de 100644 --- a/lib/PublicInbox/LeiMailSync.pm +++ b/lib/PublicInbox/LeiMailSync.pm @@ -5,6 +5,7 @@ package PublicInbox::LeiMailSync; use strict; use v5.10.1; +use parent qw(PublicInbox::Lock); use DBI; use PublicInbox::ContentHash qw(git_sha); use Carp (); @@ -13,6 +14,11 @@ sub dbh_new { my ($self, $rw) = @_; my $f = $self->{filename}; my $creat = $rw && !-s $f; + if ($creat) { + require PublicInbox::Spawn; + open my $fh, '+>>', $f or Carp::croak "open($f): $!"; + PublicInbox::Spawn::nodatacow_fd(fileno($fh)); + } my $dbh = DBI->connect("dbi:SQLite:dbname=$f",'','', { AutoCommit => 1, RaiseError => 1, @@ -21,7 +27,7 @@ sub dbh_new { sqlite_use_immediate_transaction => 1, }); # no sqlite_unicode, here, all strings are binary - create_tables($dbh) if $rw; + create_tables($self, $dbh) if $rw; $dbh->do('PRAGMA journal_mode = WAL') if $creat; $dbh->do('PRAGMA case_sensitive_like = ON'); $dbh; @@ -29,15 +35,25 @@ sub dbh_new { sub new { my ($cls, $f) = @_; - bless { filename => $f, fmap => {} }, $cls; + bless { + filename => $f, + fmap => {}, + lock_path => "$f.flock", + }, $cls; } -sub lms_commit { delete($_[0]->{dbh})->commit } +sub lms_write_prepare { ($_[0]->{dbh} //= dbh_new($_[0], 1)); $_[0] } -sub lms_begin { ($_[0]->{dbh} //= dbh_new($_[0], 1))->begin_work }; +sub lms_pause { + my ($self) = @_; + $self->{fmap} = {}; + my $dbh = delete $self->{dbh}; + eval { $dbh->do('PRAGMA optimize') } if $dbh; +} sub create_tables { - my ($dbh) = @_; + my ($self, $dbh) = @_; + my $lk = $self->lock_for_scope; $dbh->do(<<''); CREATE TABLE IF NOT EXISTS folders ( @@ -51,6 +67,7 @@ CREATE TABLE IF NOT EXISTS blob2num ( oidbin VARBINARY NOT NULL, fid INTEGER NOT NULL, /* folder ID */ uid INTEGER NOT NULL, /* NNTP article number, IMAP UID, MH number */ + /* not UNIQUE(fid, uid), since we may have broken servers */ UNIQUE (oidbin, fid, uid) ) @@ -63,6 +80,7 @@ CREATE TABLE IF NOT EXISTS blob2name ( oidbin VARBINARY NOT NULL, fid INTEGER NOT NULL, /* folder ID */ name VARBINARY NOT NULL, /* Maildir basename, JMAP blobId */ + /* not UNIQUE(fid, name), since we may have broken software */ UNIQUE (oidbin, fid, name) ) @@ -79,8 +97,8 @@ sub fid_for { my ($fid) = $dbh->selectrow_array($sel, undef, $folder); return $fid if defined $fid; + # caller had trailing slash (LeiToMail) if ($folder =~ s!\A((?:maildir|mh):.*?)/+\z!$1!i) { - warn "folder: $folder/ had trailing slash in arg\n"; ($fid) = $dbh->selectrow_array($sel, undef, $folder); if (defined $fid) { $dbh->do(<lock_for_scope; + for my $f (@folders) { $self->{fmap}->{$f} //= fid_for($self, $f, 1) } +} + sub set_src { - my ($self, $oidhex, $folder, $id) = @_; + my ($self, $oidbin, $folder, $id) = @_; + my $lk = $self->lock_for_scope; my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1); my $sth; if (ref($id)) { # scalar name @@ -131,11 +156,12 @@ INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?) INSERT OR IGNORE INTO blob2num (oidbin, fid, uid) VALUES (?, ?, ?) } - $sth->execute(pack('H*', $oidhex), $fid, $id); + $sth->execute($oidbin, $fid, $id); } sub clear_src { my ($self, $folder, $id) = @_; + my $lk = $self->lock_for_scope; my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1); my $sth; if (ref($id)) { # scalar name @@ -154,6 +180,7 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ? # Maildir-only sub mv_src { my ($self, $folder, $oidbin, $id, $newbn) = @_; + my $lk = $self->lock_for_scope; my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1); my $sth = $self->{dbh}->prepare_cached(<<''); UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ? @@ -171,22 +198,49 @@ INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?) sub each_src { my ($self, $folder, $cb, @args) = @_; my $dbh = $self->{dbh} //= dbh_new($self); - my ($fid, $sth); + my ($fid, @rng); + my $and_ge_le = ''; if (ref($folder) eq 'HASH') { $fid = $folder->{fid} // die "BUG: no `fid'"; + @rng = grep(defined, @$folder{qw(min max)}); + $and_ge_le = 'AND uid >= ? AND uid <= ?' if @rng; } else { $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder) // return; } - $sth = $dbh->prepare('SELECT oidbin,uid FROM blob2num WHERE fid = ?'); - $sth->execute($fid); - while (my ($oidbin, $id) = $sth->fetchrow_array) { - $cb->($oidbin, $id, @args); + + # minimize implicit txn time to avoid blocking writers by + # batching SELECTs. This looks wonky but is necessary since + # $cb-> may access the DB on its own. + my $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng); +SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ? $and_ge_le +ORDER BY _rowid_ ASC LIMIT 1000 + + my $min = @$ary ? $ary->[-1]->[0] : undef; + while (defined $min) { + for my $row (@$ary) { $cb->($row->[1], $row->[2], @args) } + + $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng, $min); +SELECT _rowid_,oidbin,uid FROM blob2num +WHERE fid = ? $and_ge_le AND _rowid_ > ? +ORDER BY _rowid_ ASC LIMIT 1000 + + $min = @$ary ? $ary->[-1]->[0] : undef; } - $sth = $dbh->prepare('SELECT oidbin,name FROM blob2name WHERE fid = ?'); - $sth->execute($fid); - while (my ($oidbin, $id) = $sth->fetchrow_array) { - $cb->($oidbin, \$id, @args); + + $ary = $dbh->selectall_arrayref(<<'', undef, $fid); +SELECT _rowid_,oidbin,name FROM blob2name WHERE fid = ? +ORDER BY _rowid_ ASC LIMIT 1000 + + $min = @$ary ? $ary->[-1]->[0] : undef; + while (defined $min) { + for my $row (@$ary) { $cb->($row->[1], \($row->[2]), @args) } + + $ary = $dbh->selectall_arrayref(<<'', undef, $fid, $min); +SELECT _rowid_,oidbin,name FROM blob2name WHERE fid = ? AND _rowid_ > ? +ORDER BY _rowid_ ASC LIMIT 1000 + + $min = @$ary ? $ary->[-1]->[0] : undef; } } @@ -200,12 +254,14 @@ sub location_stats { SELECT COUNT(name) FROM blob2name WHERE fid = ? $ret->{'name.count'} = $row if $row; + my $ntype = ($folder =~ m!\A(?:nntps?|s?news)://!i) ? 'article' : + (($folder =~ m!\Aimaps?://!i) ? 'uid' : "TODO<$folder>"); for my $op (qw(count min max)) { ($row) = $dbh->selectrow_array(<<"", undef, $fid); SELECT $op(uid) FROM blob2num WHERE fid = ? $row or last; - $ret->{"uid.$op"} = $row; + $ret->{"$ntype.$op"} = $row; } $ret; } @@ -244,16 +300,19 @@ sub locations_for { # returns a list of folders used for completion sub folders { - my ($self, $pfx) = @_; - my $dbh = $self->{dbh} //= dbh_new($self); + my ($self, @pfx) = @_; my $sql = 'SELECT loc FROM folders'; - my @pfx; - if (defined $pfx) { + if (defined($pfx[0])) { $sql .= ' WHERE loc LIKE ? ESCAPE ?'; - @pfx = ($pfx, '\\'); + my $anywhere = !!$pfx[1]; + $pfx[1] = '\\'; $pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars $pfx[0] .= '%'; + substr($pfx[0], 0, 0, '%') if $anywhere; + } else { + @pfx = (); # [0] may've been undef } + my $dbh = $self->{dbh} //= dbh_new($self); map { $_->[0] } @{$dbh->selectall_arrayref($sql, undef, @pfx)}; } @@ -280,9 +339,12 @@ WHERE b.oidbin = ? next unless -s $fh; local $/; my $raw = <$fh>; - if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) { - warn "$f changed $oidhex\n"; - next; + if ($vrfy) { + my $got = git_sha(1, \$raw)->hexdigest; + if ($got ne $oidhex) { + warn "$f changed $oidhex => $got\n"; + next; + } } return \$raw; } @@ -322,6 +384,30 @@ sub match_imap_url { "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n"; } +sub match_nntp_url ($$$) { + my ($self, $url, $all) = @_; # $all = [ $lms->folders ]; + $all //= [ $self->folders ]; + require PublicInbox::URInntps; + my $want = PublicInbox::URInntps->new($url)->canonical; + my ($s, $h, $p) = ($want->scheme, $want->host, $want->port); + my $ng = $want->group; # force scalar (no article ranges) + my @uri = map { PublicInbox::URInntps->new($_)->canonical } + grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$ng\E\b!, @$all); + my @match; + for my $x (@uri) { + next if $x->group ne $ng || $x->host ne $h || $x->port != $p; + # maybe user was forgotten on CLI: + if (defined($x->userinfo) && !defined($want->userinfo)) { + push @match, $x; + } elsif (($x->userinfo//"\0") eq ($want->userinfo//"\0")) { + push @match, $x; + } + } + return @match if wantarray; + scalar(@match) <= 1 ? $match[0] : + "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n"; +} + # returns undef on failure, number on success sub group2folders { my ($self, $lei, $all, $folders) = @_; @@ -330,6 +416,7 @@ sub group2folders { EOM my %x = map { $_ => $_ } split(/,/, $all); my @ok = grep(defined, delete(@x{qw(local remote), ''})); + push(@ok, '') if $all eq ''; my @no = keys %x; if (@no) { @no = (join(',', @no)); @@ -349,9 +436,7 @@ EOM } else { @inc = @all; } - for (@inc) { - push(@$folders, $_) unless $seen{$_}++; - } + push(@$folders, (grep { !$seen{$_}++ } @inc)); } scalar(@$folders) || $lei->fail(<folders; my %all = map { $_ => 1 } @all; - my ($err, @no); + my @no; for (@$folders) { next if $all{$_}; # ok if (m!\A(maildir|mh):(.+)!i) { @@ -380,11 +465,23 @@ sub arg2folder { my $res = match_imap_url($self, $orig, \@all); if (ref $res) { $_ = $$res; - push(@{$err->{qerr}}, <qerr(<qerr(<err($res) if defined $res; + warn($res, "\n") if defined $res; push @no, $orig; } } else { @@ -393,33 +490,58 @@ EOM } if (@no) { my $no = join("\n\t", @no); - $err->{fail} = <{fmap}->{$folder}) // - fid_for($self, $folder) // return; - my $dbh = $self->{dbh}; - $dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid); - $dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid); - $dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid); +sub forget_folders { + my ($self, @folders) = @_; + my $lk = $self->lock_for_scope; + for my $folder (@folders) { + my $fid = delete($self->{fmap}->{$folder}) // + fid_for($self, $folder) // next; + for my $t (qw(blob2name blob2num folders)) { + $self->{dbh}->do("DELETE FROM $t WHERE fid = ?", + undef, $fid); + } + } +} + +# only used for changing canonicalization errors +sub rename_folder { + my ($self, $old, $new) = @_; + my $lk = $self->lock_for_scope; + my $ofid = delete($self->{fmap}->{$old}) // + fid_for($self, $old) // return; + eval { + $self->{dbh}->do(<{fmap}->{$new} // fid_for($self, $new); + for my $t (qw(blob2name blob2num)) { + $self->{dbh}->do(<{dbh}->do(<{fmap}->{$url} //= fid_for($self, $url) // return; +sub num_oidbin ($$$) { + my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY if IMAP + my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return (); my $sth = $self->{dbh}->prepare_cached(<execute($fid, $uid); - $sth->fetchrow_array; + map { $_->[0] } @{$sth->fetchall_arrayref}; } sub name_oidbin ($$$) { @@ -429,33 +551,17 @@ sub name_oidbin ($$$) { SELECT oidbin FROM blob2name WHERE fid = ? AND name = ? EOM $sth->execute($fid, $nm); - $sth->fetchrow_array; + map { $_->[0] } @{$sth->fetchall_arrayref}; } -sub imap_oid { +sub imap_oidhex { my ($self, $lei, $uid_uri) = @_; my $mailbox_uri = $uid_uri->clone; $mailbox_uri->uid(undef); my $folders = [ $$mailbox_uri ]; - if (my $err = $self->arg2folder($lei, $folders)) { - if ($err->{fail}) { - $lei->qerr("# no sync information for $mailbox_uri"); - return; - } - $lei->qerr(@{$err->{qerr}}) if $err->{qerr}; - } - my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid); - $oidbin ? unpack('H*', $oidbin) : undef; -} - -# FIXED? something with "lei " is causing uncommitted transaction -# TODO: remove soon -sub DESTROY { - my ($self) = @_; - my $dbh = delete($self->{dbh}) or return; - return if $dbh->{ReadOnly}; - undef $dbh; - warn "BUG $$ $0 $self {dbh} OPEN ppid=".getppid.' '.Carp::longmess(); + eval { $self->arg2folder($lei, $folders) }; + $lei->qerr("# no sync information for $mailbox_uri") if $@; + map { unpack('H*',$_) } num_oidbin($self, $folders->[0], $uid_uri->uid) } 1;