lei import: speed up kw updates for old IMAP messages

[public-inbox.git] / lib / PublicInbox / LeiMailSync.pm

diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index d9c305800bad91bb4eab51f45e6609dec3afcc31..75603d89e1788127487e83a5b396b882d60ba8c1 100644 (file)
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num (
        UNIQUE (oidbin, fid, uid)
 )
 
+       # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x:
+       $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid)
+
        $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS blob2name (
        oidbin VARBINARY NOT NULL,
@@ -64,9 +68,9 @@ CREATE TABLE IF NOT EXISTS blob2name (
 
 }
 
-sub _fid_for {
+sub fid_for {
        my ($self, $folder, $rw) = @_;
-       my $dbh = $self->{dbh};
+       my $dbh = $self->{dbh} //= dbh_new($self, $rw);
        my $sel = 'SELECT fid FROM folders WHERE loc = ? LIMIT 1';
        my ($fid) = $dbh->selectrow_array($sel, undef, $folder);
        return $fid if defined $fid;
@@ -111,7 +115,7 @@ EOM
 
 sub set_src {
        my ($self, $oidhex, $folder, $id) = @_;
-       my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+       my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
        my $sth;
        if (ref($id)) { # scalar name
                $id = $$id;
@@ -128,7 +132,7 @@ INSERT OR IGNORE INTO blob2num (oidbin, fid, uid) VALUES (?, ?, ?)
 
 sub clear_src {
        my ($self, $folder, $id) = @_;
-       my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+       my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
        my $sth;
        if (ref($id)) { # scalar name
                $id = $$id;
@@ -146,7 +150,7 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
 # Maildir-only
 sub mv_src {
        my ($self, $folder, $oidbin, $id, $newbn) = @_;
-       my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+       my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
        my $sth = $self->{dbh}->prepare_cached(<<'');
 UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
 
@@ -158,7 +162,12 @@ sub each_src {
        my ($self, $folder, $cb, @args) = @_;
        my $dbh = $self->{dbh} //= dbh_new($self);
        my ($fid, $sth);
-       $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+       if (ref($folder) eq 'HASH') {
+               $fid = $folder->{fid} // die "BUG: no `fid'";
+       } else {
+               $fid = $self->{fmap}->{$folder} //=
+                       fid_for($self, $folder) // return;
+       }
        $sth = $dbh->prepare('SELECT oidbin,uid FROM blob2num WHERE fid = ?');
        $sth->execute($fid);
        while (my ($oidbin, $id) = $sth->fetchrow_array) {
@@ -176,7 +185,7 @@ sub location_stats {
        my $dbh = $self->{dbh} //= dbh_new($self);
        my $fid;
        my $ret = {};
-       $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+       $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder) // return;
        my ($row) = $dbh->selectrow_array(<<"", undef, $fid);
 SELECT COUNT(name) FROM blob2name WHERE fid = ?
 
@@ -349,11 +358,48 @@ sub forget_folder {
        my ($self, $folder) = @_;
        my ($fid, $sth);
        $fid = delete($self->{fmap}->{$folder}) //
-               _fid_for($self, $folder) // return;
+               fid_for($self, $folder) // return;
        my $dbh = $self->{dbh};
        $dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid);
        $dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid);
        $dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
 }
 
+sub imap_oidbin ($$$) {
+       my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
+       my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
+       my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+       $sth->execute($fid, $uid);
+       $sth->fetchrow_array;
+}
+
+sub imap_oid {
+       my ($self, $lei, $uid_uri) = @_;
+       my $mailbox_uri = $uid_uri->clone;
+       $mailbox_uri->uid(undef);
+       my $folders = [ $$mailbox_uri ];
+       if (my $err = $self->arg2folder($lei, $folders)) {
+               if ($err->{fail}) {
+                       $lei->qerr("# no sync information for $mailbox_uri");
+                       return;
+               }
+               $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+       }
+       my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid);
+       $oidbin ? unpack('H*', $oidbin) : undef;
+}
+
+# FIXME: something with "lei <up|q>" is causing uncommitted transaction
+# warnings, not sure what...
+sub DESTROY {
+       my ($self) = @_;
+       my $dbh = delete($self->{dbh}) or return;
+       return if $dbh->{ReadOnly};
+       use Carp;
+       undef $dbh;
+       warn "BUG $$ $0 $self {dbh} OPEN ppid=".getppid.' '.Carp::longmess();
+}
+
 1;