lei: avoid SQLite COUNT() for dedupe

SQLite COUNT() is a slow operation that does a full table scan
with no conditions.  There's no need for it, since lei dedupe
only needs to know if it's empty or not to decide between
new/ and cur/ for Maildir outputs.

diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index ed52e41755b828467867642acf93bca6c72dc2c7..32f99cd0a30c3eafeed64940c7a0bd21940861ad 100644 (file)
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -127,10 +127,9 @@ sub pause_dedupe {
        delete($skv->{dbh}) if $skv;
 }
 
-sub dedupe_nr {
+sub has_entries {
        my $skv = $_[0]->[0] or return undef;
-       my @n = $skv->count;
-       $n[0];
+       $skv->has_entries;
 }
 
 1;

diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 929380edc6fd821920d018f245128d6fe35260e4..cfbf68c3491a9405480fa42e4f270de90e36fa92 100644 (file)
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -315,11 +315,11 @@ E: rename($dir_old, $dir_new) error: $!
 EOM
 }
 
-# cf. LeiDedupe->dedupe_nr
-sub dedupe_nr {
+# cf. LeiDedupe->has_entries
+sub has_entries {
        my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
-       my @n = $oidx->{dbh}->selectrow_array('SELECT COUNT(*) FROM over');
-       $n[0];
+       my @n = $oidx->{dbh}->selectrow_array('SELECT num FROM over LIMIT 1');
+       scalar(@n) ? 1 : undef;
 }
 
 no warnings 'once';

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9405c0c68047165f5bfcb80cded011ce274ae81..d782d4c793c7e7ca97455b4a985ed60172b443a9 100644 (file)
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -198,7 +198,7 @@ sub _mbox_write_cb ($$) {
        my $dedupe = $lei->{dedupe};
        $dedupe->prepare_dedupe;
        my $lse = $lei->{lse}; # may be undef
-       my $set_recent = $dedupe->dedupe_nr;
+       my $set_recent = $dedupe->has_entries;
        sub { # for git_to_mail
                my ($buf, $smsg, $eml) = @_;
                $eml //= PublicInbox::Eml->new($buf);
@@ -293,7 +293,7 @@ sub _maildir_write_cb ($$) {
        # Favor cur/ and only write to new/ when augmenting.  This
        # saves MUAs from having to do a mass rename when the initial
        # search result set is huge.
-       my $dir = $dedupe && $dedupe->dedupe_nr ? 'new/' : 'cur/';
+       my $dir = $dedupe && $dedupe->has_entries ? 'new/' : 'cur/';
        sub { # for git_to_mail
                my ($bref, $smsg, $eml) = @_;
                $dst // return $lei->fail; # dst may be undef-ed in last run

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index cac7fb7d5902c9aa51cbd40b395721fe4768a9f3..3414e87dd41ec917021bd017f96efeda1cb39b0a 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -504,7 +504,7 @@ sub do_query {
                my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
                if ($l2m->{-wq_nr_workers} > 1 &&
                                $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
-                       # setup two barriers to coordinate dedupe_nr
+                       # setup two barriers to coordinate ->has_entries
                        # between l2m workers
                        pipe(my ($a_r, $a_w)) or die "pipe: $!";
                        fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;

diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 8347b1956c8124f9adab6ec200d37a6e21652ed3..3487e82086f47e2049d29b7f7d74bb8a3806aa33 100644 (file)
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -154,6 +154,13 @@ SELECT COUNT(k) FROM kv
        $sth->fetchrow_array;
 }
 
+# faster than ->count due to how SQLite works
+sub has_entries {
+       my ($self) = @_;
+       my @n = $self->{dbh}->selectrow_array('SELECT k FROM kv LIMIT 1');
+       scalar(@n) ? 1 : undef;
+}
+
 sub dbh_release {
        my ($self, $lock) = @_;
        my $dbh = delete $self->{dbh} or return;