lei up: automatically use dt: for remote externals

[public-inbox.git] / lib / PublicInbox / LEI.pm

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec10323155cd8cbeeea258b7acb732f454fd72cf..8b0614f27980f8e4e53f31681562ae25cc83b119 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -183,7 +183,8 @@ our %CMD = ( # sorted in order of importance/use:
        shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'up' => [ 'OUTPUT...|--all', 'update saved search',
-       qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+       qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
+       remote-fudge-time=s all:s), @c_opt ],
 
 'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
        'stdin|', # /|\z/ must be first for lone dash
@@ -258,12 +259,12 @@ our %CMD = ( # sorted in order of importance/use:
         @c_opt ],
 'import' => [ 'LOCATION...|--stdin',
        'one-time import/update from URL or filesystem',
-       qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+       qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
        lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
        @net_opt, @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
        'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
        'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
 'export-kw' => [ 'LOCATION...|--all',
        'one-time export of keywords of sync sources',
@@ -420,7 +421,9 @@ my %OPTDESC = (
 'remote' => 'limit operations to those requiring network access',
 'remote!' => 'prevent operations requiring network access',
 
-'all:s up' => ['local', 'update all (local) saved searches' ],
+'all:s up' => ['local|remote', 'update all remote or local saved searches' ],
+'remote-fudge-time=s' => [ 'INTERVAL',
+       'look for mail INTERVAL older than the last successful query' ],
 
 'mid=s' => 'specify the Message-ID of a message',
 'oid=s' => 'specify the git object ID of a message',
@@ -616,6 +619,7 @@ sub pkt_ops {
        $ops->{x_it} = [ \&x_it, $lei ];
        $ops->{child_error} = [ \&child_error, $lei ];
        $ops->{incr} = [ \&incr, $lei ];
+       $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
        $ops;
 }
 
@@ -626,6 +630,7 @@ sub workers_start {
        my $end = $lei->pkt_op_pair;
        my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
        $flds->{lei} = $lei;
+       $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
        $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
        delete $lei->{pkt_op_p};
        my $op_c = delete $lei->{pkt_op_c};
@@ -1440,20 +1445,16 @@ sub refresh_watches {
        }
 
        # add all known Maildir folders as implicit watches
-       my $sto = $lei->_lei_store;
-       my $renames = 0;
-       if (my $lms = $sto ? $sto->search->lms : undef) {
+       my $lms = $lei->lms;
+       if ($lms) {
+               $lms->lms_write_prepare;
                for my $d ($lms->folders('maildir:')) {
                        substr($d, 0, length('maildir:')) = '';
-                       my $cd = canonpath_harder($d);
-                       my $f = "maildir:$cd";
 
                        # fixup old bugs while we're iterating:
-                       if ($d ne $cd) {
-                               $sto->ipc_do('lms_rename_folder',
-                                               "maildir:$d", $f);
-                               ++$renames;
-                       }
+                       my $cd = canonpath_harder($d);
+                       my $f = "maildir:$cd";
+                       $lms->rename_folder("maildir:$d", $f) if $d ne $cd;
                        next if $watches->{$f}; # may be set to pause
                        require PublicInbox::LeiWatch;
                        $watches->{$f} = PublicInbox::LeiWatch->new($f);
@@ -1461,7 +1462,6 @@ sub refresh_watches {
                        add_maildir_watch($cd, $cfg_f);
                }
        }
-       $lei->sto_done_request if $renames;
        if ($old) { # cull old non-existent entries
                for my $url (keys %$old) {
                        next if exists $seen{$url};
@@ -1488,13 +1488,12 @@ sub git_oid {
        git_sha(1, $eml);
 }
 
-sub lms { # read-only LeiMailSync
-       my ($lei) = @_;
-       my $lse = $lei->{lse} // do {
-               my $sto = $lei->{sto} // _lei_store($lei);
-               $sto ? $sto->search : undef
-       };
-       $lse ? $lse->lms : undef;
+sub lms {
+       my ($lei, $rw) = @_;
+       my $sto = $lei->{sto} // _lei_store($lei) // return;
+       require PublicInbox::LeiMailSync;
+       my $f = "$sto->{priv_eidx}->{topdir}/mail_sync.sqlite3";
+       (-f $f || $rw) ? PublicInbox::LeiMailSync->new($f) : undef;
 }
 
 sub sto_done_request { # only call this from lei-daemon process (not workers)