lei import: help + completion for --new-only

[public-inbox.git] / lib / PublicInbox / LEI.pm

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 265b7047fdf63541fb0bd263071efc445e674c26..546fa773fc4c334cc83a27358ba74f016cf62dab 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -116,6 +116,12 @@ sub cache_dir ($) {
                .'/lei');
 }
 
+sub url_folder_cache {
+       my ($self) = @_;
+       require PublicInbox::SharedKV; # URI => updated_at_sec_
+       PublicInbox::SharedKV->new(cache_dir($self).'/uri_folder');
+}
+
 sub ale {
        my ($self) = @_;
        $self->{ale} //= do {
@@ -196,6 +202,8 @@ our %CMD = ( # sorted in order of importance/use:
 'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
 'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
                qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
+'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
+               qw(z|0 ascii l url), @c_opt ],
 'forget-external' => [ 'LOCATION...|--prune',
        'exclude further results from a publicinbox|extindex',
        qw(prune), @c_opt ],
@@ -240,7 +248,7 @@ our %CMD = ( # sorted in order of importance/use:
         @c_opt ],
 'import' => [ 'LOCATION...|--stdin',
        'one-time import/update from URL or filesystem',
-       qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
+       qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
        lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
        qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
@@ -345,6 +353,7 @@ my %OPTDESC = (
        "or\xa0`-'\x{a0}for\x{a0}stdout)" ],
 'mua=s' => [ 'CMD',
        "MUA to run on --output Maildir or mbox (e.g.\xa0`mutt\xa0-f\xa0%f')" ],
+'new-only      import' => 'only import new messages from IMAP source',
 
 'inbox-version=i' => [ 'NUM|1|2',
                'force a public-inbox version with --mirror'],
@@ -381,6 +390,8 @@ my %OPTDESC = (
 'format|f=s    ls-search' => ['OUT|json|jsonl|concatjson',
                        'listing output format' ],
 'l     ls-search' => 'long listing format',
+'l     ls-mail-source' => 'long listing format',
+'url   ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
 'format|f=s    ls-external' => $ls_format,
 
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
@@ -443,6 +454,7 @@ sub x_it ($$) {
        # make sure client sees stdout before exit
        $self->{1}->autoflush(1) if $self->{1};
        dump_and_clear_log();
+       stop_pager($self);
        if ($self->{pkt_op_p}) { # to top lei-daemon
                $self->{pkt_op_p}->pkt_do('x_it', $code);
        } elsif ($self->{sock}) { # to lei(1) client
@@ -567,6 +579,11 @@ sub pkt_op_pair {
        $end;
 }
 
+sub incr {
+       my ($self, $field, $nr) = @_;
+       $self->{counters}->{$field} += $nr;
+}
+
 sub workers_start {
        my ($lei, $wq, $jobs, $ops, $flds) = @_;
        $ops = {
@@ -574,6 +591,7 @@ sub workers_start {
                '|' => [ \&sigpipe_handler, $lei ],
                'x_it' => [ \&x_it, $lei ],
                'child_error' => [ \&child_error, $lei ],
+               'incr' => [ \&incr, $lei ],
                ($ops ? %$ops : ()),
        };
        $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
@@ -583,8 +601,6 @@ sub workers_start {
        $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
        delete $lei->{pkt_op_p};
        my $op_c = delete $lei->{pkt_op_c};
-       # {-lei_sock} persists script/lei process until ops->{''} EOF callback
-       $op_c->{-lei_sock} = $lei->{sock};
        @$end = ();
        $lei->event_step_init;
        ($op_c, $ops);
@@ -1092,10 +1108,11 @@ sub event_step {
 
 sub event_step_init {
        my ($self) = @_;
-       return if $self->{-event_init_done}++;
-       if (my $sock = $self->{sock}) { # using DS->EventLoop
+       my $sock = $self->{sock} or return;
+       $self->{-event_init_done} //= do { # persist til $ops done
                $self->SUPER::new($sock, EPOLLIN|EPOLLET);
-       }
+               $sock;
+       };
 }
 
 sub noop {}
@@ -1246,6 +1263,12 @@ sub busy { 1 } # prevent daemon-shutdown if client is connected
 # can immediately reread it
 sub DESTROY {
        my ($self) = @_;
+       if (my $counters = delete $self->{counters}) {
+               for my $k (sort keys %$counters) {
+                       my $nr = $counters->{$k};
+                       $self->child_error(1 << 8, "$nr $k messages");
+               }
+       }
        $self->{1}->autoflush(1) if $self->{1};
        stop_pager($self);
        # preserve $? for ->fail or ->x_it code