lei import: --incremental default for NNTP and IMAP

No point in burning through bandwidth to import stuff we already
saw.  All this logic is shared with -watch but uses a different
pathname for lei since it's tied to lei/store (and not a
public-inbox).

diff --git a/Documentation/lei-store-format.pod b/Documentation/lei-store-format.pod
index a42c770ee7d4232bd2e5698db321894794a5614a..3e1ddc658579b6cbcc2873157713c33ff805e8ff 100644 (file)
--- a/Documentation/lei-store-format.pod
+++ b/Documentation/lei-store-format.pod
@@ -32,6 +32,7 @@ prevent them from being accidentally treated as a v2 inbox.
   ~/.local/share/lei/store
   - ipc.lock                        # lock file for internal lei IPC
   - local/$EPOCH.git                # normal bare git repositories
+  - net_last.sqlite3                # import state for IMAP & NNTP
 
 Additionally, the following share the same roles they do in extindex:
 

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 2e1aa2469559ba49fbee00d827ab3017474c3cb0..d9e644ebe637070755815329c24d1a05f3393429 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -193,7 +193,8 @@ our %CMD = ( # sorted in order of importance/use:
 'import' => [ 'LOCATION...|--stdin',
        'one-time import/update from URL or filesystem',
        qw(stdin| offset=i recursive|r exclude=s include|I=s
-       lock=s@ in-format|F=s kw|keywords|flags! verbose|v+), @c_opt ],
+       lock=s@ in-format|F=s kw|keywords|flags! verbose|v+
+       incremental!), @c_opt ],
 'convert' => [ 'LOCATION...|--stdin',
        'one-time conversion from URL or filesystem to another format',
        qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
@@ -244,6 +245,7 @@ my %OPTDESC = (
 'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
        'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
 
+'incremental!  import' => 'import already seen IMAP and NNTP articles',
 'globoff|g' => "do not match locations using '*?' wildcards ".
                "and\xa0'[]'\x{a0}ranges",
 'verbose|v+' => 'be more verbose',

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 1627160387ba8257fa97bf27ecbc68b701b93237..accf08f52b43d62567dd5b7179be1f034d9190c1 100644 (file)
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -58,6 +58,11 @@ sub lei_import { # the main "lei import" method
        my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
        if (my $net = $lei->{net}) {
                # $j = $net->net_concurrency($j); TODO
+               if ($lei->{opt}->{incremental} // 1) {
+                       $net->{incremental} = 1;
+                       $net->{itrk_fn} = $lei->store_path .
+                                               '/net_last.sqlite3';
+               }
        } else {
                my $nproc = $self->detect_nproc;
                $j = $nproc if $j > $nproc;

diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 0ef66fd84cb8bf56cd7737cacd2b059d1b30969b..c7b43f014f6bd31d5c2e435f8ef3b71640cbfa17 100644 (file)
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -373,6 +373,13 @@ sub run_commit_cb ($) {
        $cb->(@args);
 }
 
+sub _itrk ($$) {
+       my ($self, $uri) = @_;
+       return unless $self->{incremental};
+       # itrk_fn is set by lei
+       PublicInbox::IMAPTracker->new($$uri, $self->{itrk_fn});
+}
+
 sub _imap_fetch_all ($$$) {
        my ($self, $mic, $uri) = @_;
        my $sec = uri_section($uri);
@@ -389,8 +396,7 @@ sub _imap_fetch_all ($$$) {
                return "E: $uri cannot get UIDVALIDITY";
        $r_uidnext //= $mic->uidnext($mbx) //
                return "E: $uri cannot get UIDNEXT";
-       my $itrk = $self->{incremental} ?
-                       PublicInbox::IMAPTracker->new($$uri) : 0;
+       my $itrk = _itrk($self, $uri);
        my ($l_uidval, $l_uid) = $itrk ? $itrk->get_last : ();
        $l_uidval //= $r_uidval; # first time
        $l_uid //= 0;
@@ -543,8 +549,7 @@ sub _nntp_fetch_all ($$$) {
        # IMAPTracker is also used for tracking NNTP, UID == article number
        # LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
        # expensive.  So we assume newsgroups don't change:
-       my $itrk = $self->{incremental} ?
-                       PublicInbox::IMAPTracker->new($$uri) : 0;
+       my $itrk = _itrk($self, $uri);
        my (undef, $l_art) = $itrk ? $itrk->get_last : ();
 
        # allow users to specify articles to refetch

diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 7e4d44b9d246f14dc3267cf53851002c5eaf9db8..490ea9beaa05d2e251d3ee33daa8908e191c767d 100644 (file)
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -24,5 +24,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
        for (@$out) { $r{ref($_)}++ }
        is_deeply(\%r, { 'HASH' => scalar(@$out) }, 'all hashes');
        lei_ok([qw(tag +kw:seen), "imap://$host_port/t.v2.0"], undef, undef);
+
+       my $f = "$ENV{HOME}/.local/share/lei/store/net_last.sqlite3";
+       ok(-s $f, 'net tracked for redundant imports');
 });
 done_testing;

diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 1fc6dbadfc90a546a1e95ad2f4857f87a6743cad..d795a86a8486e190f54cb456fe87c72b180bbc61 100644 (file)
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -26,5 +26,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
        my %r;
        for (@$out) { $r{ref($_)}++ }
        is_deeply(\%r, { 'HASH' => scalar(@$out) }, 'all hashes');
+
+       my $f = "$ENV{HOME}/.local/share/lei/store/net_last.sqlite3";
+       ok(-s $f, 'net tracked for redundant imports');
 });
 done_testing;