clone: support --include and --exclude with multi-clone

These will be handy when someone is interested in a subset of
inboxes on a large hosting site.

diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
index c80c3c5fded6dc156228ae95ebc5111d2b1ddccc..7e95146e44a781a30f5f131ad034401ab6d5cbf1 100644 (file)
--- a/Documentation/public-inbox-clone.pod
+++ b/Documentation/public-inbox-clone.pod
@@ -51,6 +51,20 @@ C<--epoch=~2..> clones the three latest epochs.
 Default: C<0..~0> or C<0..> or C<..~0>
 (all epochs, all three examples are equivalent)
 
+=item -I PATTERN
+
+=item --include=PATTERN
+
+When cloning a top-level with multiple inboxes, only clone inboxes and
+repositories matching a given wildcard pattern (using C<*?> and C<[]> is
+supported).
+
+=item --exclude=PATTERN
+
+When cloning a top-level with multiple inboxes, ignore inboxes and
+repositories matching the given wildcard pattern.  Supports the same
+wildcards as L</--include>
+
 =item -q
 
 =item --quiet

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index e356b5c57e553ce9d8f4b22619660e10d9fe7821..d501764213c42a4fadf5b873128cc943bd7ec4b7 100644 (file)
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -347,6 +347,8 @@ sub decode_manifest ($$$) {
 
 sub multi_inbox ($$$) {
        my ($self, $path, $m) = @_;
+       my $incl = $self->{lei}->{opt}->{include};
+       my $excl = $self->{lei}->{opt}->{exclude};
 
        # assuming everything not v2 is v1, for now
        my @v1 = sort grep(!m!.+/git/[0-9]+\.git\z!, keys %$m);
@@ -354,13 +356,35 @@ sub multi_inbox ($$$) {
        my $v2 = {};
 
        for (@v2_epochs) {
-               m!\A/(.+)/git/[0-9]+\.git\z! or die "BUG: $_";
+               m!\A(/.+)/git/[0-9]+\.git\z! or die "BUG: $_";
                push @{$v2->{$1}}, $_;
        }
        my $n = scalar(keys %$v2) + scalar(@v1);
-       my $ret; # { v1 => [ ... ], v2 => { $inbox_name => [ epochs ] }}
+       my @orig = defined($incl // $excl) ? (keys %$v2, @v1) : ();
+       if (defined $incl) {
+               my $re = '(?:'.join('|', map {
+                               $self->{lei}->glob2re($_) // qr/\A\Q$_\E\z/
+                       } @$incl).')';
+               my @gone = delete @$v2{grep(!/$re/, keys %$v2)};
+               delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1;
+               delete @$m{grep(!/$re/, @v1)} and $self->{-culled_manifest} = 1;
+               @v1 = grep(/$re/, @v1);
+       }
+       if (defined $excl) {
+               my $re = '(?:'.join('|', map {
+                               $self->{lei}->glob2re($_) // qr/\A\Q$_\E\z/
+                       } @$excl).')';
+               my @gone = delete @$v2{grep(/$re/, keys %$v2)};
+               delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1;
+               delete @$m{grep(/$re/, @v1)} and $self->{-culled_manifest} = 1;
+               @v1 = grep(!/$re/, @v1);
+       }
+       my $ret; # { v1 => [ ... ], v2 => { "/$inbox_name" => [ epochs ] }}
        $ret->{v1} = \@v1 if @v1;
        $ret->{v2} = $v2 if keys %$v2;
+       $ret //= @orig ? "Nothing to clone, available repositories:\n\t".
+                               join("\n\t", sort @orig)
+                       : "Nothing available to clone\n";
        my $path_pfx = '';
 
        # PSGI mount prefixes and manifest.js.gz prefixes don't always align...
@@ -407,6 +431,7 @@ sub try_manifest {
                return try_scrape($self);
        }
        my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m);
+       return $lei->child_error(1, $multi) if !ref($multi);
        if (my $v2 = delete $multi->{v2}) {
                for my $name (sort keys %$v2) {
                        my $epochs = delete $v2->{$name};
@@ -449,7 +474,7 @@ EOM
                        clone_v1($self, 1);
                }
        }
-       if (delete $self->{-culled_manifest}) { # set by clone_v2
+       if (delete $self->{-culled_manifest}) { # set by clone_v2/-I/--exclude
                # write the smaller manifest if epochs were skipped so
                # users won't have to delete manifest if they +w an
                # epoch they no longer want to skip

diff --git a/script/public-inbox-clone b/script/public-inbox-clone
index 54059d036f8675cb89636a93e0440c9b84b87c86..4244e0c8cc688a9fbb3e8f9d224f3575895a0547 100755 (executable)
--- a/script/public-inbox-clone
+++ b/script/public-inbox-clone
@@ -21,7 +21,7 @@ options:
     --quiet | -q      increase verbosity (may be repeated)
     -C DIR            chdir to specified directory
 EOF
-GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@ include|I=s@ exclude=s@
                no-torsocks torsocks=s epoch=s)) or die $help;
 if ($opt->{help}) { print $help; exit };
 require PublicInbox::Admin; # loads Config

diff --git a/t/www_listing.t b/t/www_listing.t
index e88bfbc5b003690f23e31d3888b02ec3787dcd39..e6bb1bda27baf66db53bee30622c2355a6ee6343 100644 (file)
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -135,8 +135,29 @@ EOM
        my $opt = { 2 => \(my $clone_err = '') };
        ok(run_script(['-clone', "http://$host:$port/pfx", "$tmpdir/pfx" ],
                undef, $opt), 'pfx clone w/pfx') or diag "clone_err=$clone_err";
+
+       open my $mh, '<', "$tmpdir/pfx/manifest.js.gz" or xbail "open: $!";
+       gunzip(\(do { local $/; <$mh> }) => \(my $mjs = ''));
+       my $mf = $json->decode($mjs);
+       is_deeply([sort keys %$mf], [ qw(/alt /bare /v2/git/0.git
+                                       /v2/git/1.git /v2/git/2.git) ],
+               'manifest saved');
+       for (keys %$mf) { ok(-d "$tmpdir/pfx$_", "pfx/$_ cloned") }
+
+       $clone_err = '';
+       ok(run_script(['-clone', '--include=*/alt',
+                       "http://$host:$port/pfx", "$tmpdir/incl" ],
+               undef, $opt), 'clone w/include') or diag "clone_err=$clone_err";
+       ok(-d "$tmpdir/incl/alt", 'alt cloned');
+       ok(!-d "$tmpdir/incl/v2" && !-d "$tmpdir/incl/bare", 'only alt cloned');
+
        undef $td;
 
+       open $mh, '<', "$tmpdir/incl/manifest.js.gz" or xbail "open: $!";
+       gunzip(\(do { local $/; <$mh> }) => \($mjs = ''));
+       $mf = $json->decode($mjs);
+       is_deeply([keys %$mf], [ '/alt' ], 'excluded keys skipped in manifest');
+
        $td = start_script($cmd, $env, { 3 => $sock });
 
        # default publicinboxGrokManifest match=domain default
@@ -146,6 +167,7 @@ EOM
        $clone_err = '';
        ok(run_script(['-clone', "http://$host:$port/", "$tmpdir/full" ],
                undef, $opt), 'full clone') or diag "clone_err=$clone_err";
+       ok(-d "$tmpdir/full/$_", "$_ cloned") for qw(alt v2 bare);
 
        undef $td;