Documentation/public-inbox-clone.pod |  71 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 Documentation/public-inbox-fetch.pod |  63 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 MANIFEST                             |   5 +++++
 lib/PublicInbox/Admin.pm             |   8 ++++++++
 lib/PublicInbox/Fetch.pm             | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/PublicInbox/LEI.pm               |   6 +++++-
 lib/PublicInbox/LeiMirror.pm         |  95 +++++++++++++++++++++++++++++++----------------------
 script/public-inbox-clone            |  58 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 script/public-inbox-fetch            |  35 +++++++++++++++++++++++++++++++++++
 t/lei-mirror.t                       |  29 +++++++++++++++++++++++++++++

diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
new file mode 100644
index 0000000000000000000000000000000000000000..fdb57663ceb97db93187ac02c0d27896f03f8256
--- /dev/null
+++ b/Documentation/public-inbox-clone.pod
@@ -0,0 +1,71 @@
+=head1 NAME
+
+public-inbox-clone - "git clone --mirror" wrapper
+
+=head1 SYNOPSIS
+
+public-inbox-clone INBOX_URL [INBOX_DIR]
+
+=head1 DESCRIPTION
+
+public-inbox-clone is a wrapper around C<git clone --mirror> for
+making the initial clone of a remote HTTP(S) public-inbox.  It
+allows cloning multi-epoch v2 inboxes with a single command and
+zero configuration.
+
+It does not run L<public-inbox-init(1)> nor
+L<public-inbox-index(1)>.  Those commands must be run separately
+if serving/searching the mirror is required.  As-is,
+public-inbox-clone is suitable for creating a git-only backup.
+
+public-inbox-clone does not use nor require any extra
+configuration files (not even C<~/.public-inbox/config>).
+
+L<public-inbox-fetch(1)> may be used to keep C<INBOX_DIR>
+up-to-date.
+
+For v2 inboxes, it will create a C<$INBOX_DIR/manifest.js.gz>
+file to speed up subsequent L<public-inbox-fetch(1)>.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Quiets down progress messages, also passed to L<git-fetch(1)>.
+
+=item -v
+
+=item --verbose
+
+Increases verbosity, also passed to L<git-fetch(1)>.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-fetch(1)>, L<public-inbox-init(1)>, L<public-inbox-index(1)>
diff --git a/Documentation/public-inbox-fetch.pod b/Documentation/public-inbox-fetch.pod
new file mode 100644
index 0000000000000000000000000000000000000000..7944fdcdcace3876ba935c22fc909d3f4f7eb0d6
--- /dev/null
+++ b/Documentation/public-inbox-fetch.pod
@@ -0,0 +1,63 @@
+=head1 NAME
+
+public-inbox-fetch - "git fetch" wrapper for v2 inbox mirrors
+
+=head1 SYNOPSIS
+
+public-inbox-fetch -C INBOX_DIR
+
+=head1 DESCRIPTION
+
+public-inbox-fetch updates git storage of public-inbox mirrors.
+With v2 inboxes, it allows detection of new epochs and avoids
+unnecessary traffic on old epochs.
+
+public-inbox-fetch does not use nor require any configuration
+files of its own.
+
+It does not run L<public-inbox-index(1)>, making it suitable
+for maintaining git-only backups.
+
+For v2 inboxes, it will maintain C<$INBOX_DIR/manifest.js.gz>
+file to speed up future invocations.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Quiets down progress messages, also passed to L<git-fetch(1)>.
+
+=item -v
+
+=item --verbose
+
+Increases verbosity, also passed to L<git-fetch(1)>.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-index(1)>
diff --git a/MANIFEST b/MANIFEST
index c64f7d94fab3f582696d44e1f9de9e6cbafc9b8e..a14508807f7be0cdbb26332691e1dcd9485c626f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -51,6 +51,7 @@ Documentation/lei.pod
 Documentation/lei_design_notes.txt
 Documentation/marketing.txt
 Documentation/mknews.perl
+Documentation/public-inbox-clone.pod
 Documentation/public-inbox-compact.pod
 Documentation/public-inbox-config.pod
 Documentation/public-inbox-convert.pod
@@ -58,6 +59,7 @@ Documentation/public-inbox-daemon.pod
 Documentation/public-inbox-edit.pod
 Documentation/public-inbox-extindex-format.pod
 Documentation/public-inbox-extindex.pod
+Documentation/public-inbox-fetch.pod
 Documentation/public-inbox-glossary.pod
 Documentation/public-inbox-httpd.pod
 Documentation/public-inbox-imapd.pod
@@ -163,6 +165,7 @@ lib/PublicInbox/ExtSearchIdx.pm
 lib/PublicInbox/FakeImport.pm
 lib/PublicInbox/FakeInotify.pm
 lib/PublicInbox/Feed.pm
+lib/PublicInbox/Fetch.pm
 lib/PublicInbox/Filter/Base.pm
 lib/PublicInbox/Filter/Gmane.pm
 lib/PublicInbox/Filter/Mirror.pm
@@ -329,10 +332,12 @@ sa_config/README
 sa_config/root/etc/spamassassin/public-inbox.pre
 sa_config/user/.spamassassin/user_prefs
 script/lei
+script/public-inbox-clone
 script/public-inbox-compact
 script/public-inbox-convert
 script/public-inbox-edit
 script/public-inbox-extindex
+script/public-inbox-fetch
 script/public-inbox-httpd
 script/public-inbox-imapd
 script/public-inbox-index
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 2534958b121e691ecee1e963fde736d03383c30f..9ff59bca3eea304198090585e93eb6f2946853a0 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -372,4 +372,12 @@ 	}
 	$env;
 }
 
+sub do_chdir ($) {
+	my $chdir = $_[0] // return;
+	for my $d (@$chdir) {
+		next if $d eq ''; # same as git(1)
+		chdir $d or die "cd $d: $!";
+	}
+}
+
 1;
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
new file mode 100644
index 0000000000000000000000000000000000000000..d795731ccb62eed5d9d2fe8a43c8ae792f56bd9b
--- /dev/null
+++ b/lib/PublicInbox/Fetch.pm
@@ -0,0 +1,145 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to "git fetch" remote public-inboxes
+package PublicInbox::Fetch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+use URI ();
+use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Admin;
+use PublicInbox::LEI;
+use PublicInbox::LeiCurl;
+use PublicInbox::LeiMirror;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+use File::Temp ();
+
+sub new { bless {}, __PACKAGE__ }
+
+sub fetch_cmd ($$) {
+	my ($lei, $opt) = @_;
+	my @cmd = qw(git);
+	$opt->{$_} = $lei->{$_} for (0..2);
+	# we support "-c $key=$val" for arbitrary git config options
+	# e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
+	push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
+	push @cmd, 'fetch';
+	push @cmd, '-q' if $lei->{opt}->{quiet};
+	push @cmd, '-v' if $lei->{opt}->{verbose};
+	@cmd;
+}
+
+sub remote_url ($$) {
+	my ($lei, $dir) = @_; # TODO: support non-"origin"?
+	my $cmd = [ qw(git config remote.origin.url) ];
+	my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+	my $url = <$fh>;
+	close $fh or return;
+	chomp $url;
+	$url;
+}
+
+sub do_fetch {
+	my ($cls, $lei, $cd) = @_;
+	my $ibx_ver;
+	my $curl = PublicInbox::LeiCurl->new($lei) or return;
+	my $dir = PublicInbox::Admin::resolve_inboxdir($cd, \$ibx_ver);
+	if ($ibx_ver == 1) {
+		my $url = remote_url($lei, $dir) //
+			die "E: $dir missing remote.origin.url\n";
+		my $uri = URI->new($url);
+		my $torsocks = $curl->torsocks($lei, $uri);
+		my $opt = { -C => $dir };
+		my $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
+		my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
+		$lei->child_error($cerr, "@$cmd failed") if $cerr;
+		return;
+	}
+	# v2:
+	opendir my $dh, "$dir/git" or die "opendir $dir/git: $!";
+	my @epochs = sort { $b <=> $a } map { substr($_, 0, -4) + 0 }
+				grep(/\A[0-9]+\.git\z/, readdir($dh));
+	my ($git_url, $epoch);
+	for my $nr (@epochs) { # try newest epoch, first
+		my $edir = "$dir/git/$nr.git";
+		if (defined(my $url = remote_url($lei, $edir))) {
+			$git_url = $url;
+			$epoch = $nr;
+			last;
+		} else {
+			warn "W: $edir missing remote.origin.url\n";
+		}
+	}
+	$git_url or die "Unable to determine git URL\n";
+	my $inbox_url = $git_url;
+	$inbox_url =~ s!/git/$epoch(?:\.git)?/?\z!! or
+		$inbox_url =~ s!/$epoch(?:\.git)?/?\z!! or die <<EOM;
+Unable to infer inbox URL from <$git_url>
+EOM
+	$lei->qerr("# inbox URL: $inbox_url/");
+	my $muri = URI->new("$inbox_url/manifest.js.gz");
+	my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
+				UNLINK => 1, DIR => $dir);
+	my $fn = $ft->filename;
+	my @opt = (qw(-R -o), $fn);
+	my $mf = "$dir/manifest.js.gz";
+	my $m0; # current manifest.js.gz contents
+	if (open my $fh, '<', $mf) {
+		$m0 = eval {
+			PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
+		};
+		$lei->err($@) if $@;
+		push @opt, '-z', $mf if defined($m0);
+	}
+	my $curl_cmd = $curl->for_uri($lei, $muri, @opt);
+	my $opt = {};
+	$opt->{$_} = $lei->{$_} for (0..2);
+	my $cerr = PublicInbox::LeiMirror::run_reap($lei, $curl_cmd, $opt);
+	return $lei->child_error($cerr, "@$curl_cmd failed") if $cerr;
+	return if !-s $ft; # 304 Not Modified via curl -z
+
+	my $m1 = PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+	my $mdiff = { %$m1 };
+
+	# filter out unchanged entries
+	while (my ($k, $v0) = each %{$m0 // {}}) {
+		my $cur = $m1->{$k} // next;
+		my $f0 = $v0->{fingerprint} // next;
+		my $f1 = $cur->{fingerprint} // next;
+		my $t0 = $v0->{modified} // next;
+		my $t1 = $cur->{modified} // next;
+		delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1;
+	}
+	my $ibx_uri = URI->new("$inbox_url/");
+	my ($path_pfx, $v1_bare, @v2_epochs) =
+		PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+	defined($v1_bare) and die <<EOM;
+E: got v1 `$v1_bare' when expecting v2 epoch(s) in <$muri>, WTF?
+EOM
+	my @epoch_nr = sort { $a <=> $b }
+		map { my ($nr) = (m!/([0-9]+)\.git\z!g) } @v2_epochs;
+
+	# n.b. this expects all epochs are from the same host
+	my $torsocks = $curl->torsocks($lei, $muri);
+	for my $nr (@epoch_nr) {
+		my $dir = "$dir/git/$nr.git";
+		my $cmd;
+		my $opt = {};
+		if (-d $dir) {
+			$opt->{-C} = $dir;
+			$cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
+		} else {
+			my $e_uri = $ibx_uri->clone;
+			$e_uri->path($ibx_uri->path."git/$nr.git");
+			$cmd = [ @$torsocks,
+				PublicInbox::LeiMirror::clone_cmd($lei, $opt),
+				$$e_uri, $dir ];
+		}
+		my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
+		return $lei->child_error($cerr, "@$cmd failed") if $cerr;
+	}
+	rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
+	$ft->unlink_on_destroy(0);
+}
+
+1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index aff2bf19ff8534bcea661f0d8ad52a27712a344a..6d5d3c03319846310f8c16d219eebb0c372cd79c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -468,6 +468,8 @@ 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('x_it', $code);
 	} elsif ($self->{sock}) { # to lei(1) client
 		send($self->{sock}, "x_it $code", MSG_EOR);
+	} elsif ($quit == \&CORE::exit) { # an admin command
+		exit($code >> 8);
 	} # else ignore if client disconnected
 }
 
@@ -511,7 +513,7 @@ sub fail ($$;$) {
 	my ($self, $buf, $exit_code) = @_;
 	$self->{failed}++;
 	err($self, $buf) if defined $buf;
-	# calls fail_handler:
+	# calls fail_handler
 	$self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
 	x_it($self, ($exit_code // 1) << 8);
 	undef;
@@ -536,6 +538,8 @@ 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('child_error', $child_error);
 	} elsif ($self->{sock}) { # to lei(1) client
 		send($self->{sock}, "child_error $child_error", MSG_EOR);
+	} else { # non-lei admin command
+		$self->{child_error} ||= $child_error;
 	} # else noop if client disconnected
 }
 
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 355813bdc7af0faac86a23959dcc568d2173d1e2..c128d13d09855be78eb43e54f647458ca0107eea 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -8,6 +8,7 @@ use v5.10.1;
 use parent qw(PublicInbox::IPC);
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use PublicInbox::Spawn qw(popen_rd spawn);
+use File::Temp ();
 
 sub do_finish_mirror { # dwaitpid callback
 	my ($arg, $pid) = @_;
@@ -18,7 +19,9 @@ 		$lei->child_error($?);
 	} elsif (!unlink($f)) {
 		$lei->err("unlink($f): $!") unless $!{ENOENT};
 	} else {
-		$lei->add_external_finish($mrr->{dst});
+		if ($lei->{cmd} ne 'public-inbox-clone') {
+			$lei->add_external_finish($mrr->{dst});
+		}
 		$lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
 	}
 	$lei->dclose;
@@ -121,33 +124,38 @@ }
 
 sub index_cloned_inbox {
 	my ($self, $iv) = @_;
-	my $ibx = delete($self->{ibx}) // {
-		address => [ 'lei@example.com' ],
-		version => $iv,
-	};
-	$ibx->{inboxdir} = $self->{dst};
-	PublicInbox::Inbox->new($ibx);
-	PublicInbox::InboxWritable->new($ibx);
-	my $opt = {};
 	my $lei = $self->{lei};
-	for my $sw ($lei->index_opt) {
-		my ($k) = ($sw =~ /\A([\w-]+)/);
-		$opt->{$k} = $lei->{opt}->{$k};
+
+	# n.b. public-inbox-clone works w/o (SQLite || Xapian)
+	# lei is useless without Xapian + SQLite
+	if ($lei->{cmd} ne 'public-inbox-clone') {
+		my $ibx = delete($self->{ibx}) // {
+			address => [ 'lei@example.com' ],
+			version => $iv,
+		};
+		$ibx->{inboxdir} = $self->{dst};
+		PublicInbox::Inbox->new($ibx);
+		PublicInbox::InboxWritable->new($ibx);
+		my $opt = {};
+		for my $sw ($lei->index_opt) {
+			my ($k) = ($sw =~ /\A([\w-]+)/);
+			$opt->{$k} = $lei->{opt}->{$k};
+		}
+		# force synchronous dwaitpid for v2:
+		local $PublicInbox::DS::in_loop = 0;
+		my $cfg = PublicInbox::Config->new(undef, $lei->{2});
+		my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
+		local %ENV = (%ENV, %$env) if $env;
+		PublicInbox::Admin::progress_prepare($opt, $lei->{2});
+		PublicInbox::Admin::index_inbox($ibx, undef, $opt);
 	}
-	# force synchronous dwaitpid for v2:
-	local $PublicInbox::DS::in_loop = 0;
-	my $cfg = PublicInbox::Config->new(undef, $lei->{2});
-	my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
-	local %ENV = (%ENV, %$env) if $env;
-	PublicInbox::Admin::progress_prepare($opt, $lei->{2});
-	PublicInbox::Admin::index_inbox($ibx, undef, $opt);
 	open my $x, '>', "$self->{dst}/mirror.done"; # for do_finish_mirror
 }
 
 sub run_reap {
 	my ($lei, $cmd, $opt) = @_;
 	$lei->qerr("# @$cmd");
-	$opt->{pgid} = 0;
+	$opt->{pgid} = 0 if $lei->{sock};
 	my $pid = spawn($cmd, undef, $opt);
 	my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
 	waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
@@ -205,6 +213,7 @@ sub deduce_epochs ($$) {
 	my ($m, $path) = @_;
 	my ($v1_bare, @v2_epochs);
 	my $path_pfx = '';
+	$path =~ s!/+\z!!;
 	do {
 		$v1_bare = $m->{$path};
 		@v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
@@ -213,6 +222,18 @@ 		$path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
 	($path_pfx, $v1_bare, @v2_epochs);
 }
 
+sub decode_manifest ($$$) {
+	my ($fh, $fn, $uri) = @_;
+	my $js;
+	my $gz = do { local $/; <$fh> } // die "slurp($fn): $!";
+	gunzip(\$gz => \$js, MultiStream => 1) or
+		die "gunzip($uri): $GunzipError\n";
+	my $m = eval { PublicInbox::Config->json->decode($js) };
+	die "$uri: error decoding `$js': $@\n" if $@;
+	ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
+	$m;
+}
+
 sub try_manifest {
 	my ($self) = @_;
 	my $uri = URI->new($self->{src});
@@ -221,26 +242,19 @@ 	my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
 	my $path = $uri->path;
 	chop($path) eq '/' or die "BUG: $uri not canonicalized";
 	$uri->path($path . '/manifest.js.gz');
-	my $cmd = $curl->for_uri($lei, $uri);
-	$lei->qerr("# @$cmd");
-	my $opt = { 0 => $lei->{0}, 2 => $lei->{2} };
-	my ($fh, $pid) = popen_rd($cmd, undef, $opt);
-	my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
-	my $gz = do { local $/; <$fh> } // die "read(curl $uri): $!";
-	close $fh;
-	waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
-	@$reap = ();
-	if ($?) {
-		return try_scrape($self) if ($? >> 8) == 22; # 404 missing
-		return $lei->child_error($?, "@$cmd failed");
+	my $pdir = $lei->rel2abs($self->{dst});
+	$pdir =~ s!/[^/]+/?\z!!;
+	my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
+				UNLINK => 1, DIR => $pdir);
+	my $fn = $ft->filename;
+	my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $fn);
+	my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
+	my $cerr = run_reap($lei, $cmd, $opt);
+	if ($cerr) {
+		return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
+		return $lei->child_error($cerr, "@$cmd failed");
 	}
-	my $js;
-	gunzip(\$gz => \$js, MultiStream => 1) or
-		die "gunzip($uri): $GunzipError";
-	my $m = eval { PublicInbox::Config->json->decode($js) };
-	die "$uri: error decoding `$js': $@" if $@;
-	ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
-
+	my $m = decode_manifest($ft, $fn, $uri);
 	my ($path_pfx, $v1_bare, @v2_epochs) = deduce_epochs($m, $path);
 	if (@v2_epochs) {
 		# It may be possible to have v1 + v2 in parallel someday:
@@ -254,6 +268,9 @@ 			$uri->path($path_pfx.$_);
 			$uri->clone
 		} @v2_epochs;
 		clone_v2($self, \@v2_epochs);
+		my $fin = "$self->{dst}/manifest.js.gz";
+		rename($fn, $fin) or die "E: rename($fn, $fin): $!";
+		$ft->unlink_on_destroy(0);
 	} elsif (defined $v1_bare) {
 		clone_v1($self);
 	} else {
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
new file mode 100755
index 0000000000000000000000000000000000000000..2b18969f75c2b6dacbb37abc56b0cd901d75e185
--- /dev/null
+++ b/script/public-inbox-clone
@@ -0,0 +1,58 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to git clone remote public-inboxes
+use strict;
+use v5.10.1;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+my $opt = {};
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-clone INBOX_URL [DESTINATION]
+
+  clone remote public-inboxes
+
+options:
+
+  --torsocks VAL      whether or not to wrap git and curl commands with
+                      torsocks (default: `auto')
+                      Must be one of: `auto', `no' or `yes'
+  --verbose | -v      increase verbosity (may be repeated)
+    --quiet | -q      increase verbosity (may be repeated)
+    -C DIR            chdir to specified directory
+EOF
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
+		no-torsocks torsocks=s)) or die $help;
+if ($opt->{help}) { print $help; exit };
+require PublicInbox::Admin; # loads Config
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+PublicInbox::Admin::setup_signals();
+$SIG{PIPE} = 'IGNORE';
+
+my ($url, $dst, $extra) = @ARGV;
+die $help if !defined($url) || defined($extra);
+defined($dst) or ($dst) = ($url =~ m!/([^/]+)/?\z!);
+index($dst, "\n") >= 0 and die "`\\n' not allowed in `$dst'";
+
+# n.b. this is still a truckload of code...
+require URI;
+require PublicInbox::LEI;
+require PublicInbox::LeiExternal;
+require PublicInbox::LeiMirror;
+require PublicInbox::LeiCurl;
+require PublicInbox::Lock;
+
+$url = PublicInbox::LeiExternal::ext_canonicalize($url);
+my $lei = bless {
+	env => \%ENV, opt => $opt, cmd => 'public-inbox-clone',
+	0 => *STDIN{GLOB}, 2 => *STDERR{GLOB},
+}, 'PublicInbox::LEI';
+open $lei->{1}, '+<&=', 1 or die "dup: $!";
+open $lei->{3}, '.' or die "open . $!";
+my $mrr = bless {
+	lei => $lei,
+	src => $url,
+	dst => $dst,
+}, 'PublicInbox::LeiMirror';
+$mrr->do_mirror;
+$mrr->can('do_finish_mirror')->([$mrr, $lei], $$);
+exit(($lei->{child_error} // 0) >> 8);
diff --git a/script/public-inbox-fetch b/script/public-inbox-fetch
new file mode 100755
index 0000000000000000000000000000000000000000..5d303574c8b140c0a19fed8522393ffe9f7668c9
--- /dev/null
+++ b/script/public-inbox-fetch
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to git fetch remote public-inboxes
+use strict;
+use v5.10.1;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+my $opt = {};
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-fetch -C DESTINATION
+
+  fetch remote public-inboxes
+
+options:
+
+  --torsocks VAL      whether or not to wrap git and curl commands with
+                      torsocks (default: `auto')
+                      Must be one of: `auto', `no' or `yes'
+  --verbose | -v      increase verbosity (may be repeated)
+    --quiet | -q      increase verbosity (may be repeated)
+    -C DIR            chdir to specified directory
+EOF
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
+	no-torsocks torsocks=s)) or die $help;
+if ($opt->{help}) { print $help; exit };
+require PublicInbox::Fetch; # loads Admin
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+PublicInbox::Admin::setup_signals();
+$SIG{PIPE} = 'IGNORE';
+
+my $lei = bless {
+	env => \%ENV, opt => $opt, cmd => 'public-inbox-fetch',
+	0 => *STDIN{GLOB}, 1 => *STDOUT{GLOB}, 2 => *STDERR{GLOB},
+}, 'PublicInbox::LEI';
+PublicInbox::Fetch->do_fetch($lei, '.');
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index a61a756596fc5b6c14d4db511c37664108af62eb..75e25b3f05ef196abd6ee59b0a75c009378f2f2e 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -46,6 +46,7 @@ 	unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
 
 	lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
 			\'--mirror v1 w/ PSGI prefix');
+	ok(!-e "$t1-pfx/mirror.done", 'no leftover mirror.done');
 
 	my $d = "$home/404";
 	ok(!lei(qw(add-external --mirror), "$http/404", $d), 'mirror 404');
@@ -76,6 +77,34 @@ 		like($lei_err, qr/curl.*404/, "curl 404 shown for $k");
 	} # SKIP
 	} # for
 });
+
+SKIP: {
+	undef $sock;
+	my $d = "$tmpdir/d";
+	mkdir $d or xbail "mkdir $d $!";
+	my $opt = { -C => $d, 2 => \(my $err) };
+	ok(!run_script([qw(-clone -q), "$http/404"], undef, $opt), '404 fails');
+	ok(!-d "$d/404", 'destination not created');
+	delete $opt->{2};
+
+	ok(run_script([qw(-clone -q -C), $d, "$http/t2"], undef, $opt),
+		'-clone succeeds on v2');
+	ok(-d "$d/t2/git/0.git", 'epoch cloned');
+	ok(-f "$d/t2/manifest.js.gz", 'manifest saved');
+	ok(!-e "$d/t2/mirror.done", 'no leftover mirror.done');
+	ok(run_script([qw(-fetch -q -C), "$d/t2"], undef, $opt),
+		'-fetch succeeds w/ manifest.js.gz');
+	unlink("$d/t2/manifest.js.gz") or xbail "unlink $!";
+	ok(run_script([qw(-fetch -q -C), "$d/t2"], undef, $opt),
+		'-fetch succeeds w/o manifest.js.gz');
+
+	ok(run_script([qw(-clone -q -C), $d, "$http/t1"], undef, $opt),
+		'cloning v1 works');
+	ok(-d "$d/t1", 'v1 cloned');
+	ok(!-e "$d/t1/mirror.done", 'no leftover file');
+	ok(run_script([qw(-fetch -q -C), "$d/t1"], undef, $opt),
+		'fetching v1 works');
+}
 
 ok($td->kill, 'killed -httpd');
 $td->join;