X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiMirror.pm;h=18c825d3ccdc7bf69fe59acadeec165867eddfcd;hb=d604845055e3b0096024fd98b993a485a72653cd;hp=1d6ed51c80ee5f35d89309f080dfcf67e8d7e868;hpb=0ef7fb8a3edb8880ccb50d4d430ef92f7a8093f5;p=public-inbox.git diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 1d6ed51c..18c825d3 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -67,8 +67,9 @@ sub try_scrape { my ($n) = (m!/([0-9]+)\z!); $n => [ URI->new($_), '' ] } @v2_urls; # uniq - clone_v2($self, \%v2_epochs); - reap_live() while keys(%$LIVE); + clone_v2_prep($self, \%v2_epochs); + delete local $lei->{opt}->{epoch}; + clone_all($self); return; } @@ -98,7 +99,8 @@ sub clone_cmd { ($lei->{opt}->{jobs} // 1) > 1; push @cmd, '-v' if $lei->{opt}->{verbose}; # XXX any other options to support? - # --reference is tricky with multiple epochs... + # --reference is tricky with multiple epochs, but handled + # automatically if using manifest.js.gz @cmd; } @@ -177,16 +179,17 @@ sub set_description ($) { open my $fh, '+>>', $f or die "open($f): $!"; seek($fh, 0, SEEK_SET) or die "seek($f): $!"; my $d = do { local $/; <$fh> } // die "read($f): $!"; - my $orig = $d; + chomp(my $orig = $d); while (defined($d) && ($d =~ m!^\(\$INBOX_DIR/description missing\)! || $d =~ /^Unnamed repository/ || $d !~ /\S/)) { $d = delete($self->{'txt.description'}); } - $d //= 'mirror of '.($self->{cur_src} // $self->{src})."\n"; + $d //= 'mirror of '.($self->{cur_src} // $self->{src}); + chomp $d; return if $d eq $orig; seek($fh, 0, SEEK_SET) or die "seek($f): $!"; truncate($fh, 0) or die "truncate($f): $!"; - print $fh $d or die "print($f): $!"; + print $fh $d, "\n" or die "print($f): $!"; close $fh or die "close($f): $!"; } @@ -221,7 +224,7 @@ sub index_cloned_inbox { PublicInbox::Admin::progress_prepare($opt, $lei->{2}); PublicInbox::Admin::index_inbox($ibx, undef, $opt); } - return if defined $self->{cur_dst}; + return if defined $self->{cur_dst}; # one of many repos to clone open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait } @@ -259,10 +262,14 @@ sub clone_v1 { push @$cmd, '--reference', "$self->{dst}$ref"; start_clone($self, $cmd, $opt, $fini); - $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v1)\z/s and + if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~ + /\A(?:always|v1)\z/s) { _get_txt_start($self, '_/text/config/raw', $fini); + } + my $d = $self->{-ent} ? $self->{-ent}->{description} : undef; - defined($d) ? ($self->{'txt.description'} = $d) : + $self->{'txt.description'} = $d if defined $d; + (!defined($d) && !$nohang) and _get_txt_start($self, 'description', $fini); reap_live() until ($nohang || !keys(%$LIVE)); # for non-manifest clone @@ -333,11 +340,14 @@ EOM EOM } close $fh or die "close($f): $!"; - if (defined $ent->{head}) { - $f = "$edst/HEAD"; - open $fh, '>', $f or die "open($f): $!"; - print $fh $ent->{head}, "\n" or die "print($f): $!"; - close $fh or die "close($f): $!"; + my %map = (head => 'HEAD', description => undef); + while (my ($key, $fn) = each %map) { + my $val = $ent->{$key} // next; + $fn //= $key; + $fn = "$edst/$fn"; + open $fh, '>', $fn or die "open($fn): $!"; + print $fh $val, "\n" or die "print($fn): $!"; + close $fh or die "close($fn): $!"; } } @@ -370,6 +380,7 @@ sub v1_done { # called via OnDestroy die "rename($f, $o/info/alternates): $!"; $f->unlink_on_destroy(0); } + return if $self->{-is_epoch}; write_makefile($dst, 1); index_cloned_inbox($self, 1); } @@ -378,6 +389,7 @@ sub v2_done { # called via OnDestroy my ($self) = @_; return if $self->{dry_run} || !$LIVE; my $dst = $self->{cur_dst} // $self->{dst}; + require PublicInbox::Lock; my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock'; my $lck = $lk->lock_for_scope($$); _write_inbox_config($self); @@ -385,11 +397,6 @@ sub v2_done { # called via OnDestroy my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git'); $mg->fill_alternates; for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) } - my $edst_owner = delete($self->{-owner}) // []; - while (@$edst_owner) { - my ($edst, $o) = splice(@$edst_owner); - run_die [qw(git config -f), "$edst/config", 'gitweb.owner', $o]; - } for my $edst (@{delete($self->{-read_only}) // []}) { my @st = stat($edst) or die "stat($edst): $!"; chmod($st[2] & 0555, $edst) or die "chmod(a-w, $edst): $!"; @@ -409,7 +416,7 @@ sub reap_live { } } -sub clone_v2 ($$;$) { +sub clone_v2_prep ($$;$) { my ($self, $v2_epochs, $m) = @_; # $m => manifest.js.gz hashref my $lei = $self->{lei}; my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return; @@ -418,7 +425,8 @@ sub clone_v2 ($$;$) { my $dst = $self->{cur_dst} // $self->{dst}; my $want = parse_epochs($lei->{opt}->{epoch}, $v2_epochs); my $task = $m ? bless { %$self }, __PACKAGE__ : $self; - my (@src_edst, @skip); + delete $task->{todo}; # $self->{todo} still exists + my (@src_edst, @skip, $desc, @entv); for my $nr (sort { $a <=> $b } keys %$v2_epochs) { my ($uri, $key) = @{$v2_epochs->{$nr}}; my $src = $uri->as_string; @@ -428,34 +436,47 @@ failed to extract epoch number from $src $1 + 0 == $nr or die "BUG: <$uri> miskeyed $1 != $nr"; $edst .= "/git/$nr.git"; - $m->{$key} // die "BUG: `$key' not in manifest.js.gz"; + my $ent; + if ($m) { + $ent = $m->{$key} // + die("BUG: `$key' not in manifest.js.gz"); + if (defined(my $d = $ent->{description})) { + $d =~ s/ \[epoch [0-9]+\]\z//s; + $desc = $d; + } + } if (!$want || $want->{$nr}) { push @src_edst, $src, $edst; - my $o = $m->{$key}->{owner}; - push(@{$task->{-owner}}, $edst, $o) if defined($o); + push @entv, $edst, $ent; + $self->{any_want}->{$key} = 1; } else { # create a placeholder so users only need to chmod +w - init_placeholder($src, $edst, $m->{$key}); + init_placeholder($src, $edst, $ent); push @{$task->{-read_only}}, $edst; push @skip, $key; } } # filter out the epochs we skipped - $self->{-culled_manifest} = 1 if delete(@$m{@skip}); + $self->{-culled_manifest} = 1 if $m && delete(@$m{@skip}); (!$self->{dry_run} && !-d $dst) and File::Path::mkpath($dst); - require PublicInbox::Lock; my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task); $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v2)\z/s and _get_txt_start($task, '_/text/config/raw', $fini); - _get_txt_start($task, 'description', $fini); - - my @cmd = clone_cmd($lei, my $opt = {}); - while (@src_edst && !$lei->{child_error}) { - my $cmd = [ @$pfx, @cmd, splice(@src_edst, 0, 2) ]; - start_clone($self, $cmd, $opt, $fini); + defined($desc) ? ($task->{'txt.description'} = $desc) : + _get_txt_start($task, 'description', $fini); + while (@entv) { + my ($edst, $ent) = splice(@entv, 0, 2); + my $etask = bless { %$task }, __PACKAGE__; + $etask->{-ent} = $ent; # may have {reference} + $etask->{cur_src} = shift @src_edst // die 'BUG: no cur_src'; + $etask->{cur_dst} = shift @src_edst // die 'BUG: no cur_dst'; + $etask->{cur_dst} eq $edst or + die "BUG: `$etask->{cur_dst}' != `$edst'"; + $etask->{-is_epoch} = $fini; + push @{$self->{todo}->{($ent->{reference} // '')}}, $etask; } } @@ -488,18 +509,18 @@ sub multi_inbox ($$$) { my $n = scalar(keys %$v2) + scalar(@v1); my @orig = defined($incl // $excl) ? (keys %$v2, @v1) : (); if (defined $incl) { - my $re = '(?:'.join('|', map { - $self->{lei}->glob2re($_) // qr/\A\Q$_\E\z/ - } @$incl).')'; + my $re = '(?:'.join('\\z|', map { + $self->{lei}->glob2re($_) // qr/\A\Q$_\E/ + } @$incl).'\\z)'; my @gone = delete @$v2{grep(!/$re/, keys %$v2)}; delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1; delete @$m{grep(!/$re/, @v1)} and $self->{-culled_manifest} = 1; @v1 = grep(/$re/, @v1); } if (defined $excl) { - my $re = '(?:'.join('|', map { - $self->{lei}->glob2re($_) // qr/\A\Q$_\E\z/ - } @$excl).')'; + my $re = '(?:'.join('\\z|', map { + $self->{lei}->glob2re($_) // qr/\A\Q$_\E/ + } @$excl).'\\z)'; my @gone = delete @$v2{grep(/$re/, keys %$v2)}; delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1; delete @$m{grep(/$re/, @v1)} and $self->{-culled_manifest} = 1; @@ -529,18 +550,27 @@ sub multi_inbox ($$$) { } sub clone_all { - my ($self, $todo, $m) = @_; + my ($self, $m) = @_; + my $todo = delete $self->{todo}; + my $nodep = delete $todo->{''}; + + # do not download unwanted deps + my $any_want = delete $self->{any_want}; + my @unwanted = grep { !$any_want->{$_} } keys %$todo; + my @nodep = delete(@$todo{@unwanted}); + push(@$nodep, @$_) for @nodep; + # handle no-dependency repos, first - for (@{delete($todo->{''}) // []}) { + for (@$nodep) { clone_v1($_, 1); return if $self->{lei}->{child_error}; } # resolve references, deepest, first: while (scalar keys %$todo) { for my $x (keys %$todo) { - my $nr; + my ($nr, $nxt); # resolve multi-level references - while (defined(my $nxt = $m->{$x}->{reference})) { + while ($m && defined($nxt = $m->{$x}->{reference})) { exists($todo->{$nxt}) or last; die < 1000; E: dependency loop detected (`$x' => `$nxt') @@ -555,6 +585,7 @@ EOM last; # restart %$todo iteration } } + reap_live() while keys(%$LIVE); } # FIXME: this gets confused by single inbox instance w/ global manifest.js.gz @@ -585,6 +616,7 @@ sub try_manifest { my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m); return $lei->child_error(1, $multi) if !ref($multi); my $v2 = delete $multi->{v2}; + local $self->{todo} = {}; if ($v2) { for my $name (sort keys %$v2) { my $epochs = delete $v2->{$name}; @@ -606,17 +638,14 @@ sub try_manifest { index($self->{cur_dst}, "\n") >= 0 and die <{cur_dst}' must not contain newline EOM - clone_v2($self, \%v2_epochs, $m); + clone_v2_prep($self, \%v2_epochs, $m); return if $self->{lei}->{child_error}; } } if (my $v1 = delete $multi->{v1}) { - delete local $lei->{opt}->{epoch} if defined($v2); my $p = $path_pfx.$path; chop($p) if substr($p, -1, 1) eq '/'; $uri->path($p); - my $todo = {}; - my %want = map { $_ => 1 } @$v1; for my $name (@$v1) { my $task = bless { %$self }, __PACKAGE__; $task->{-ent} = $m->{$name} // @@ -632,15 +661,16 @@ E: `$task->{cur_dst}' must not contain newline EOM $task->{cur_src} .= '/'; my $dep = $task->{-ent}->{reference} // ''; - $dep = '' if !$want{$dep}; - push @{$todo->{$dep}}, $task; + push @{$self->{todo}->{$dep}}, $task; # for clone_all + $self->{any_want}->{$name} = 1; } - clone_all($self, $todo, $m); } - reap_live() while keys(%$LIVE); + delete local $lei->{opt}->{epoch} if defined($v2); + clone_all($self, $m); return if $self->{lei}->{child_error} || $self->{dry_run}; - if (delete $self->{-culled_manifest}) { # set by clone_v2/-I/--exclude + # set by clone_v2_prep/-I/--exclude + if (delete $self->{-culled_manifest}) { # write the smaller manifest if epochs were skipped so # users won't have to delete manifest if they +w an # epoch they no longer want to skip