lei_mirror: set info/web/last-modified from manifest

[public-inbox.git] / lib / PublicInbox / LeiMirror.pm

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index cc5ea1d2a5c0bbbe576ce624e66dbf85e8be1169..e284f55deb6133cb25eaaf0113426ff7517434d6 100644 (file)
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -19,6 +19,7 @@ use PublicInbox::Inbox;
 use PublicInbox::LeiCurl;
 use PublicInbox::OnDestroy;
 use Digest::SHA qw(sha256_hex sha1_hex);
+use POSIX qw(strftime);
 
 our $LIVE; # pid => callback
 our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
@@ -51,7 +52,7 @@ sub try_scrape {
        my $uri = URI->new($self->{src});
        my $lei = $self->{lei};
        my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
-       my $cmd = $curl->for_uri($lei, $uri, qw(-f --compressed));
+       my $cmd = $curl->for_uri($lei, $uri, '--compressed');
        my $opt = { 0 => $lei->{0}, 2 => $lei->{2} };
        my $fh = popen_rd($cmd, undef, $opt);
        my $html = do { local $/; <$fh> } // die "read(curl $uri): $!";
@@ -151,7 +152,7 @@ sub _get_txt_start { # non-fatal
        my $f = (split(m!/!, $endpoint))[-1];
        my $ft = File::Temp->new(TEMPLATE => "$f-XXXX", TMPDIR => 1);
        my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
-       my $cmd = $self->{curl}->for_uri($lei, $uri, qw(-f --compressed -R -o),
+       my $cmd = $self->{curl}->for_uri($lei, $uri, qw(--compressed -R -o),
                                        $ft->filename);
        do_reap($self);
        $lei->qerr("# @$cmd");
@@ -200,22 +201,14 @@ sub _write_inbox_config {
 sub set_description ($) {
        my ($self) = @_;
        my $dst = $self->{cur_dst} // $self->{dst};
-       my $f = "$dst/description";
-       open my $fh, '+>>', $f or die "open($f): $!";
-       seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-       my $d = do { local $/; <$fh> } // die "read($f): $!";
-       chomp(my $orig = $d);
+       chomp(my $orig = PublicInbox::Git::try_cat("$dst/description"));
+       my $d = $orig;
        while (defined($d) && ($d =~ m!^\(\$INBOX_DIR/description missing\)! ||
                        $d =~ /^Unnamed repository/ || $d !~ /\S/)) {
                $d = delete($self->{'txt.description'});
        }
        $d //= 'mirror of '.($self->{cur_src} // $self->{src});
-       chomp $d;
-       return if $d eq $orig;
-       seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-       truncate($fh, 0) or die "truncate($f): $!";
-       print $fh $d, "\n" or die "print($f): $!";
-       close $fh or die "close($f): $!";
+       atomic_write($dst, 'description', $d."\n") if $d ne $orig;
 }
 
 sub index_cloned_inbox {
@@ -533,6 +526,8 @@ sub clone_v1 {
        my $lei = $self->{lei};
        my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
        my $uri = URI->new($self->{cur_src} // $self->{src});
+       my $path = $uri->path;
+       $path =~ s!/*\z!! and $uri->path($path);
        defined($lei->{opt}->{epoch}) and
                die "$uri is a v1 inbox, --epoch is not supported\n";
        $self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
@@ -666,6 +661,14 @@ sub up_fp_done {
        push @{$self->{chg}->{fp_mismatch}}, $self->{-key};
 }
 
+sub atomic_write ($$$) {
+       my ($dn, $bn, $raw) = @_;
+       my $ft = File::Temp->new(DIR => $dn, TEMPLATE => "$bn-XXXX");
+       print $ft $raw or die "print($ft): $!";
+       $ft->flush or die "flush($ft): $!";
+       ft_rename($ft, "$dn/$bn", 0666);
+}
+
 # modifies the to-be-written manifest entry, and sets values from it, too
 sub update_ent {
        my ($self) = @_;
@@ -681,7 +684,6 @@ sub update_ent {
                my $done = PublicInbox::OnDestroy->new($$, \&up_fp_done, $self);
                start_cmd($self, $cmd, $opt, $done);
        }
-
        $new = $self->{-ent}->{head};
        $cur = $self->{-local_manifest}->{$key}->{head} // "\0";
        if (defined($new) && $new ne $cur) {
@@ -717,6 +719,14 @@ sub update_ent {
                        symlink($tgt, $ln) or die "symlink($tgt, $ln): $!";
                }
        }
+       if (defined(my $t = $self->{-ent}->{modified})) {
+               my ($dn, $bn) = ("$dst/info/web", 'last-modified');
+               my $orig = PublicInbox::Git::try_cat("$dn/$bn");
+               $t = strftime('%F %T', gmtime($t))." +0000\n";
+               File::Path::mkpath($dn);
+               atomic_write($dn, $bn, $t) if $orig ne $t;
+       }
+
        $new = $self->{-ent}->{owner} // return;
        $cur = $self->{-local_manifest}->{$key}->{owner} // "\0";
        return if $cur eq $new;
@@ -994,7 +1004,7 @@ sub try_manifest {
                delete $opt{TMPDIR};
        }
        my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt);
-       my $cmd = $curl->for_uri($lei, $uri, qw(-f -R -o), $ft->filename);
+       my $cmd = $curl->for_uri($lei, $uri, qw(-R -o), $ft->filename);
        push(@$cmd, '-z', $manifest) if -f $manifest;
        my $mf_url = "$uri";
        %opt = map { $_ => $lei->{$_} } (0..2);
@@ -1074,8 +1084,7 @@ EOM
        my $mis = delete $self->{chg}->{fp_mismatch};
        if ($mis) {
                my $t = (stat($ft))[9];
-               require POSIX;
-               $t = POSIX::strftime('%Y-%m-%d %k:%M:%S %z', localtime($t));
+               $t = strftime('%F %k:%M:%S %z', localtime($t));
                warn <<EOM;
 W: Fingerprints for the following repositories do not match
 W: $mf_url @ $t: