lib/PublicInbox/LeiMirror.pm | 13 +++++++------ lib/PublicInbox/Msgmap.pm | 9 ++++++++- lib/PublicInbox/WwwText.pm | 3 +++ t/lei-mirror.t | 31 ++++++++++++++++++++++++++++--- t/psgi_v2.t | 10 +++++++++- diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 5cfa6fea851c9d11b6979228e9e950c49adef69d..4be8f70ae4015b63c6cd8622715dc67f69a045ea 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -96,15 +96,15 @@ my $lei = $self->{lei}; my $path = $uri->path; chop($path) eq '/' or die "BUG: $uri not canonicalized"; $uri->path("$path/$endpoint"); - my $cmd = $self->{curl}->for_uri($lei, $uri, '--compressed'); - my $ce = "$self->{dst}/$file"; - my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", - UNLINK => 1, DIR => $self->{dst}); - my $opt = { 0 => $lei->{0}, 1 => $ft, 2 => $lei->{2} }; + my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", DIR => $self->{dst}); + my $f = $ft->filename; + my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} }; + my $cmd = $self->{curl}->for_uri($lei, $uri, + qw(--compressed -R -o), $f); my $cerr = run_reap($lei, $cmd, $opt); return "$uri missing" if ($cerr >> 8) == 22; return "# @$cmd failed (non-fatal)" if $cerr; - my $f = $ft->filename; + my $ce = "$self->{dst}/$file"; rename($f, $ce) or return "rename($f, $ce): $! (non-fatal)"; $ft->unlink_on_destroy(0); undef; # success @@ -122,6 +122,7 @@ } my $err = _get_txt($self, qw(_/text/config/raw inbox.config.example)); return $self->{lei}->err($err) if $err; my $f = "$self->{dst}/inbox.config.example"; + chmod((stat($f))[2] & 0444, $f) or die "chmod(a-w, $f): $!"; my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2}); my $ibx = $self->{ibx} = {}; for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) { diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 94a0cbeb5f6a6f6f445839b45f0f73d5c5c71406..e71f16f80ebd423343e467e2abeebe01d47e58fb 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -32,8 +32,15 @@ my $dbh = $self->{dbh} = PublicInbox::Over::dbh_new($self, $rw); if ($rw) { $dbh->begin_work; create_tables($dbh); - $self->created_at(time) unless $self->created_at; + unless ($self->created_at) { + my $t; + if (blessed($ibx) && + -f "$ibx->{inboxdir}/inbox.config.example") { + $t = (stat(_))[9]; # mtime set by "curl -R" + } + $self->created_at($t // time); + } $self->num_highwater(max($self)); $dbh->commit; } diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm index bb9a0a0ff54b5e3edd1b6cd83c428a70a72a6de6..8b929f7431560979ddb076b321016241e854459a 100644 --- a/lib/PublicInbox/WwwText.pm +++ b/lib/PublicInbox/WwwText.pm @@ -8,6 +8,7 @@ use v5.10.1; use PublicInbox::Linkify; use PublicInbox::WwwStream; use PublicInbox::Hval qw(ascii_html prurl); +use HTTP::Date qw(time2str); use URI::Escape qw(uri_escape_utf8); use PublicInbox::GzipFilter qw(gzf_maybe); our $QP_URL = 'https://xapian.org/docs/queryparser.html'; @@ -171,6 +172,8 @@ sub inbox_config ($$$) { my ($ctx, $hdr, $txt) = @_; my $ibx = $ctx->{ibx}; push @$hdr, 'Content-Disposition', 'inline; filename=inbox.config'; + my $t = eval { $ibx->mm->created_at }; + push(@$hdr, 'Last-Modified', time2str($t)) if $t; my $name = dq_escape($ibx->{name}); my $inboxdir = '/path/to/top-level-inbox'; my $base_url = $ibx->base_url($ctx->{env}); diff --git a/t/lei-mirror.t b/t/lei-mirror.t index de5246b60bce27c707c2a6e833bfe2655f79771f..b449e0b48a41eb8358c25bb047327c88acd177aa 100644 --- a/t/lei-mirror.t +++ b/t/lei-mirror.t @@ -3,8 +3,9 @@ # Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ use strict; use v5.10.1; use PublicInbox::TestCommon; use PublicInbox::Inbox; -require_mods(qw(-httpd lei)); +require_mods(qw(-httpd lei DBD::SQLite)); require_cmd('curl'); +require PublicInbox::Msgmap; my $sock = tcp_server(); my ($tmpdir, $for_destroy) = tmpdir(); my $http = 'http://'.tcp_host_port($sock); @@ -12,25 +13,40 @@ my ($ro_home, $cfg_path) = setup_public_inboxes; my $cmd = [ qw(-httpd -W0 ./t/lei-mirror.psgi), "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ]; my $td = start_script($cmd, { PI_CONFIG => $cfg_path }, { 3 => $sock }); +my %created; test_lei({ tmpdir => $tmpdir }, sub { my $home = $ENV{HOME}; my $t1 = "$home/t1-mirror"; + my $mm_orig = "$ro_home/t1/public-inbox/msgmap.sqlite3"; + $created{v1} = PublicInbox::Msgmap->new_file($mm_orig)->created_at; lei_ok('add-external', $t1, '--mirror', "$http/t1/", \'--mirror v1'); - ok(-f "$t1/public-inbox/msgmap.sqlite3", 't1-mirror indexed'); + my $mm_dup = "$t1/public-inbox/msgmap.sqlite3"; + ok(-f $mm_dup, 't1-mirror indexed'); is(PublicInbox::Inbox::try_cat("$t1/description"), "mirror of $http/t1/\n", 'description set'); ok(-f "$t1/Makefile", 'convenience Makefile added (v1)'); + ok(-f "$t1/inbox.config.example", 'inbox.config.example downloaded'); + is((stat(_))[9], $created{v1}, + 'inbox.config.example mtime is ->created_at'); + is((stat(_))[2] & 0222, 0, 'inbox.config.example not writable'); + my $tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at; + is($tb, $created{v1}, 'created_at matched in mirror'); lei_ok('ls-external'); like($lei_out, qr!\Q$t1\E!, 't1 added to ls-externals'); my $t2 = "$home/t2-mirror"; + $mm_orig = "$ro_home/t2/msgmap.sqlite3"; + $created{v2} = PublicInbox::Msgmap->new_file($mm_orig)->created_at; lei_ok('add-external', $t2, '--mirror', "$http/t2/", \'--mirror v2'); - ok(-f "$t2/msgmap.sqlite3", 't2-mirror indexed'); + $mm_dup = "$t2/msgmap.sqlite3"; + ok(-f $mm_dup, 't2-mirror indexed'); ok(-f "$t2/description", 't2 description'); ok(-f "$t2/Makefile", 'convenience Makefile added (v2)'); is(PublicInbox::Inbox::try_cat("$t2/description"), "mirror of $http/t2/\n", 'description set'); + $tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at; + is($tb, $created{v2}, 'created_at matched in v2 mirror'); lei_ok('ls-external'); like($lei_out, qr!\Q$t2\E!, 't2 added to ls-externals'); @@ -150,6 +166,15 @@ like($err, qr/git --git-dir=\S+ fetch/, 'no fetch done w/ manifest'); ok(unlink("$d/t1/manifest.js.gz"), 'manifest created'); my $after = [ glob("$d/t1/*") ]; is_deeply($before, $after, 'no new files created'); + + ok(run_script([qw(-index -Lbasic), "$d/t1"]), 'index v1'); + ok(run_script([qw(-index -Lbasic), "$d/t2"]), 'index v2'); + my $f = "$d/t1/public-inbox/msgmap.sqlite3"; + my $ca = PublicInbox::Msgmap->new_file($f)->created_at; + is($ca, $created{v1}, 'clone + index v1 synced ->created_at'); + $f = "$d/t2/msgmap.sqlite3"; + $ca = PublicInbox::Msgmap->new_file($f)->created_at; + is($ca, $created{v2}, 'clone + index v1 synced ->created_at'); } ok($td->kill, 'killed -httpd'); diff --git a/t/psgi_v2.t b/t/psgi_v2.t index 1f19070894571eef0dea0650b06e4718016ea775..e057068226ac0eac778ec6cb82cea85ec38c8656 100644 --- a/t/psgi_v2.t +++ b/t/psgi_v2.t @@ -9,7 +9,7 @@ use PublicInbox::Eml; use PublicInbox::Config; use PublicInbox::MID qw(mids); require_mods(qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test - URI::Escape Plack::Builder)); + URI::Escape Plack::Builder HTTP::Date)); use_ok($_) for (qw(HTTP::Request::Common Plack::Test)); use_ok 'PublicInbox::WWW'; my ($tmpdir, $for_destroy) = tmpdir(); @@ -113,6 +113,14 @@ $im->done; my $client1 = sub { my ($cb) = @_; + $res = $cb->(GET('/v2test/_/text/config/raw')); + my $lm = $res->header('Last-Modified'); + ok($lm, 'Last-Modified set w/ ->mm'); + $lm = HTTP::Date::str2time($lm); + is($lm, $ibx->mm->created_at, + 'Last-Modified for text/config/raw matches ->created_at'); + delete $ibx->{mm}; + $res = $cb->(GET("/v2test/$third/raw")); $raw = $res->content; like($raw, qr/^hello ghosts$/m, 'got third message');