X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fnntpd.t;h=d8a44334a55bf13e0b0037a3b3c8ba3e90085df9;hb=cee907599bb7eda0695ae91cf8d633670a88d6c7;hp=243b8e5637ce80ba7659a6df3b2468f288ffcd6c;hpb=a09f678d8254064af7ca6dcfb3c3f84b5ae37b51;p=public-inbox.git diff --git a/t/nntpd.t b/t/nntpd.t index 243b8e56..d8a44334 100644 --- a/t/nntpd.t +++ b/t/nntpd.t @@ -1,22 +1,27 @@ -# Copyright (C) 2015-2019 all contributors +# Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ use strict; use warnings; use Test::More; use PublicInbox::TestCommon; +use PublicInbox::Spawn qw(which); require_mods(qw(DBD::SQLite)); -require PublicInbox::SearchIdx; -require PublicInbox::Msgmap; require PublicInbox::InboxWritable; -use Email::Simple; +use PublicInbox::Eml; use IO::Socket; use Socket qw(IPPROTO_TCP TCP_NODELAY); use Net::NNTP; use Sys::Hostname; +use POSIX qw(_exit); +use Digest::SHA; +use_ok 'PublicInbox::Msgmap'; -# FIXME: make easier to test both versions -my $version = $ENV{PI_TEST_VERSION} || 2; +# t/nntpd-v2.t wraps this for v2 +my $version = $ENV{PI_TEST_VERSION} || 1; require_git('2.6') if $version == 2; +my $lsof = which('lsof'); +my $fast_idle = eval { require Linux::Inotify2; 1 } // + eval { require IO::KQueue; 1 }; my ($tmpdir, $for_destroy) = tmpdir(); my $home = "$tmpdir/pi-home"; @@ -25,10 +30,6 @@ my $out = "$tmpdir/stdout.log"; my $inboxdir = "$tmpdir/main.git"; my $group = 'test-nntpd'; my $addr = $group . '@example.com'; -SKIP: { - skip "git 2.6+ required for V2Writable", 1 if $version == 1; - use_ok 'PublicInbox::V2Writable'; -} my %opts; my $sock = tcp_server(); @@ -45,20 +46,17 @@ my $ibx = { $ibx = PublicInbox::Inbox->new($ibx); { local $ENV{HOME} = $home; - my @cmd = ('-init', $group, $inboxdir, 'http://example.com/', $addr); - push @cmd, "-V$version", '-Lbasic'; + my @cmd = ('-init', $group, $inboxdir, 'http://example.com/', $addr, + "-V$version", '-Lbasic', '--newsgroup', $group); ok(run_script(\@cmd), 'init OK'); - is(system(qw(git config), "--file=$home/.public-inbox/config", - "publicinbox.$group.newsgroup", $group), - 0, 'enabled newsgroup'); - my $len; + my $len; $ibx = PublicInbox::InboxWritable->new($ibx); - my $im = $ibx->importer; + my $im = $ibx->importer(0); # ensure successful message delivery { - my $mime = Email::MIME->new(<new(< From: =?utf-8?Q?El=C3=A9anor?= Cc: $addr @@ -74,12 +72,15 @@ EOF my $list_id = $addr; $list_id =~ s/@/./; $mime->header_set('List-Id', "<$list_id>"); - $len = length($mime->as_string); + my $str = $mime->as_string; + $str =~ s/(?add($mime); $im->done; if ($version == 1) { - my $s = PublicInbox::SearchIdx->new($ibx, 1); - $s->index_sync; + ok(run_script(['-index', $ibx->{inboxdir}]), + 'indexed v1'); } } @@ -104,11 +105,6 @@ EOF $n->can('starttls') or skip('Net::NNTP too old to support STARTTLS', 2); require_mods('IO::Socket::SSL', 2); - eval { - IO::Socket::SSL->VERSION(2.007); - } or skip(<starttls, 'STARTTLS fails when unconfigured'); is($n->code, 580, 'got 580 code on server w/o TLS'); }; @@ -203,6 +199,7 @@ EOF is_deeply($n->head(1), $n->head(''), 'HEAD OK'); is_deeply($n->body(1), $n->body(''), 'BODY OK'); + is_deeply($n->nntpstat(1), '', 'STAT'); is($n->body(1)->[0], "This is a test message for El\xc3\xa9anor\n", 'body really matches'); my $art = $n->article(1); @@ -242,7 +239,7 @@ EOF ok($date <= $t1, 'valid date before stop'); } if ('leafnode interop') { - my $for_leafnode = PublicInbox::MIME->new(<<""); + my $for_leafnode = PublicInbox::Eml->new(<<""); From: longheader\@example.com To: $addr Subject: none @@ -253,8 +250,8 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000 $im->add($for_leafnode); $im->done; if ($version == 1) { - my $s = PublicInbox::SearchIdx->new($ibx, 1); - $s->index_sync; + ok(run_script(['-index', $ibx->{inboxdir}]), + 'indexed v1'); } my $hdr = $n->head("<$long_hdr>"); my $expect = qr/\AMessage-ID: /i . qr/\Q<$long_hdr>\E/; @@ -268,7 +265,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000 $for_leafnode->header_set('Message-ID', @mids); $for_leafnode->body_set('not-a-dupe'); my $warn = ''; - $SIG{__WARN__} = sub { $warn .= join('', @_) }; + local $SIG{__WARN__} = sub { $warn .= join('', @_) }; $im->add($for_leafnode); $im->done; like($warn, qr/reused/, 'warned for reused MID'); @@ -282,33 +279,82 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000 # pipelined requests: { my $nreq = 90; + my $nart = 2; syswrite($s, "GROUP $group\r\n"); my $res = <$s>; my $rdr = fork; if ($rdr == 0) { - use POSIX qw(_exit); for (1..$nreq) { <$s> =~ /\A224 / or _exit(1); <$s> =~ /\A1/ or _exit(2); <$s> eq ".\r\n" or _exit(3); } + my %sums; + for (1..$nart) { + <$s> =~ /\A220 / or _exit(4); + my $dig = Digest::SHA->new(1); + while (my $l = <$s>) { + last if $l eq ".\r\n"; + $dig->add($l); + } + $dig = $dig->hexdigest; + $sums{$dig}++; + } + if ($nart) { + scalar(keys(%sums)) == 1 or _exit(5); + (values(%sums))[0] == $nart or _exit(6); + } _exit(0); } for (1..$nreq) { syswrite($s, "XOVER 1\r\n"); } + syswrite($s, "ARTICLE 1\r\n" x $nart); is($rdr, waitpid($rdr, 0), 'reader done'); is($? >> 8, 0, 'no errors'); } + my $noerr = { 2 => \(my $null) }; SKIP: { - if ($INC{'Search/Xapian.pm'} && ($ENV{TEST_RUN_MODE}//1)) { + if ($INC{'Search/Xapian.pm'} && ($ENV{TEST_RUN_MODE}//2)) { skip 'Search/Xapian.pm pre-loaded (by t/run.perl?)', 1; } - my @of = `lsof -p $td->{pid} 2>/dev/null`; + $lsof or skip 'lsof missing', 1; + my @of = xqx([$lsof, '-p', $td->{pid}], undef, $noerr); skip('lsof broken', 1) if (!scalar(@of) || $?); my @xap = grep m!Search/Xapian!, @of; is_deeply(\@xap, [], 'Xapian not loaded in nntpd'); } + # -compact requires Xapian + SKIP: { + require_mods('Search::Xapian', 2); + which('xapian-compact') or skip 'xapian-compact missing', 2; + is(xsys(qw(git config), "--file=$home/.public-inbox/config", + "publicinbox.$group.indexlevel", 'medium'), + 0, 'upgraded indexlevel'); + my $ex = eml_load('t/data/0001.patch'); + is($n->article($ex->header('Message-ID')), undef, + 'article did not exist'); + $im->add($ex); + $im->done; + { + my $f = $ibx->mm->{dbh}->sqlite_db_filename; + my $tmp = "$tmpdir/tmp.sqlite3"; + $ibx->mm->{dbh}->sqlite_backup_to_file($tmp); + delete $ibx->{mm}; + rename($tmp, $f) or BAIL_OUT "rename($tmp, $f): $!"; + } + ok(run_script([qw(-index -c -j0 --reindex), $ibx->{inboxdir}], + undef, $noerr), '-compacted'); + select(undef, undef, undef, $fast_idle ? 0.1 : 2.1); + $art = $n->article($ex->header('Message-ID')); + ok($art, 'new article retrieved after compact'); + $lsof or skip 'lsof missing', 1; + ($^O =~ /\A(?:linux)\z/) or + skip "lsof /(deleted)/ check untested on $^O", 1; + my @of = xqx([$lsof, '-p', $td->{pid}], undef, $noerr); + is(scalar(grep(/\(deleted\)/, @of)), 0, 'no deleted files'); + }; + SKIP: { test_watch($tmpdir, $sock, $group) }; { setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1); syswrite($s, 'HDR List-id 1-'); @@ -325,15 +371,14 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000 'got 5xx response for unoptimized HDR'); is(scalar @r, 1, 'only one response line'); } - $n = $s = undef; $td->join; - my $eout = eval { - local $/; + is($?, 0, 'no error in exited process'); + my $eout = do { open my $fh, '<', $err or die "open $err failed: $!"; + local $/; <$fh>; }; - is($?, 0, 'no error in exited process'); unlike($eout, qr/wide/i, 'no Wide character warnings'); } @@ -349,4 +394,56 @@ sub read_til_dot { $buf; } +sub test_watch { + my ($tmpdir, $sock, $group) = @_; + use_ok 'PublicInbox::Watch'; + use_ok 'PublicInbox::InboxIdle'; + use_ok 'PublicInbox::Config'; + require_git('1.8.5', 1) or skip('git 1.8.5+ needed for --urlmatch', 4); + my $old_env = { HOME => $ENV{HOME} }; + my $home = "$tmpdir/watch_home"; + mkdir $home or BAIL_OUT $!; + mkdir "$home/.public-inbox" or BAIL_OUT $!; + local $ENV{HOME} = $home; + my $name = 'watchnntp'; + my $addr = "i1\@example.com"; + my $url = "http://example.com/i1"; + my $inboxdir = "$tmpdir/watchnntp"; + my $cmd = ['-init', '-V1', '-Lbasic', $name, $inboxdir, $url, $addr]; + my ($ihost, $iport) = ($sock->sockhost, $sock->sockport); + my $nntpurl = "nntp://$ihost:$iport/$group"; + run_script($cmd) or BAIL_OUT("init $name"); + xsys(qw(git config), "--file=$home/.public-inbox/config", + "publicinbox.$name.watch", + $nntpurl) == 0 or BAIL_OUT "git config $?"; + # try again with polling + xsys(qw(git config), "--file=$home/.public-inbox/config", + 'nntp.PollInterval', 0.11) == 0 + or BAIL_OUT "git config $?"; + my $cfg = PublicInbox::Config->new; + PublicInbox::DS->Reset; + my $ii = PublicInbox::InboxIdle->new($cfg); + my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) }; + my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup'; + $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) }); + my $watcherr = "$tmpdir/watcherr"; + open my $err_wr, '>', $watcherr or BAIL_OUT $!; + open my $err, '<', $watcherr or BAIL_OUT $!; + my $w = start_script(['-watch'], undef, { 2 => $err_wr }); + + diag 'waiting for initial fetch...'; + PublicInbox::DS->EventLoop; + diag 'inbox unlocked on initial fetch'; + $w->kill; + $w->join; + is($?, 0, 'no error in exited -watch process'); + $cfg->each_inbox(sub { shift->unsubscribe_unlock('ident') }); + $ii->close; + PublicInbox::DS->Reset; + my @err = grep(!/^I:/, <$err>); + is(@err, 0, 'no warnings/errors from -watch'.join(' ', @err)); + my @ls = xqx(['git', "--git-dir=$inboxdir", qw(ls-tree -r HEAD)]); + isnt(scalar(@ls), 0, 'imported something'); +} + 1;