X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fsearch.t;h=92f3305d556c5773fadc1e64f6e9f544904ff06f;hb=b714ab45d30d6f0298d73ef4281c1d0263a02493;hp=538baeff4e0afec319d0df27dba1b94f1e8fd0e3;hpb=792c643ba76e8b06ae17d3653402d381f59857c2;p=public-inbox.git diff --git a/t/search.t b/t/search.t index 538baeff..92f3305d 100644 --- a/t/search.t +++ b/t/search.t @@ -1,85 +1,87 @@ -# Copyright (C) 2015-2018 all contributors +# Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ use strict; use warnings; use Test::More; -my @mods = qw(DBI DBD::SQLite Search::Xapian); -foreach my $mod (@mods) { - eval "require $mod"; - plan skip_all => "missing $mod for $0" if $@; -}; +use PublicInbox::TestCommon; +require_mods(qw(DBD::SQLite Search::Xapian)); require PublicInbox::SearchIdx; -use File::Temp qw/tempdir/; -use Email::MIME; -my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1); +require PublicInbox::Inbox; +require PublicInbox::InboxWritable; +use PublicInbox::MIME; +my ($tmpdir, $for_destroy) = tmpdir(); my $git_dir = "$tmpdir/a.git"; +my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir }); my ($root_id, $last_id); -is(0, system(qw(git init --shared -q --bare), $git_dir), "git init (main)"); -eval { PublicInbox::Search->new($git_dir)->xdb }; +is(0, xsys(qw(git init --shared -q --bare), $git_dir), "git init (main)") + or BAIL_OUT("`git init --shared' failed, weird FS or seccomp?"); +eval { PublicInbox::Search->new($ibx)->xdb }; ok($@, "exception raised on non-existent DB"); -my $rw = PublicInbox::SearchIdx->new($git_dir, 1); -my $ibx = $rw->{-inbox}; +my $rw = PublicInbox::SearchIdx->new($ibx, 1); $ibx->with_umask(sub { $rw->_xdb_acquire; $rw->_xdb_release; }); $rw = undef; -my $ro = PublicInbox::Search->new($git_dir); +my $ro = PublicInbox::Search->new($ibx); my $rw_commit = sub { $rw->commit_txn_lazy if $rw; - $rw = PublicInbox::SearchIdx->new($git_dir, 1); + $rw = PublicInbox::SearchIdx->new($ibx, 1); $rw->{qp_flags} = 0; # quiet a warning $rw->begin_txn_lazy; }; +sub oct_is ($$$) { + my ($got, $exp, $msg) = @_; + is(sprintf('0%03o', $got), sprintf('0%03o', $exp), $msg); +} + { # git repository perms - is($ibx->_git_config_perm(), &PublicInbox::InboxWritable::PERM_GROUP, - "undefined permission is group"); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('0644')), - 0022, "644 => umask(0022)"); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('0600')), - 0077, "600 => umask(0077)"); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('0640')), - 0027, "640 => umask(0027)"); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('group')), - 0007, 'group => umask(0007)'); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('everybody')), - 0002, 'everybody => umask(0002)'); - is(PublicInbox::InboxWritable::_umask_for( - PublicInbox::InboxWritable->_git_config_perm('umask')), - umask, 'umask => existing umask'); + oct_is($ibx->_git_config_perm(), + &PublicInbox::InboxWritable::PERM_GROUP, + 'undefined permission is group'); + my @t = ( + [ '0644', 0022, '644 => umask(0022)' ], + [ '0600', 0077, '600 => umask(0077)' ], + [ '0640', 0027, '640 => umask(0027)' ], + [ 'group', 0007, 'group => umask(0007)' ], + [ 'everybody', 0002, 'everybody => umask(0002)' ], + [ 'umask', umask, 'umask => existing umask' ], + ); + for (@t) { + my ($perm, $exp, $msg) = @$_; + my $got = PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm($perm)); + oct_is($got, $exp, $msg); + } } $ibx->with_umask(sub { - my $root = Email::MIME->create( - header_str => [ - Date => 'Fri, 02 Oct 1993 00:00:00 +0000', - Subject => 'Hello world', - 'Message-ID' => '', - From => 'John Smith ', - To => 'list@example.com', - ], - body => "\\m/\n"); - my $last = Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:00 +0000', - Subject => 'Re: Hello world', - 'In-Reply-To' => '', - 'Message-ID' => '', - From => 'John Smith ', - To => 'list@example.com', - Cc => 'foo@example.com', - ], - body => "goodbye forever :<\n"); - + my $root = PublicInbox::MIME->new(<<'EOF'); +Date: Fri, 02 Oct 1993 00:00:00 +0000 +Subject: Hello world +Message-ID: +From: John Smith +To: list@example.com +List-Id: I'm not mad + +\m/ +EOF + my $last = PublicInbox::MIME->new(<<'EOF'); +Date: Sat, 02 Oct 2010 00:00:00 +0000 +Subject: Re: Hello world +In-Reply-To: +Message-ID: +From: John Smith +To: list@example.com +Cc: foo@example.com +List-Id: there's nothing + +goodbye forever :< +EOF my $rv; $rw_commit->(); $root_id = $rw->add_message($root); @@ -126,31 +128,29 @@ sub filter_mids { $ibx->with_umask(sub { $rw_commit->(); my $rmid = ''; - my $reply_to_ghost = Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:00 +0000', - Subject => 'Re: ghosts', - 'Message-ID' => '', - 'In-Reply-To' => $rmid, - From => 'Time Traveler ', - To => 'list@example.com', - ], - body => "-_-\n"); - + my $reply_to_ghost = PublicInbox::MIME->new(<<"EOF"); +Date: Sat, 02 Oct 2010 00:00:00 +0000 +Subject: Re: ghosts +Message-ID: +In-Reply-To: $rmid +From: Time Traveler +To: list\@example.com + +-_- +EOF my $rv; my $reply_id = $rw->add_message($reply_to_ghost); is($reply_id, int($reply_id), "reply_id is an integer: $reply_id"); - my $was_ghost = Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:01 +0000', - Subject => 'ghosts', - 'Message-ID' => $rmid, - From => 'Laggy Sender ', - To => 'list@example.com', - ], - body => "are real\n"); + my $was_ghost = PublicInbox::MIME->new(<<"EOF"); +Date: Sat, 02 Oct 2010 00:00:01 +0000 +Subject: ghosts +Message-ID: $rmid +From: Laggy Sender +To: list\@example.com +are real +EOF my $ghost_id = $rw->add_message($was_ghost); is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id"); my $msgs = $rw->{over}->get_thread('ghost-message@s'); @@ -191,18 +191,17 @@ $ibx->with_umask(sub { $rw_commit->(); $ro->reopen; my $long_mid = 'last' . ('x' x 60). '@s'; - - my $long = Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:00 +0000', - Subject => 'long message ID', - 'References' => ' ', - 'In-Reply-To' => '', - 'Message-ID' => "<$long_mid>", - From => '"Long I.D." ', - To => 'list@example.com', - ], - body => "wut\n"); + my $long = PublicInbox::MIME->new(< +In-Reply-To: +Message-ID: <$long_mid>, +From: "Long I.D." +To: list\@example.com + +wut +EOF my $long_id = $rw->add_message($long); is($long_id, int($long_id), "long_id is an integer: $long_id"); @@ -212,24 +211,22 @@ $ibx->with_umask(sub { my @res; my $long_reply_mid = 'reply-to-long@1'; - my $long_reply = Email::MIME->create( - header_str => [ - Subject => 'I break references', - Date => 'Sat, 02 Oct 2010 00:00:00 +0000', - 'Message-ID' => "<$long_reply_mid>", - # No References: - # 'References' => ' <'.$long_mid.'>', - 'In-Reply-To' => "<$long_mid>", - From => 'no1 ', - To => 'list@example.com', - ], - body => "no References\n"); + my $long_reply = PublicInbox::MIME->new(< +In-Reply-To: <$long_mid> +From: no1 +To: list\@example.com + +no References +EOF ok($rw->add_message($long_reply) > $long_id, "inserted long reply"); $rw_commit->(); $ro->reopen; my $t = $ro->{over_ro}->get_thread('root@s'); - is(scalar(@$t), 4, "got all 4 mesages in thread"); + is(scalar(@$t), 4, "got all 4 messages in thread"); my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_mid); @res = filter_mids($t); is_deeply(\@res, \@exp, "get_thread works"); @@ -238,25 +235,26 @@ $ibx->with_umask(sub { # quote prioritization $ibx->with_umask(sub { $rw_commit->(); - $rw->add_message(Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:01 +0000', - Subject => 'Hello', - 'Message-ID' => '', - From => 'Quoter ', - To => 'list@example.com', - ], - body => "> theatre illusions\nfade\n")); - - $rw->add_message(Email::MIME->create( - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:02 +0000', - Subject => 'Hello', - 'Message-ID' => '', - From => 'Non-Quoter', - To => 'list@example.com', - ], - body => "theatre\nfade\n")); + $rw->add_message(PublicInbox::MIME->new(<<'EOF')); +Date: Sat, 02 Oct 2010 00:00:01 +0000 +Subject: Hello +Message-ID: +From: Quoter +To: list@example.com + +> theatre illusions +fade +EOF + $rw->add_message(PublicInbox::MIME->new(<<'EOF')); +Date: Sat, 02 Oct 2010 00:00:02 +0000 +Subject: Hello +Message-ID: +From: Non-Quoter +To: list@example.com + +theatre +fade +EOF my $res = $rw->query("theatre"); is(scalar(@$res), 2, "got both matches"); is($res->[0]->mid, 'nquote@a', "non-quoted scores higher") if scalar(@$res); @@ -271,17 +269,17 @@ $ibx->with_umask(sub { # circular references $ibx->with_umask(sub { my $s = 'foo://'. ('Circle' x 15).'/foo'; - my $doc_id = $rw->add_message(Email::MIME->create( - header => [ Subject => $s ], - header_str => [ - Date => 'Sat, 02 Oct 2010 00:00:01 +0000', - 'Message-ID' => '', - 'References' => '', - 'In-Reply-To' => '', - From => 'Circle ', - To => 'list@example.com', - ], - body => "LOOP!\n")); + my $doc_id = $rw->add_message(PublicInbox::MIME->new(< +References: +In-Reply-To: +From: Circle +To: list\@example.com + +LOOP! +EOF ok($doc_id > 0, "doc_id defined with circular reference"); my $smsg = $rw->query('m:circle@a', {limit=>1})->[0]; is(defined($smsg), 1, 'found m:circl@a'); @@ -290,14 +288,7 @@ $ibx->with_umask(sub { }); $ibx->with_umask(sub { - my $str = eval { - my $mbox = 't/utf8.mbox'; - open(my $fh, '<', $mbox) or die "failed to open mbox: $mbox\n"; - local $/; - <$fh> - }; - $str =~ s/\AFrom [^\n]+\n//s; - my $mime = Email::MIME->new($str); + my $mime = mime_load 't/utf8.eml'; my $doc_id = $rw->add_message($mime); ok($doc_id > 0, 'message indexed doc_id with UTF-8'); my $msg = $rw->query('m:testmessage@example.com', {limit => 1})->[0]; @@ -319,14 +310,14 @@ $ibx->with_umask(sub { my $mset = $ro->query('t:list@example.com', {mset => 1}); is($mset->size, 6, 'searched To: successfully'); foreach my $m ($mset->items) { - my $smsg = $ro->lookup_article($m->get_docid); + my $smsg = $ro->{over_ro}->get_art($m->get_docid); like($smsg->to, qr/\blist\@example\.com\b/, 'to appears'); } $mset = $ro->query('tc:list@example.com', {mset => 1}); is($mset->size, 6, 'searched To+Cc: successfully'); foreach my $m ($mset->items) { - my $smsg = $ro->lookup_article($m->get_docid); + my $smsg = $ro->{over_ro}->get_art($m->get_docid); my $tocc = join("\n", $smsg->to, $smsg->cc); like($tocc, qr/\blist\@example\.com\b/, 'tocc appears'); } @@ -335,7 +326,7 @@ $ibx->with_umask(sub { my $mset = $ro->query($pfx . 'foo@example.com', { mset => 1 }); is($mset->items, 1, "searched $pfx successfully for Cc:"); foreach my $m ($mset->items) { - my $smsg = $ro->lookup_article($m->get_docid); + my $smsg = $ro->{over_ro}->get_art($m->get_docid); like($smsg->cc, qr/\bfoo\@example\.com\b/, 'cc appears'); } @@ -380,35 +371,7 @@ $ibx->with_umask(sub { } $ibx->with_umask(sub { - my $part1 = Email::MIME->create( - attributes => { - content_type => 'text/plain', - disposition => 'attachment', - charset => 'US-ASCII', - encoding => 'quoted-printable', - filename => 'attached_fart.txt', - }, - body_str => 'inside the attachment', - ); - my $part2 = Email::MIME->create( - attributes => { - content_type => 'text/plain', - disposition => 'attachment', - charset => 'US-ASCII', - encoding => 'quoted-printable', - filename => 'part_deux.txt', - }, - body_str => 'inside another', - ); - my $amsg = Email::MIME->create( - header_str => [ - Subject => 'see attachment', - 'Message-ID' => '', - From => 'John Smith ', - To => 'list@example.com', - ], - parts => [ $part1, $part2 ], - ); + my $amsg = mime_load 't/search-amsg.eml'; ok($rw->add_message($amsg), 'added attachment'); $rw_commit->(); $ro->reopen; @@ -432,15 +395,19 @@ $ibx->with_umask(sub { } $rw->unindex_blob($amsg); $rw->commit_txn_lazy; - is($ro->lookup_article($art->{num}), undef, 'gone from OVER DB') if defined($art); + SKIP: { + skip('$art not defined', 1) unless defined $art; + is($ro->{over_ro}->get_art($art->{num}), undef, + 'gone from OVER DB'); + }; }); my $all_mask = 07777; my $dir_mask = 02770; -# FreeBSD does not allow non-root users to set S_ISGID, so -# git doesn't set it, either (see DIR_HAS_BSD_GROUP_SEMANTICS in git.git) -if ($^O =~ /freebsd/i) { +# FreeBSD and apparently OpenBSD does not allow non-root users to set S_ISGID, +# so git doesn't set it, either (see DIR_HAS_BSD_GROUP_SEMANTICS in git.git) +if ($^O =~ /(?:free|open)bsd/i) { $all_mask = 0777; $dir_mask = 0770; } @@ -451,7 +418,7 @@ foreach my $f ("$git_dir/public-inbox/msgmap.sqlite3", glob("$git_dir/public-inbox/xapian*/*")) { my @st = stat($f); my ($bn) = (split(m!/!, $f))[-1]; - is($st[2] & $all_mask, -f _ ? 0660 : $dir_mask, + oct_is($st[2] & $all_mask, -f _ ? 0660 : $dir_mask, "sharedRepository respected for $bn"); } @@ -462,15 +429,13 @@ $ibx->with_umask(sub { my $mid = "$ua.$digits.2460-100000\@penguin.transmeta.com"; is($ro->reopen->query("m:$digits", { mset => 1})->size, 0, 'no results yet'); - my $pine = Email::MIME->create( - header_str => [ - Subject => 'blah', - 'Message-ID' => "<$mid>", - From => 'torvalds@transmeta', - To => 'list@example.com', - ], - body => "" - ); + my $pine = PublicInbox::MIME->new(< +From: torvalds\@transmeta +To: list\@example.com + +EOF my $x = $rw->add_message($pine); $rw->commit_txn_lazy; is($ro->reopen->query("m:$digits", { mset => 1})->size, 1, @@ -485,6 +450,35 @@ $ibx->with_umask(sub { is($ro->query("m:Pine m:LNX m:10010260936330", {mset=>1})->size, 1); }); +{ # List-Id searching + my $found = $ro->query('lid:i.m.just.bored'); + is_deeply([ filter_mids($found) ], [ 'root@s' ], + 'got expected mid on exact lid: search'); + + $found = $ro->query('lid:just.bored'); + is_deeply($found, [], 'got nothing on lid: search'); + + $found = $ro->query('lid:*.just.bored'); + is_deeply($found, [], 'got nothing on lid: search'); + + $found = $ro->query('l:i.m.just.bored'); + is_deeply([ filter_mids($found) ], [ 'root@s' ], + 'probabilistic search works on full List-Id contents'); + + $found = $ro->query('l:just.bored'); + is_deeply([ filter_mids($found) ], [ 'root@s' ], + 'probabilistic search works on partial List-Id contents'); + + $found = $ro->query('lid:mad'); + is_deeply($found, [], 'no match on phrase with lid:'); + + $found = $ro->query('lid:bored'); + is_deeply($found, [], 'no match on partial List-Id with lid:'); + + $found = $ro->query('l:nothing'); + is_deeply($found, [], 'matched on phrase with l:'); +} + done_testing(); 1;