X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=t%2Fsearch.t;h=54f2db906a04cd04d7cddda3bf175feed2e82c4f;hb=a46893a2b5dabfdbcf7b593ac19967daecfb1772;hp=55abe9e8bdd083d10105168731ae96f56aabcc55;hpb=6a19d29cb98823ee3794d295e122ee265ab3016a;p=public-inbox.git diff --git a/t/search.t b/t/search.t index 55abe9e8..54f2db90 100644 --- a/t/search.t +++ b/t/search.t @@ -1,14 +1,13 @@ -# Copyright (C) 2015, all contributors -# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# Copyright (C) 2015-2018 all contributors +# License: AGPL-3.0+ use strict; use warnings; use Test::More; -eval { require PublicInbox::Search; }; +eval { require PublicInbox::SearchIdx; }; plan skip_all => "Xapian missing for search" if $@; use File::Temp qw/tempdir/; use Email::MIME; -use Data::Dumper; -my $tmpdir = tempdir(CLEANUP => 1); +my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/a.git"; my ($root_id, $last_id); @@ -16,14 +15,45 @@ is(0, system(qw(git init -q --bare), $git_dir), "git init (main)"); eval { PublicInbox::Search->new($git_dir) }; ok($@, "exception raised on non-existent DB"); -my $rw = PublicInbox::Search->new($git_dir, 1); +my $rw = PublicInbox::SearchIdx->new($git_dir, 1); +my $ibx = $rw->{-inbox}; +$ibx->with_umask(sub { + $rw->_xdb_acquire; + $rw->_xdb_release; +}); +$rw = undef; my $ro = PublicInbox::Search->new($git_dir); my $rw_commit = sub { - $rw = undef; - $rw = PublicInbox::Search->new($git_dir, 1); + $rw->commit_txn_lazy if $rw; + $rw = PublicInbox::SearchIdx->new($git_dir, 1); + $rw->begin_txn_lazy; }; { + # git repository perms + is($ibx->_git_config_perm(), &PublicInbox::InboxWritable::PERM_GROUP, + "undefined permission is group"); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0644')), + 0022, "644 => umask(0022)"); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0600')), + 0077, "600 => umask(0077)"); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0640')), + 0027, "640 => umask(0027)"); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('group')), + 0007, 'group => umask(0007)'); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('everybody')), + 0002, 'everybody => umask(0002)'); + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('umask')), + umask, 'umask => existing umask'); +} + +$ibx->with_umask(sub { my $root = Email::MIME->create( header_str => [ Date => 'Fri, 02 Oct 1993 00:00:00 +0000', @@ -41,71 +71,54 @@ my $rw_commit = sub { 'Message-ID' => '', From => 'John Smith ', To => 'list@example.com', + Cc => 'foo@example.com', ], body => "goodbye forever :<\n"); my $rv; + $rw_commit->(); $root_id = $rw->add_message($root); is($root_id, int($root_id), "root_id is an integer: $root_id"); $last_id = $rw->add_message($last); is($last_id, int($last_id), "last_id is an integer: $last_id"); -} +}); sub filter_mids { - my ($res) = @_; - sort(map { $_->mid } @{$res->{msgs}}); + my ($msgs) = @_; + sort(map { $_->mid } @$msgs); } { $rw_commit->(); $ro->reopen; - my $found = $ro->lookup_message(''); - ok($found, "message found"); - is($root_id, $found->{doc_id}, 'doc_id set correctly'); - $found->ensure_metadata; - is($found->mid, 'root@s', 'mid set correctly'); - ok(int($found->thread_id) > 0, 'thread_id is an integer'); + my $found = $ro->query('m:root@s'); + is(scalar(@$found), 1, "message found"); + is($found->[0]->mid, 'root@s', 'mid set correctly'); + my ($res, @res); my @exp = sort qw(root@s last@s); - my $res = $ro->query("path:hello_world"); - my @res = filter_mids($res); - is_deeply(\@res, \@exp, 'got expected results for path: match'); - - foreach my $p (qw(hello hello_ hello_world2 hello_world_)) { - $res = $ro->query("path:$p"); - is($res->{count}, 0, "path variant `$p' does not match"); - } - $res = $ro->query('subject:(Hello world)'); + $res = $ro->query('s:(Hello world)'); @res = filter_mids($res); - is_deeply(\@res, \@exp, 'got expected results for subject:() match'); + is_deeply(\@res, \@exp, 'got expected results for s:() match'); - $res = $ro->query('subject:"Hello world"'); + $res = $ro->query('s:"Hello world"'); @res = filter_mids($res); - is_deeply(\@res, \@exp, 'got expected results for subject:"" match'); + is_deeply(\@res, \@exp, 'got expected results for s:"" match'); - $res = $ro->query('subject:"Hello world"', {limit => 1}); - is(scalar @{$res->{msgs}}, 1, "limit works"); - my $first = $res->{msgs}->[0]; + $res = $ro->query('s:"Hello world"', {limit => 1}); + is(scalar @$res, 1, "limit works"); + my $first = $res->[0]; - $res = $ro->query('subject:"Hello world"', {offset => 1}); - is(scalar @{$res->{msgs}}, 1, "offset works"); - my $second = $res->{msgs}->[0]; + $res = $ro->query('s:"Hello world"', {offset => 1}); + is(scalar @$res, 1, "offset works"); + my $second = $res->[0]; isnt($first, $second, "offset returned different result from limit"); - - foreach my $f (qw(inreplyto references)) { - $res = $ro->query($f . ':root@s'); - @res = filter_mids($res); - is_deeply(\@res, [ 'last@s' ], - "got expected results for $f: match"); - $res = $ro->query($f . ':root'); - is($res->{count}, 0, "no partial mid match"); - } } # ghost vivication -{ +$ibx->with_umask(sub { $rw_commit->(); my $rmid = ''; my $reply_to_ghost = Email::MIME->create( @@ -135,15 +148,21 @@ sub filter_mids { my $ghost_id = $rw->add_message($was_ghost); is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id"); - ok($ghost_id < $reply_id, "ghost vivified from earlier message"); -} + my $msgs = $rw->{over}->get_thread('ghost-message@s'); + is(scalar(@$msgs), 2, 'got both messages in ghost thread'); + foreach (qw(sid tid)) { + is($msgs->[0]->{$_}, $msgs->[1]->{$_}, "{$_} match"); + } + isnt($msgs->[0]->{num}, $msgs->[1]->{num}, "num do not match"); + ok($_->{num} > 0, 'positive art num') foreach @$msgs +}); # search thread on ghost { $rw_commit->(); $ro->reopen; - # Subject: + # subject my $res = $ro->query('ghost'); my @exp = sort qw(ghost-message@s ghost-reply@s); my @res = filter_mids($res); @@ -151,15 +170,21 @@ sub filter_mids { # body $res = $ro->query('goodbye'); - is($res->{msgs}->[0]->mid, 'last@s', 'got goodbye message body'); + is($res->[0]->mid, 'last@s', 'got goodbye message body'); + + # datestamp + $res = $ro->query('dt:20101002000001..20101002000001'); + @res = filter_mids($res); + is_deeply(\@res, ['ghost-message@s'], 'exact Date: match works'); + $res = $ro->query('dt:20101002000002..20101002000002'); + is_deeply($res, [], 'exact Date: match down to the second'); } # long message-id -{ +$ibx->with_umask(sub { $rw_commit->(); $ro->reopen; my $long_mid = 'last' . ('x' x 60). '@s'; - my $long_midc = Digest::SHA::sha1_hex($long_mid); my $long = Email::MIME->create( header_str => [ @@ -177,14 +202,8 @@ sub filter_mids { $rw_commit->(); $ro->reopen; - my $res = $ro->query('references:root@s'); - my @res = filter_mids($res); - is_deeply(\@res, [ sort('last@s', $long_midc) ], - "got expected results for references: match"); - - my $followups = $ro->get_followups('root@s'); - $followups = [ filter_mids($followups) ]; - is_deeply($followups, [ filter_mids($res) ], "get_followups matches"); + my $res; + my @res; my $long_reply_mid = 'reply-to-long@1'; my $long_reply = Email::MIME->create( @@ -204,14 +223,14 @@ sub filter_mids { $rw_commit->(); $ro->reopen; my $t = $ro->get_thread('root@s'); - is($t->{count}, 4, "got all 4 mesages in thread"); - my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_midc); + is(scalar(@$t), 4, "got all 4 mesages in thread"); + my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_mid); @res = filter_mids($t); is_deeply(\@res, \@exp, "get_thread works"); -} +}); # quote prioritization -{ +$ibx->with_umask(sub { $rw_commit->(); $rw->add_message(Email::MIME->create( header_str => [ @@ -233,22 +252,23 @@ sub filter_mids { ], body => "theatre\nfade\n")); my $res = $rw->query("theatre"); - is($res->{count}, 2, "got both matches"); - is($res->{msgs}->[0]->mid, 'nquote@a', "non-quoted scores higher"); - is($res->{msgs}->[1]->mid, 'quote@a', "quoted result still returned"); + is(scalar(@$res), 2, "got both matches"); + is($res->[0]->mid, 'nquote@a', "non-quoted scores higher"); + is($res->[1]->mid, 'quote@a', "quoted result still returned"); $res = $rw->query("illusions"); - is($res->{count}, 1, "got a match for quoted text"); - is($res->{msgs}->[0]->mid, 'quote@a', + is(scalar(@$res), 1, "got a match for quoted text"); + is($res->[0]->mid, 'quote@a', "quoted result returned if nothing else"); -} +}); # circular references -{ +$ibx->with_umask(sub { + my $s = 'foo://'. ('Circle' x 15).'/foo'; my $doc_id = $rw->add_message(Email::MIME->create( + header => [ Subject => $s ], header_str => [ Date => 'Sat, 02 Oct 2010 00:00:01 +0000', - Subject => 'Circle', 'Message-ID' => '', 'References' => '', 'In-Reply-To' => '', @@ -257,11 +277,189 @@ sub filter_mids { ], body => "LOOP!\n")); ok($doc_id > 0, "doc_id defined with circular reference"); - my $smsg = $rw->lookup_message('circle@a'); - $smsg->ensure_metadata; - is($smsg->{references}, undef, "no references created"); + my $smsg = $rw->query('m:circle@a', {limit=>1})->[0]; + is($smsg->references, '', "no references created"); + is($s, $smsg->subject, 'long subject not rewritten'); +}); + +$ibx->with_umask(sub { + my $str = eval { + my $mbox = 't/utf8.mbox'; + open(my $fh, '<', $mbox) or die "failed to open mbox: $mbox\n"; + local $/; + <$fh> + }; + $str =~ s/\AFrom [^\n]+\n//s; + my $mime = Email::MIME->new($str); + my $doc_id = $rw->add_message($mime); + ok($doc_id > 0, 'message indexed doc_id with UTF-8'); + my $msg = $rw->query('m:testmessage@example.com', {limit => 1})->[0]; + is($mime->header('Subject'), $msg->subject, 'UTF-8 subject preserved'); +}); + +{ + my $msgs = $ro->query('d:19931002..20101002'); + ok(scalar(@$msgs) > 0, 'got results within range'); + $msgs = $ro->query('d:20101003..'); + is(scalar(@$msgs), 0, 'nothing after 20101003'); + $msgs = $ro->query('d:..19931001'); + is(scalar(@$msgs), 0, 'nothing before 19931001'); } +# names and addresses +{ + my $mset = $ro->query('t:list@example.com', {mset => 1}); + is($mset->size, 6, 'searched To: successfully'); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); + like($smsg->to, qr/\blist\@example\.com\b/, 'to appears'); + } + + $mset = $ro->query('tc:list@example.com', {mset => 1}); + is($mset->size, 6, 'searched To+Cc: successfully'); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); + my $tocc = join("\n", $smsg->to, $smsg->cc); + like($tocc, qr/\blist\@example\.com\b/, 'tocc appears'); + } + + foreach my $pfx ('tcf:', 'c:') { + my $mset = $ro->query($pfx . 'foo@example.com', { mset => 1 }); + is($mset->items, 1, "searched $pfx successfully for Cc:"); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); + like($smsg->cc, qr/\bfoo\@example\.com\b/, + 'cc appears'); + } + } + + foreach my $pfx ('', 'tcf:', 'f:') { + my $res = $ro->query($pfx . 'Laggy'); + is(scalar(@$res), 1, + "searched $pfx successfully for From:"); + foreach my $smsg (@$res) { + like($smsg->from, qr/Laggy Sender/, + "From appears with $pfx"); + } + } +} + +{ + $rw_commit->(); + $ro->reopen; + my $res = $ro->query('b:hello'); + is(scalar(@$res), 0, 'no match on body search only'); + $res = $ro->query('bs:smith'); + is(scalar(@$res), 0, + 'no match on body+subject search for From'); + + $res = $ro->query('q:theatre'); + is(scalar(@$res), 1, 'only one quoted body'); + like($res->[0]->from, qr/\AQuoter/, 'got quoted body'); + + $res = $ro->query('nq:theatre'); + is(scalar @$res, 1, 'only one non-quoted body'); + like($res->[0]->from, qr/\ANon-Quoter/, 'got non-quoted body'); + + foreach my $pfx (qw(b: bs:)) { + $res = $ro->query($pfx . 'theatre'); + is(scalar @$res, 2, "searched both bodies for $pfx"); + like($res->[0]->from, qr/\ANon-Quoter/, + "non-quoter first for $pfx"); + } +} + +$ibx->with_umask(sub { + my $part1 = Email::MIME->create( + attributes => { + content_type => 'text/plain', + disposition => 'attachment', + charset => 'US-ASCII', + encoding => 'quoted-printable', + filename => 'attached_fart.txt', + }, + body_str => 'inside the attachment', + ); + my $part2 = Email::MIME->create( + attributes => { + content_type => 'text/plain', + disposition => 'attachment', + charset => 'US-ASCII', + encoding => 'quoted-printable', + filename => 'part_deux.txt', + }, + body_str => 'inside another', + ); + my $amsg = Email::MIME->create( + header_str => [ + Subject => 'see attachment', + 'Message-ID' => '', + From => 'John Smith ', + To => 'list@example.com', + ], + parts => [ $part1, $part2 ], + ); + ok($rw->add_message($amsg), 'added attachment'); + $rw_commit->(); + $ro->reopen; + my $n = $ro->query('n:attached_fart.txt'); + is(scalar @$n, 1, 'got result for n:'); + my $res = $ro->query('part_deux.txt'); + is(scalar @$res, 1, 'got result without n:'); + is($n->[0]->mid, $res->[0]->mid, + 'same result with and without'); + my $txt = $ro->query('"inside another"'); + is($txt->[0]->mid, $res->[0]->mid, + 'search inside text attachments works'); + + my $mid = $n->[0]->mid; + my ($id, $prev); + my $art = $ro->next_by_mid($mid, \$id, \$prev); + ok($art, 'article exists in OVER DB'); + $rw->unindex_blob($amsg); + $rw->commit_txn_lazy; + is($ro->lookup_article($art->{num}), undef, 'gone from OVER DB'); +}); + +foreach my $f ("$git_dir/public-inbox/msgmap.sqlite3", + glob("$git_dir/public-inbox/xapian*/"), + glob("$git_dir/public-inbox/xapian*/*")) { + my @st = stat($f); + my ($bn) = (split(m!/!, $f))[-1]; + is($st[2] & 07777, -f _ ? 0660 : 0770, + "sharedRepository respected for $bn"); +} + +$ibx->with_umask(sub { + $rw_commit->(); + my $digits = '10010260936330'; + my $ua = 'Pine.LNX.4.10'; + my $mid = "$ua.$digits.2460-100000\@penguin.transmeta.com"; + is($ro->reopen->query("m:$digits", { mset => 1})->size, 0, + 'no results yet'); + my $pine = Email::MIME->create( + header_str => [ + Subject => 'blah', + 'Message-ID' => "<$mid>", + From => 'torvalds@transmeta', + To => 'list@example.com', + ], + body => "" + ); + my $x = $rw->add_message($pine); + $rw->commit_txn_lazy; + is($ro->reopen->query("m:$digits", { mset => 1})->size, 1, + 'searching only digit yielded result'); + + my $wild = $digits; + for my $i (1..6) { + chop($wild); + is($ro->query("m:$wild*", { mset => 1})->size, 1, + "searching chopped($i) digit yielded result $wild "); + } + is($ro->query("m:Pine m:LNX m:10010260936330", {mset=>1})->size, 1); +}); + done_testing(); 1;