lei: reinstate JSON smsg output deduplication

[public-inbox.git] / t / lei.t

diff --git a/t/lei.t b/t/lei.t
index ef820fe3f0d046769dfb86035b942519330bdf5c..f826a96676a256b4832d80bdd0701bdcd54f6257 100644 (file)
--- a/t/lei.t
+++ b/t/lei.t
@@ -8,11 +8,16 @@ use PublicInbox::TestCommon;
 use PublicInbox::Config;
 use File::Path qw(rmtree);
 use Fcntl qw(SEEK_SET);
+use PublicInbox::Spawn qw(which);
 require_git 2.6;
 require_mods(qw(json DBD::SQLite Search::Xapian));
 my $opt = { 1 => \(my $out = ''), 2 => \(my $err = '') };
 my ($home, $for_destroy) = tmpdir();
 my $err_filter;
+my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
+       http://czquwvybam4bgbro.onion/meta/
+       http://ou63pmih66umazou.onion/meta/);
+my $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
 my $lei = sub {
        my ($cmd, $env, $xopt) = @_;
        $out = $err = '';
@@ -138,8 +143,7 @@ my $setup_publicinboxes = sub {
                my ($ibx) = @_;
                my $im = PublicInbox::InboxWritable->new($ibx)->importer(0);
                my $V = $ibx->version;
-               my @eml = glob('t/*.eml');
-               push(@eml, 't/data/0001.patch') if $V == 2;
+               my @eml = (glob('t/*.eml'), 't/data/0001.patch');
                for (@eml) {
                        next if $_ eq 't/psgi_v2-old.eml'; # dup mid
                        $im->add(eml_load($_)) or BAIL_OUT "v$V add $_";
@@ -155,6 +159,32 @@ my $setup_publicinboxes = sub {
        $seen || BAIL_OUT 'no imports';
 };
 
+my $test_external_remote = sub {
+       my ($url, $k) = @_;
+SKIP: {
+       my $nr = 4;
+       skip "$k unset", $nr if !$url;
+       which('curl') or skip 'no curl', $nr;
+       which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!;
+       $lei->('ls-external');
+       for my $e (split(/^/ms, $out)) {
+               $e =~ s/\s+boost.*//s;
+               $lei->('forget-external', '-q', $e) or
+                       fail "error forgetting $e: $err"
+       }
+       $lei->('add-external', $url);
+       my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
+       ok($lei->('q', "m:$mid"), "query $url");
+       is($err, '', "no errors on $url");
+       my $res = $json->decode($out);
+       is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
+       ok($lei->('q', "m:$mid", 'd:..20101002'), 'no results, no error');
+       like($err, qr/404/, 'noted 404');
+       is($out, "[null]\n", 'got null results');
+       $lei->('forget-external', $url);
+} # /SKIP
+}; # /sub
+
 my $test_external = sub {
        $setup_publicinboxes->();
        $cleanup->();
@@ -163,6 +193,10 @@ my $test_external = sub {
        ok(!-e $config_file && !-e $store_dir,
                'nothing created by ls-external');
 
+       ok(!$lei->('add-external', "$home/nonexistent"),
+               "fails on non-existent dir");
+       $lei->('ls-external');
+       is($out.$err, '', 'ls-external still has no output');
        my $cfg = PublicInbox::Config->new;
        $cfg->each_inbox(sub {
                my ($ibx) = @_;
@@ -180,6 +214,15 @@ my $test_external = sub {
        });
        $lei->('ls-external');
        like($out, qr/boost=0\n/s, 'ls-external has output');
+       ok($lei->(qw(add-external -q https://EXAMPLE.com/ibx)), 'add remote');
+       is($err, '', 'no warnings after add-external');
+       $lei->('ls-external');
+       like($out, qr!https://example\.com/ibx/!s, 'added canonical URL');
+       is($err, '', 'no warnings on ls-external');
+       ok($lei->(qw(forget-external -q https://EXAMPLE.com/ibx)),
+               'forget');
+       $lei->('ls-external');
+       unlike($out, qr!https://example\.com/ibx/!s, 'removed canonical URL');
 
        ok(!$lei->(qw(q s:prefix -o /dev/null -f maildir)), 'bad maildir');
        like($err, qr!/dev/null exists and is not a directory!,
@@ -203,6 +246,19 @@ my $test_external = sub {
        # No double-quoting should be imposed on users on the CLI
        $lei->('q', 's:use boolean prefix');
        like($out, qr/search: use boolean prefix/, 'phrase search got result');
+       my $res = $json->decode($out);
+       is(scalar(@$res), 2, 'only 2 element array (1 result)');
+       is($res->[1], undef, 'final element is undef'); # XXX should this be?
+       is(ref($res->[0]), 'HASH', 'first element is hashref');
+       $lei->('q', '--pretty', 's:use boolean prefix');
+       my $pretty = $json->decode($out);
+       is_deeply($res, $pretty, '--pretty is identical after decode');
+
+       for my $fmt (qw(ldjson ndjson jsonl)) {
+               $lei->('q', '-f', $fmt, 's:use boolean prefix');
+               is($out, $json->encode($pretty->[0])."\n", "-f $fmt");
+       }
+
        require IO::Uncompress::Gunzip;
        for my $sfx ('', '.gz') {
                my $f = "$home/mbox$sfx";
@@ -234,6 +290,18 @@ my $test_external = sub {
        }
        ok(!$lei->('q', '-o', "$home/mbox", 's:nope'),
                        'fails if mbox format unspecified');
+       ok(!$lei->(qw(q --no-local s:see)), '--no-local');
+       is($? >> 8, 1, 'proper exit code');
+       like($err, qr/no local or remote.+? to search/, 'no inbox');
+       my %e = (
+               TEST_LEI_EXTERNAL_HTTPS => 'https://public-inbox.org/meta/',
+               TEST_LEI_EXTERNAL_ONION => $onions[int(rand(scalar(@onions)))],
+       );
+       for my $k (keys %e) {
+               my $url = $ENV{$k} // '';
+               $url = $e{$k} if $url eq '1';
+               $test_external_remote->($url, $k);
+       }
 };
 
 my $test_lei_common = sub {