2 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
3 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
7 use PublicInbox::TestCommon;
8 use PublicInbox::Config;
9 use File::Path qw(rmtree);
10 use Fcntl qw(SEEK_SET);
11 use PublicInbox::Spawn qw(which);
12 my $req_sendcmd = 'Socket::MsgHdr or Inline::C missing or unconfigured';
13 undef($req_sendcmd) if PublicInbox::Spawn->can('send_cmd4');
14 eval { require Socket::MsgHdr; undef $req_sendcmd };
16 require_mods(qw(json DBD::SQLite Search::Xapian));
17 my $opt = { 1 => \(my $out = ''), 2 => \(my $err = '') };
18 my ($home, $for_destroy) = tmpdir();
20 my $curl = which('curl');
21 my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
22 http://czquwvybam4bgbro.onion/meta/
23 http://ou63pmih66umazou.onion/meta/);
24 my $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
26 my ($cmd, $env, $xopt) = @_;
29 ($env, $xopt) = grep { (!defined) || ref } @_;
30 $cmd = [ grep { defined && !ref } @_ ];
32 my $res = run_script(['lei', @$cmd], $env, $xopt // $opt);
34 $err = join('', grep(!/$err_filter/, split(/^/m, $err)));
38 delete local $ENV{XDG_DATA_HOME};
39 delete local $ENV{XDG_CONFIG_HOME};
40 local $ENV{GIT_COMMITTER_EMAIL} = 'lei@example.com';
41 local $ENV{GIT_COMMITTER_NAME} = 'lei user';
42 local $ENV{XDG_RUNTIME_DIR} = "$home/xdg_run";
43 local $ENV{HOME} = $home;
44 mkdir "$home/xdg_run", 0700 or BAIL_OUT "mkdir: $!";
45 my $home_trash = [ "$home/.local", "$home/.config", "$home/junk" ];
46 my $cleanup = sub { rmtree([@$home_trash, @_]) };
47 my $config_file = "$home/.config/lei/config";
48 my $store_dir = "$home/.local/share/lei";
51 ok(!$lei->(), 'no args fails');
52 is($? >> 8, 1, '$? is 1');
53 is($out, '', 'nothing in stdout');
54 like($err, qr/^usage:/sm, 'usage in stderr');
56 for my $arg (['-h'], ['--help'], ['help'], [qw(daemon-pid --help)]) {
57 ok($lei->($arg), "lei @$arg");
58 like($out, qr/^usage:/sm, "usage in stdout (@$arg)");
59 is($err, '', "nothing in stderr (@$arg)");
62 for my $arg ([''], ['--halp'], ['halp'], [qw(daemon-pid --halp)]) {
63 ok(!$lei->($arg), "lei @$arg");
64 is($? >> 8, 1, '$? set correctly');
65 isnt($err, '', 'something in stderr');
66 is($out, '', 'nothing in stdout');
68 ok($lei->(qw(init -h)), 'init -h');
69 like($out, qr! \Q$home\E/\.local/share/lei/store\b!,
70 'actual path shown in init -h');
71 ok($lei->(qw(init -h), { XDG_DATA_HOME => '/XDH' }),
72 'init with XDG_DATA_HOME');
73 like($out, qr! /XDH/lei/store\b!, 'XDG_DATA_HOME in init -h');
74 is($err, '', 'no errors from init -h');
76 ok($lei->(qw(config -h)), 'config-h');
77 like($out, qr! \Q$home\E/\.config/lei/config\b!,
78 'actual path shown in config -h');
79 ok($lei->(qw(config -h), { XDG_CONFIG_HOME => '/XDC' }),
80 'config with XDG_CONFIG_HOME');
81 like($out, qr! /XDC/lei/config\b!, 'XDG_CONFIG_HOME in config -h');
82 is($err, '', 'no errors from config -h');
85 my $ok_err_info = sub {
87 is(grep(!/^I:/, split(/^/, $err)), 0, $msg) or
88 diag "$msg: err=$err";
93 ok($lei->('init'), 'init w/o args');
94 $ok_err_info->('after init w/o args');
95 ok($lei->('init'), 'idempotent init w/o args');
96 $ok_err_info->('after idempotent init w/o args');
98 ok(!$lei->('init', "$home/x"), 'init conflict');
99 is(grep(/^E:/, split(/^/, $err)), 1, 'got error on conflict');
100 ok(!-e "$home/x", 'nothing created on conflict');
103 ok($lei->('init', "$home/x"), 'init conflict resolved');
104 $ok_err_info->('init w/ arg');
105 ok($lei->('init', "$home/x"), 'init idempotent w/ path');
106 $ok_err_info->('init idempotent w/ arg');
107 ok(-d "$home/x", 'created dir');
108 $cleanup->("$home/x");
110 ok(!$lei->('init', "$home/x", "$home/2"), 'too many args fails');
111 like($err, qr/too many/, 'noted excessive');
112 ok(!-e "$home/x", 'x not created on excessive');
113 for my $d (@$home_trash) {
114 my $base = (split(m!/!, $d))[-1];
115 ok(!-d $d, "$base not created");
117 is($out, '', 'nothing in stdout on init failure');
120 my $test_config = sub {
122 ok($lei->(qw(config a.b c)), 'config set var');
123 is($out.$err, '', 'no output on var set');
124 ok($lei->(qw(config -l)), 'config -l');
125 is($err, '', 'no errors on listing');
126 is($out, "a.b=c\n", 'got expected output');
127 ok(!$lei->(qw(config -f), "$home/.config/f", qw(x.y z)),
128 'config set var with -f fails');
129 like($err, qr/not supported/, 'not supported noted');
130 ok(!-f "$home/config/f", 'no file created');
133 my $setup_publicinboxes = sub {
135 return if $done eq $home;
136 use PublicInbox::InboxWritable;
138 run_script([qw(-init), "-V$V", "t$V",
139 '--newsgroup', "t.$V",
140 "$home/t$V", "http://example.com/t$V",
141 "t$V\@example.com" ]) or BAIL_OUT "init v$V";
143 my $cfg = PublicInbox::Config->new;
145 $cfg->each_inbox(sub {
147 my $im = PublicInbox::InboxWritable->new($ibx)->importer(0);
148 my $V = $ibx->version;
149 my @eml = (glob('t/*.eml'), 't/data/0001.patch');
151 next if $_ eq 't/psgi_v2-old.eml'; # dup mid
152 $im->add(eml_load($_)) or BAIL_OUT "v$V add $_";
157 run_script(['-index', $ibx->{inboxdir}]) or
162 $seen || BAIL_OUT 'no imports';
165 my $test_external_remote = sub {
169 skip "$k unset", $nr if !$url;
170 skip $req_sendcmd, $nr if $req_sendcmd;
171 $curl or skip 'no curl', $nr;
172 which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!;
173 my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
174 my @cmd = ('q', '--only', $url, '-q', "m:$mid");
175 ok($lei->(@cmd), "query $url");
176 is($err, '', "no errors on $url");
177 my $res = $json->decode($out);
178 is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
179 ok($lei->(@cmd, 'd:..20101002'), 'no results, no error');
180 is($err, '', 'no output on 404, matching local FS behavior');
181 is($out, "[null]\n", 'got null results');
185 my $test_external = sub {
186 $setup_publicinboxes->();
188 $lei->('ls-external');
189 is($out.$err, '', 'ls-external no output, yet');
190 ok(!-e $config_file && !-e $store_dir,
191 'nothing created by ls-external');
193 ok(!$lei->('add-external', "$home/nonexistent"),
194 "fails on non-existent dir");
195 $lei->('ls-external');
196 is($out.$err, '', 'ls-external still has no output');
197 my $cfg = PublicInbox::Config->new;
198 $cfg->each_inbox(sub {
200 ok($lei->(qw(add-external -q), $ibx->{inboxdir}),
202 is($out.$err, '', 'no output');
204 ok(-s $config_file && -e $store_dir,
205 'add-external created config + store');
206 my $lcfg = PublicInbox::Config->new($config_file);
207 $cfg->each_inbox(sub {
209 is($lcfg->{"external.$ibx->{inboxdir}.boost"}, 0,
210 "configured boost on $ibx->{name}");
212 $lei->('ls-external');
213 like($out, qr/boost=0\n/s, 'ls-external has output');
214 ok($lei->(qw(add-external -q https://EXAMPLE.com/ibx)), 'add remote');
215 is($err, '', 'no warnings after add-external');
217 ok($lei->(qw(_complete lei forget-external)), 'complete for externals');
218 my %comp = map { $_ => 1 } split(/\s+/, $out);
219 ok($comp{'https://example.com/ibx/'}, 'forget external completion');
220 $cfg->each_inbox(sub {
222 ok($comp{$ibx->{inboxdir}}, "local $ibx->{name} completion");
224 for my $u (qw(h http https https: https:/ https:// https://e
225 https://example https://example. https://example.co
226 https://example.com https://example.com/
227 https://example.com/i https://example.com/ibx)) {
228 ok($lei->(qw(_complete lei forget-external), $u),
229 "partial completion for URL $u");
230 is($out, "https://example.com/ibx/\n",
231 "completed partial URL $u");
232 for my $qo (qw(-I --include --exclude --only)) {
233 ok($lei->(qw(_complete lei q), $qo, $u),
234 "partial completion for URL q $qo $u");
235 is($out, "https://example.com/ibx/\n",
236 "completed partial URL $u on q $qo");
239 ok($lei->(qw(_complete lei add-external), 'https://'),
240 'add-external hostname completion');
241 is($out, "https://example.com/\n", 'completed up to hostname');
243 $lei->('ls-external');
244 like($out, qr!https://example\.com/ibx/!s, 'added canonical URL');
245 is($err, '', 'no warnings on ls-external');
246 ok($lei->(qw(forget-external -q https://EXAMPLE.com/ibx)),
248 $lei->('ls-external');
249 unlike($out, qr!https://example\.com/ibx/!s, 'removed canonical URL');
252 skip $req_sendcmd, 52 if $req_sendcmd;
253 ok(!$lei->(qw(q s:prefix -o /dev/null -f maildir)), 'bad maildir');
254 like($err, qr!/dev/null exists and is not a directory!,
256 is($? >> 8, 1, 'errored out with exit 1');
258 ok(!$lei->(qw(q s:prefix -f mboxcl2 -o), $home), 'bad mbox');
259 like($err, qr!\Q$home\E exists and is not a writable file!,
261 is($? >> 8, 1, 'errored out with exit 1');
263 ok(!$lei->(qw(q s:prefix -o /dev/stdout -f Mbox2)), 'bad format');
264 like($err, qr/bad mbox --format=mbox2/, 'error shown');
265 is($? >> 8, 1, 'errored out with exit 1');
267 # note, on a Bourne shell users should be able to use either:
268 # s:"use boolean prefix"
269 # "s:use boolean prefix"
270 # or use single quotes, it should not matter. Users only need
271 # to know shell quoting rules, not Xapian quoting rules.
272 # No double-quoting should be imposed on users on the CLI
273 $lei->('q', 's:use boolean prefix');
274 like($out, qr/search: use boolean prefix/, 'phrase search got result');
275 my $res = $json->decode($out);
276 is(scalar(@$res), 2, 'only 2 element array (1 result)');
277 is($res->[1], undef, 'final element is undef'); # XXX should this be?
278 is(ref($res->[0]), 'HASH', 'first element is hashref');
279 $lei->('q', '--pretty', 's:use boolean prefix');
280 my $pretty = $json->decode($out);
281 is_deeply($res, $pretty, '--pretty is identical after decode');
284 open my $fh, '+>', undef or BAIL_OUT $!;
286 print $fh 's:use' or BAIL_OUT $!;
287 seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
288 ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $fh }),
289 '--stdin on regular file works');
290 like($out, qr/use boolean prefix/, '--stdin on regular file');
293 pipe(my ($r, $w)) or BAIL_OUT $!;
294 print $w 's:use' or BAIL_OUT $!;
295 close $w or BAIL_OUT $!;
296 ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $r }),
297 '--stdin on pipe file works');
298 like($out, qr/use boolean prefix/, '--stdin on pipe');
300 ok(!$lei->(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
302 for my $fmt (qw(ldjson ndjson jsonl)) {
303 $lei->('q', '-f', $fmt, 's:use boolean prefix');
304 is($out, $json->encode($pretty->[0])."\n", "-f $fmt");
307 require IO::Uncompress::Gunzip;
308 for my $sfx ('', '.gz') {
309 my $f = "$home/mbox$sfx";
310 $lei->('q', '-o', "mboxcl2:$f", 's:use boolean prefix');
311 my $cat = $sfx eq '' ? sub {
312 open my $mb, '<', $f or fail "no mbox: $!";
315 my $z = IO::Uncompress::Gunzip->new($f, MultiStream=>1);
318 my @s = grep(/^Subject:/, $cat->());
319 is(scalar(@s), 1, "1 result in mbox$sfx");
320 $lei->('q', '-a', '-o', "mboxcl2:$f", 's:see attachment');
321 is(grep(!/^#/, $err), 0, 'no errors from augment');
322 @s = grep(/^Subject:/, my @wtf = $cat->());
323 is(scalar(@s), 2, "2 results in mbox$sfx");
325 $lei->('q', '-a', '-o', "mboxcl2:$f", 's:nonexistent');
326 is(grep(!/^#/, $err), 0, "no errors on no results ($sfx)");
328 my @s2 = grep(/^Subject:/, $cat->());
330 "same 2 old results w/ --augment and bad search $sfx");
332 $lei->('q', '-o', "mboxcl2:$f", 's:nonexistent');
334 is_deeply(\@res, [], "clobber w/o --augment $sfx");
336 ok(!$lei->('q', '-o', "$home/mbox", 's:nope'),
337 'fails if mbox format unspecified');
338 ok(!$lei->(qw(q --no-local s:see)), '--no-local');
339 is($? >> 8, 1, 'proper exit code');
340 like($err, qr/no local or remote.+? to search/, 'no inbox');
342 TEST_LEI_EXTERNAL_HTTPS => 'https://public-inbox.org/meta/',
343 TEST_LEI_EXTERNAL_ONION => $onions[int(rand(scalar(@onions)))],
345 for my $k (keys %e) {
346 my $url = $ENV{$k} // '';
347 $url = $e{$k} if $url eq '1';
348 $test_external_remote->($url, $k);
353 my $test_completion = sub {
354 ok($lei->(qw(_complete lei)), 'no errors on complete');
355 my %out = map { $_ => 1 } split(/\s+/s, $out);
356 ok($out{'q'}, "`lei q' offered as completion");
357 ok($out{'add-external'}, "`lei add-external' offered as completion");
359 ok($lei->(qw(_complete lei q)), 'complete q (no args)');
360 %out = map { $_ => 1 } split(/\s+/s, $out);
361 for my $sw (qw(-f --format -o --output --mfolder --augment -a
362 --mua --mua-cmd --no-local --local --verbose -v
363 --save-as --no-remote --remote --torsocks
365 ok($out{$sw}, "$sw offered as `lei q' completion");
368 ok($lei->(qw(_complete lei q --form)), 'complete q --format');
369 is($out, "--format\n", 'complete lei q --format');
370 for my $sw (qw(-f --format)) {
371 ok($lei->(qw(_complete lei q), $sw), "complete q $sw ARG");
372 %out = map { $_ => 1 } split(/\s+/s, $out);
373 for my $f (qw(mboxrd mboxcl2 mboxcl mboxo json jsonl
374 concatjson maildir)) {
375 ok($out{$f}, "got $sw $f as output format");
378 ok($lei->(qw(_complete lei import)), 'complete import');
379 %out = map { $_ => 1 } split(/\s+/s, $out);
380 for my $sw (qw(--flags --no-flags --no-kw --kw --no-keywords
382 ok($out{$sw}, "$sw offered as `lei import' completion");
386 my $test_fail = sub {
388 skip $req_sendcmd, 3 if $req_sendcmd;
389 $lei->(qw(q --only http://127.0.0.1:99999/bogus/ t:m));
390 is($? >> 8, 3, 'got curl exit for bogus URL');
391 $lei->(qw(q --only http://127.0.0.1:99999/bogus/ t:m -o), "$home/junk");
392 is($? >> 8, 3, 'got curl exit for bogus URL with Maildir');
393 is($out, '', 'no output');
397 my $test_lei_common = sub {
402 $test_completion->();
406 if ($ENV{TEST_LEI_ONESHOT}) {
407 require_ok 'PublicInbox::LEI';
408 # force sun_path[108] overflow, ($lei->() filters out this path)
409 my $xrd = "$home/1shot-test".('.sun_path' x 108);
410 local $ENV{XDG_RUNTIME_DIR} = $xrd;
411 $err_filter = qr!\Q$xrd!;
412 $test_lei_common->();
414 SKIP: { # real socket
415 skip $req_sendcmd, 115 if $req_sendcmd;
416 local $ENV{XDG_RUNTIME_DIR} = "$home/xdg_run";
417 my $sock = "$ENV{XDG_RUNTIME_DIR}/lei/5.seq.sock";
418 my $err_log = "$ENV{XDG_RUNTIME_DIR}/lei/errors.log";
420 ok($lei->('daemon-pid'), 'daemon-pid');
421 is($err, '', 'no error from daemon-pid');
422 like($out, qr/\A[0-9]+\n\z/s, 'pid returned') or BAIL_OUT;
423 chomp(my $pid = $out);
424 ok(kill(0, $pid), 'pid is valid');
425 ok(-S $sock, 'sock created');
427 $test_lei_common->();
428 is(-s $err_log, 0, 'nothing in errors.log');
429 open my $efh, '>>', $err_log or BAIL_OUT $!;
430 print $efh "phail\n" or BAIL_OUT $!;
431 close $efh or BAIL_OUT $!;
433 ok($lei->('daemon-pid'), 'daemon-pid');
434 chomp(my $pid_again = $out);
435 is($pid, $pid_again, 'daemon-pid idempotent');
436 like($err, qr/phail/, 'got mock "phail" error previous run');
438 ok($lei->(qw(daemon-kill)), 'daemon-kill');
439 is($out, '', 'no output from daemon-kill');
440 is($err, '', 'no error from daemon-kill');
442 kill(0, $pid) or last;
445 ok(-S $sock, 'sock still exists');
446 ok(!kill(0, $pid), 'pid gone after stop');
448 ok($lei->(qw(daemon-pid)), 'daemon-pid');
449 chomp(my $new_pid = $out);
450 ok(kill(0, $new_pid), 'new pid is running');
451 ok(-S $sock, 'sock still exists');
453 for my $sig (qw(-0 -CHLD)) {
454 ok($lei->('daemon-kill', $sig), "handles $sig");
456 is($out.$err, '', 'no output on innocuous signals');
457 ok($lei->('daemon-pid'), 'daemon-pid');
459 is($out, $new_pid, 'PID unchanged after -0/-CHLD');
461 if ('socket inaccessible') {
462 chmod 0000, $sock or BAIL_OUT "chmod 0000: $!";
463 ok($lei->('help'), 'connect fail, one-shot fallback works');
464 like($err, qr/\bconnect\(/, 'connect error noted');
465 like($out, qr/^usage: /, 'help output works');
466 chmod 0700, $sock or BAIL_OUT "chmod 0700: $!";
468 unlink $sock or BAIL_OUT "unlink($sock) $!";
470 kill('CHLD', $new_pid) or last;
473 ok(!kill(0, $new_pid), 'daemon exits after unlink');
474 # success over socket, can't test without