1 # Copyright (C) 2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # "lei inspect" general purpose inspector for stuff in SQLite and
5 # Xapian. Will eventually be useful with plain public-inboxes,
6 # not just lei/store. This is totally half-baked at the moment
7 # but useful for testing.
8 package PublicInbox::LeiInspect;
11 use parent qw(PublicInbox::IPC);
12 use PublicInbox::Config;
13 use PublicInbox::MID qw(mids);
14 use PublicInbox::NetReader qw(imap_uri nntp_uri);
18 $smsg->{$_} += 0 for qw(bytes lines); # integerize
19 +{ %$smsg } # unbless and scalarize
22 sub inspect_blob ($$) {
23 my ($lei, $oidhex) = @_;
25 if (my $lse = $lei->{lse}) {
26 my $oidbin = pack('H*', $oidhex);
27 my @docids = $lse ? $lse->over->oidbin_exists($oidbin) : ();
28 $ent->{'lei/store'} = \@docids if @docids;
30 if (my $loc = $lms ? $lms->locations_for($oidbin) : undef) {
31 $ent->{'mail-sync'} = $loc;
37 sub inspect_imap_uid ($$) {
38 my ($lei, $uid_uri) = @_;
40 my $lms = $lei->lms or return $ent;
41 my @oidhex = $lms->imap_oidhex($lei, $uid_uri);
42 $ent->{$$uid_uri} = @oidhex == 1 ? $oidhex[0] :
43 ((@oidhex == 0) ? undef : \@oidhex);
47 sub inspect_nntp_range {
49 my ($ng, $beg, $end) = $uri->group;
53 my $ret = { "$uri" => $ent };
54 my $lms = $lei->lms or return $ret;
55 my $folders = [ $$uri ];
56 eval { $lms->arg2folder($lei, $folders) };
57 $lei->qerr("# no folders match $$uri (non-fatal)") if $@;
59 for my $art ($beg..$end) {
60 my @oidhex = map { unpack('H*', $_) }
61 $lms->num_oidbin($folders->[0], $art);
62 $ent->{$art} = @oidhex == 1 ? $oidhex[0] :
63 ((@oidhex == 0) ? undef : \@oidhex);
68 sub inspect_sync_folder ($$) {
69 my ($lei, $folder) = @_;
71 my $lms = $lei->lms or return $ent;
72 my $folders = [ $folder ];
73 eval { $lms->arg2folder($lei, $folders) };
74 $lei->qerr("# no folders match $folder (non-fatal)") if $@;
75 for my $f (@$folders) {
76 $ent->{$f} = $lms->location_stats($f); # may be undef
81 sub _inspect_doc ($$) {
83 my $data = $doc->get_data;
84 $ent->{data_length} = length($data);
85 $ent->{description} = $doc->get_description;
86 $ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
87 my $cur = $doc->termlist_begin;
88 my $end = $doc->termlist_end;
89 for (; $cur != $end; $cur++) {
90 my $tn = $cur->get_termname;
91 $tn =~ s/\A([A-Z]+)// or warn "$tn no prefix! (???)";
92 my $term = ($1 // '');
93 push @{$ent->{terms}->{$term}}, $tn;
95 @$_ = sort(@$_) for values %{$ent->{terms} // {}};
96 $cur = $doc->values_begin;
97 $end = $doc->values_end;
98 for (; $cur != $end; $cur++) {
99 my $n = $cur->get_valueno;
100 my $v = $cur->get_value;
101 my $iv = PublicInbox::Search::sortable_unserialise($v);
102 $v = $iv + 0 if defined $iv;
103 # not using ->[$n] since we may have large gaps in $n
104 $ent->{'values'}->{$n} = $v;
109 sub inspect_docid ($$;$) {
110 my ($lei, $docid, $ent) = @_;
111 require PublicInbox::Search;
114 if ($xdb = delete $ent->{xdb}) { # from inspect_num
115 } elsif (defined(my $dir = $lei->{opt}->{dir})) {
117 $xdb = $PublicInbox::Search::X{Database}->new($dir);
118 } elsif ($lei->{lse}) {
119 $xdb = $lei->{lse}->xdb;
121 $xdb or return $lei->fail('no Xapian DB');
122 my $doc = $xdb->get_document($docid); # raises
123 $ent->{docid} = $docid;
124 _inspect_doc($ent, $doc);
128 my ($lei, $dir) = @_;
129 if (-f "$dir/ei.lock") {
130 require PublicInbox::ExtSearch;
131 PublicInbox::ExtSearch->new($dir);
132 } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
133 require PublicInbox::Inbox; # v2, v1
134 bless { inboxdir => $dir }, 'PublicInbox::Inbox';
136 $lei->fail("no (indexed) inbox or extindex at $dir");
140 sub inspect_num ($$) {
141 my ($lei, $num) = @_;
143 my $ent = { num => $num };
144 if (defined(my $dir = $lei->{opt}->{dir})) {
145 $ibx = dir2ibx($lei, $dir) or return;
146 if (my $srch = $ibx->search) {
147 $ent->{xdb} = $srch->xdb and
148 $docid = $srch->num2docid($num);
150 } elsif ($lei->{lse}) {
152 $lei->{lse}->xdb; # set {nshard} for num2docid
153 $docid = $lei->{lse}->num2docid($num);
155 if ($ibx && $ibx->over) {
156 my $smsg = $ibx->over->get_art($num);
157 $ent->{smsg} = _json_prep($smsg) if $smsg;
159 defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
162 sub inspect_mid ($$) {
163 my ($lei, $mid) = @_;
165 my $ent = { mid => $mid };
166 if (defined(my $dir = $lei->{opt}->{dir})) {
167 $ibx = dir2ibx($lei, $dir)
171 if ($ibx && $ibx->over) {
173 while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
174 push @{$ent->{smsg}}, _json_prep($smsg);
177 if ($ibx && $ibx->search) {
178 my $mset = $ibx->search->mset(qq{mid:"$mid"});
179 for (sort { $a->get_docid <=> $b->get_docid } $mset->items) {
180 my $tmp = { docid => $_->get_docid };
181 _inspect_doc($tmp, $_->get_document);
182 push @{$ent->{xdoc}}, $tmp;
189 my ($lei, $item, $more) = @_;
191 if ($item =~ /\Ablob:(.+)/) {
192 $ent = inspect_blob($lei, $1);
193 } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
194 $ent = inspect_sync_folder($lei, $item);
195 } elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
196 $ent = inspect_docid($lei, $1 + 0);
197 } elsif ($item =~ m!\Anum:([0-9]+)\z!) {
198 $ent = inspect_num($lei, $1 + 0);
199 } elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
200 $ent = inspect_mid($lei, $1);
201 } elsif (my $iuri = imap_uri($item)) {
202 if (defined($iuri->uid)) {
203 $ent = inspect_imap_uid($lei, $iuri);
205 $ent = inspect_sync_folder($lei, $item);
207 } elsif (my $nuri = nntp_uri($item)) {
208 if (defined(my $mid = $nuri->message)) {
209 $ent = inspect_mid($lei, $mid);
211 my ($group, $beg, $end) = $nuri->group;
213 $ent = inspect_nntp_range($lei, $nuri);
215 $ent = inspect_sync_folder($lei, $item);
218 } else { # TODO: more things
219 return $lei->fail("$item not understood");
221 $lei->out($lei->{json}->encode($ent));
222 $lei->out(',') if $more;
226 sub inspect_argv { # via wq_do
228 my ($lei, $argv) = delete @$self{qw(lei argv)};
229 my $multi = scalar(@$argv) > 1;
230 $lei->{1}->autoflush(0);
231 $lei->out('[') if $multi;
232 while (defined(my $x = shift @$argv)) {
233 inspect1($lei, $x, scalar(@$argv)) or return;
235 $lei->out(']') if $multi;
238 sub inspect_start ($$) {
239 my ($lei, $argv) = @_;
240 my $self = bless { lei => $lei, argv => $argv }, __PACKAGE__;
241 my ($op_c, $ops) = $lei->workers_start($self, 1);
243 $lei->wait_wq_events($op_c, $ops);
244 $self->wq_do('inspect_argv');
248 sub ins_add { # InputPipe->consume callback
249 my ($lei) = @_; # $_[1] = $rbuf
251 $_[1] eq '' and return eval {
252 my $str = delete $lei->{istr};
253 $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
254 my $eml = PublicInbox::Eml->new(\$str);
255 inspect_start($lei, [
256 'blob:'.$lei->git_oid($eml)->hexdigest,
257 map { "mid:$_" } @{mids($eml)} ]);
259 $lei->{istr} .= $_[1];
261 $lei->fail("error reading stdin: $!");
266 my ($lei, @argv) = @_;
267 $lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
268 $lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
269 my $sto = $lei->_lei_store;
270 $sto ? $sto->search : undef;
272 my $isatty = -t $lei->{1};
273 $lei->{json}->pretty(1)->indent(2) if $lei->{opt}->{pretty} || $isatty;
274 $lei->start_pager if $isatty;
275 if ($lei->{opt}->{stdin}) {
276 return $lei->fail(<<'') if @argv;
277 no args allowed on command-line with --stdin
279 require PublicInbox::InputPipe;
280 PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
282 inspect_start($lei, \@argv);
286 sub _complete_inspect {
287 require PublicInbox::LeiRefreshMailSync;
288 PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync(@_);
289 # TODO: message-ids?, blobs? could get expensive...
292 sub input_only_atfork_child {
294 $self->{lei}->_lei_atfork_child;
295 $self->SUPER::ipc_atfork_child;