1 # Copyright (C) 2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # "lei inspect" general purpose inspector for stuff in SQLite and
5 # Xapian. Will eventually be useful with plain public-inboxes,
6 # not just lei/store. This is totally half-baked at the moment
7 # but useful for testing.
8 package PublicInbox::LeiInspect;
11 use PublicInbox::Config;
13 sub inspect_blob ($$) {
14 my ($lei, $oidhex) = @_;
16 if (my $lse = $lei->{lse}) {
17 my $oidbin = pack('H*', $oidhex);
18 my @docids = $lse ? $lse->over->oidbin_exists($oidbin) : ();
19 $ent->{'lei/store'} = \@docids if @docids;
21 if (my $loc = $lms ? $lms->locations_for($oidbin) : undef) {
22 $ent->{'mail-sync'} = $loc;
28 sub inspect_imap_uid ($$) {
29 my ($lei, $uid_uri) = @_;
31 my $lse = $lei->{lse} or return $ent;
32 my $lms = $lse->lms or return $ent;
33 my $oidhex = $lms->imap_oid($lei, $uid_uri);
34 if (ref(my $err = $oidhex)) { # art2folder error
35 $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
37 $ent->{$$uid_uri} = $oidhex;
41 sub inspect_sync_folder ($$) {
42 my ($lei, $folder) = @_;
44 my $lse = $lei->{lse} or return $ent;
45 my $lms = $lse->lms or return $ent;
46 my $folders = [ $folder ];
47 my $err = $lms->arg2folder($lei, $folders);
50 $lei->qerr("# no folders match $folder (non-fatal)");
53 $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
55 for my $f (@$folders) {
56 $ent->{$f} = $lms->location_stats($f); # may be undef
61 sub inspect_docid ($$;$) {
62 my ($lei, $docid, $ent) = @_;
63 require PublicInbox::Search;
66 if ($xdb = delete $ent->{xdb}) { # from inspect_num
67 } elsif (defined(my $dir = $lei->{opt}->{dir})) {
69 $xdb = $PublicInbox::Search::X{Database}->new($dir);
71 $xdb = $lei->{lse}->xdb;
73 $xdb or return $lei->fail('no Xapian DB');
74 my $doc = $xdb->get_document($docid); # raises
75 my $data = $doc->get_data;
76 $ent->{docid} = $docid;
77 $ent->{data_length} = length($data);
78 $ent->{description} = $doc->get_description;
79 $ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
80 my $cur = $doc->termlist_begin;
81 my $end = $doc->termlist_end;
82 for (; $cur != $end; $cur++) {
83 my $tn = $cur->get_termname;
84 $tn =~ s/\A([A-Z]+)// or warn "$tn no prefix! (???)";
85 my $term = ($1 // '');
86 push @{$ent->{terms}->{$term}}, $tn;
88 @$_ = sort(@$_) for values %{$ent->{terms} // {}};
89 $cur = $doc->values_begin;
90 $end = $doc->values_end;
91 for (; $cur != $end; $cur++) {
92 my $n = $cur->get_valueno;
93 my $v = $cur->get_value;
94 my $iv = PublicInbox::Search::sortable_unserialise($v);
95 $v = $iv + 0 if defined $iv;
96 # not using ->[$n] since we may have large gaps in $n
97 $ent->{'values'}->{$n} = $v;
103 my ($lei, $dir) = @_;
104 if (-f "$dir/ei.lock") {
105 require PublicInbox::ExtSearch;
106 PublicInbox::ExtSearch->new($dir);
107 } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
108 require PublicInbox::Inbox; # v2, v1
109 bless { inboxdir => $dir }, 'PublicInbox::Inbox';
111 $lei->fail("no (indexed) inbox or extindex at $dir");
115 sub inspect_num ($$) {
116 my ($lei, $num) = @_;
118 my $ent = { num => $num };
119 if (defined(my $dir = $lei->{opt}->{dir})) {
120 $ibx = dir2ibx($lei, $dir) or return;
121 if ($ent->{xdb} = $ibx->xdb) {
122 my $num2docid = $lei->{lse}->can('num2docid');
123 $docid = $num2docid->($ibx, $num);
127 $lei->{lse}->xdb; # set {nshard} for num2docid
128 $docid = $lei->{lse}->num2docid($num);
130 if ($ibx && $ibx->over) {
131 my $smsg = $ibx->over->get_art($num);
132 $ent->{smsg} = { %$smsg } if $smsg;
134 defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
137 sub inspect_mid ($$) {
138 my ($lei, $mid) = @_;
140 my $ent = { mid => $mid };
141 if (defined(my $dir = $lei->{opt}->{dir})) {
142 my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
143 $ibx = dir2ibx($lei, $dir) or return;
144 # $ent->{xdb} = $ibx->xdb //
145 # return $lei->fail("no Xapian DB for $dir");
148 $lei->{lse}->xdb; # set {nshard} for num2docid
150 if ($ibx && $ibx->over) {
152 while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
153 push @{$ent->{smsg}}, { %$smsg }
160 my ($lei, $item, $more) = @_;
162 if ($item =~ /\Ablob:(.+)/) {
163 $ent = inspect_blob($lei, $1);
164 } elsif ($item =~ m!\Aimaps?://!i) {
165 require PublicInbox::URIimap;
166 my $uri = PublicInbox::URIimap->new($item);
167 if (defined($uri->uid)) {
168 $ent = inspect_imap_uid($lei, $uri);
170 $ent = inspect_sync_folder($lei, $item);
172 } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
173 $ent = inspect_sync_folder($lei, $item);
174 } elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
175 $ent = inspect_docid($lei, $1 + 0);
176 } elsif ($item =~ m!\Anum:([0-9]+)\z!) {
177 $ent = inspect_num($lei, $1 + 0);
178 } elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
179 $ent = inspect_mid($lei, $1);
180 } else { # TODO: more things
181 return $lei->fail("$item not understood");
183 $lei->out($lei->{json}->encode($ent));
184 $lei->out(',') if $more;
189 my ($lei, @argv) = @_;
190 $lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
191 $lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
192 my $sto = $lei->_lei_store;
193 $sto ? $sto->search : undef;
195 if ($lei->{opt}->{pretty} || -t $lei->{1}) {
196 $lei->{json}->pretty(1)->indent(2);
198 $lei->start_pager if -t $lei->{1};
199 $lei->{1}->autoflush(0);
200 my $multi = scalar(@argv) > 1;
201 $lei->out('[') if $multi;
202 while (defined(my $x = shift @argv)) {
203 inspect1($lei, $x, scalar(@argv)) or return;
205 $lei->out(']') if $multi;
208 sub _complete_inspect {
209 my ($lei, @argv) = @_;
210 my $sto = $lei->_lei_store or return;
211 my $lms = $sto->search->lms or return;
212 my $match_cb = $lei->complete_url_prepare(\@argv);
213 map { $match_cb->($_) } $lms->folders;