xt: add fsck script over over.sqlite3

I'm not sure what caused it, but I've noticed two missing
messages that failed from "lei up" on an https:// external;
and I've also seen some duplicates in the past (which I
think I fixed...).

diff --git a/MANIFEST b/MANIFEST
index 218e20e9056b0425e0de34392c797e55fba43271..2df743f8903c0ddd6c17ec408886e0c594a19e02 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -568,6 +568,7 @@ xt/msgtime_cmp.t
 xt/net_nntp_socks.t
 xt/net_writer-imap.t
 xt/nntpd-validate.t
+xt/over-fsck.perl
 xt/perf-msgview.t
 xt/perf-nntpd.t
 xt/perf-obfuscate.t

diff --git a/xt/over-fsck.perl b/xt/over-fsck.perl
new file mode 100644 (file)
index 0000000..053204f
--- /dev/null
+++ b/xt/over-fsck.perl
@@ -0,0 +1,44 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# unstable dev script, chasing a bug which may be in LeiSavedSearch->is_dup
+use v5.12;
+use Data::Dumper;
+use PublicInbox::OverIdx;
+@ARGV == 1 or die "Usage: $0 /path/to/over.sqlite3\n";
+my $over = PublicInbox::OverIdx->new($ARGV[0]);
+my $dbh = $over->dbh;
+$dbh->do('PRAGMA mmap_size = '.(2 ** 48));
+my $num = 0;
+my ($err, $none, $nr, $ids);
+$Data::Dumper::Useqq = $Data::Dumper::Sortkeys = 1;
+do {
+       $ids = $over->ids_after(\$num);
+       $nr += @$ids;
+       for my $n (@$ids) {
+               my $smsg = $over->get_art($n);
+               if (!$smsg) {
+                       warn "#$n article missing\n";
+                       ++$err;
+                       next;
+               }
+               my $exp = $smsg->{blob};
+               if ($exp eq '') {
+                       ++$none if $smsg->{bytes};
+                       next;
+               }
+               my $xr3 = $over->get_xref3($n, 1);
+               my $found;
+               for my $r (@$xr3) {
+                       $r->[2] = unpack('H*', $r->[2]);
+                       $found = 1 if $r->[2] eq $exp;
+               }
+               if (!$found) {
+                       warn Dumper([$smsg, $xr3 ]);
+                       ++$err;
+               }
+       }
+} while (@$ids);
+warn "$none/$nr had no blob (external?)\n" if $none;
+warn "$err errors\n" if $err;
+exit($err ? 1 : 0);