From c029c9e085ca1f1b1319e566157e04f9c972c55b Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sun, 19 Sep 2021 12:50:30 +0000
Subject: [PATCH] xt: add fsck script over over.sqlite3

I'm not sure what caused it, but I've noticed two missing
messages that failed from "lei up" on an https:// external;
and I've also seen some duplicates in the past (which I
think I fixed...).
---
 MANIFEST          |  1 +
 xt/over-fsck.perl | 44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 xt/over-fsck.perl

diff --git a/MANIFEST b/MANIFEST
index 218e20e9..2df743f8 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -568,6 +568,7 @@ xt/msgtime_cmp.t
 xt/net_nntp_socks.t
 xt/net_writer-imap.t
 xt/nntpd-validate.t
+xt/over-fsck.perl
 xt/perf-msgview.t
 xt/perf-nntpd.t
 xt/perf-obfuscate.t
diff --git a/xt/over-fsck.perl b/xt/over-fsck.perl
new file mode 100644
index 00000000..053204fe
--- /dev/null
+++ b/xt/over-fsck.perl
@@ -0,0 +1,44 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# unstable dev script, chasing a bug which may be in LeiSavedSearch->is_dup
+use v5.12;
+use Data::Dumper;
+use PublicInbox::OverIdx;
+@ARGV == 1 or die "Usage: $0 /path/to/over.sqlite3\n";
+my $over = PublicInbox::OverIdx->new($ARGV[0]);
+my $dbh = $over->dbh;
+$dbh->do('PRAGMA mmap_size = '.(2 ** 48));
+my $num = 0;
+my ($err, $none, $nr, $ids);
+$Data::Dumper::Useqq = $Data::Dumper::Sortkeys = 1;
+do {
+	$ids = $over->ids_after(\$num);
+	$nr += @$ids;
+	for my $n (@$ids) {
+		my $smsg = $over->get_art($n);
+		if (!$smsg) {
+			warn "#$n article missing\n";
+			++$err;
+			next;
+		}
+		my $exp = $smsg->{blob};
+		if ($exp eq '') {
+			++$none if $smsg->{bytes};
+			next;
+		}
+		my $xr3 = $over->get_xref3($n, 1);
+		my $found;
+		for my $r (@$xr3) {
+			$r->[2] = unpack('H*', $r->[2]);
+			$found = 1 if $r->[2] eq $exp;
+		}
+		if (!$found) {
+			warn Dumper([$smsg, $xr3 ]);
+			++$err;
+		}
+	}
+} while (@$ids);
+warn "$none/$nr had no blob (external?)\n" if $none;
+warn "$err errors\n" if $err;
+exit($err ? 1 : 0);
-- 
2.44.0