2 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
3 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
6 use Benchmark qw(:all);
7 use PublicInbox::Inbox;
9 use PublicInbox::TestCommon;
12 use PublicInbox::MsgIter;
13 require_mods(qw(Data::Dumper Email::MIME));
14 Data::Dumper->import('Dumper');
15 require PublicInbox::MIME;
17 my ($tmpdir, $for_destroy) = tmpdir();
18 my $inboxdir = $ENV{GIANT_INBOX_DIR};
19 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
20 my @cat = qw(cat-file --buffer --batch-check --batch-all-objects --unordered);
21 my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'cmp' });
23 my $fh = $git->popen(@cat);
24 vec(my $vec = '', fileno($fh), 1) = 1;
25 select($vec, undef, undef, 60) or die "timed out waiting for --batch-check";
28 my $dig_cls = 'Digest::MD5';
30 s/\s+\z//s; # E::M leaves trailing white space
36 my ($p, $cmp_arg) = @_;
38 push @$cmp_arg, '---'.join(', ', @$p).'---';
39 my $ct = $part->content_type // 'text/plain';
40 $ct =~ s/[ \t]+.*\z//s;
43 push @$cmp_arg, map { h 'f' } $part->header('From');
44 push @$cmp_arg, map { h 't' } $part->header('To');
45 push @$cmp_arg, map { h 'cc' } $part->header('Cc');
46 push @$cmp_arg, map { h 'mid' } $part->header('Message-ID');
47 push @$cmp_arg, map { h 'refs' } $part->header('References');
48 push @$cmp_arg, map { h 'irt' } $part->header('In-Reply-To');
49 push @$cmp_arg, map { h 's' } $part->header('Subject');
50 push @$cmp_arg, map { h 'cd' }
51 $part->header('Content-Description');
52 ($s, $err) = msg_part_text($part, $ct);
55 push @$cmp_arg, "S: ".$s;
58 push @$cmp_arg, "T: $ct";
59 if ($part =~ /[^\p{XPosixPrint}\s]/s) { # binary
60 my $dig = $dig_cls->new;
62 push @$cmp_arg, "M: ".$dig->hexdigest;
63 push @$cmp_arg, "B: ".length($part);
66 push @$cmp_arg, "X: ".$part;
72 push @$cmp_arg, "E: $@ ($err)";
78 my ($bref, $oid) = @_;
79 local $SIG{__WARN__} = sub { diag "$inboxdir $oid ", @_ };
81 PublicInbox::MIME->new($$bref)->each_part($cmp, my $m_ctx = [], 1);
82 PublicInbox::Eml->new($$bref)->each_part($cmp, my $e_ctx = [], 1);
83 if (join("\0", @$e_ctx) ne join("\0", @$m_ctx)) {
85 open my $fh, '>', "$tmpdir/mime" or die $!;
86 print $fh Dumper($m_ctx) or die $!;
88 open $fh, '>', "$tmpdir/eml" or die $!;
89 print $fh Dumper($e_ctx) or die $!;
91 diag "$inboxdir $oid differ";
92 # using `git diff', diff(1) may not be installed
93 diag xqx([qw(git diff), "$tmpdir/mime", "$tmpdir/eml"]);
96 my $t = timeit(1, sub {
98 my ($oid, $type) = split / /;
99 next if $type ne 'blob';
101 $git->cat_async($oid, $git_cb);
103 $git->async_wait_all;
105 is($m, $n, "$inboxdir rendered all $m <=> $n messages");
106 is($ndiff, 0, "$inboxdir $ndiff differences");