lib/PublicInbox/RepoAtom.pm
lib/PublicInbox/RepoSnapshot.pm
lib/PublicInbox/RepoTree.pm
+lib/PublicInbox/SHA.pm
lib/PublicInbox/SaPlugin/ListMirror.pm
lib/PublicInbox/SaPlugin/ListMirror.pod
lib/PublicInbox/Search.pm
t/search-amsg.eml
t/search-thr-index.t
t/search.t
+t/sha.t
t/shared_kv.t
t/sigfd.t
t/solve/0001-simple-mod.patch
package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
use v5.12;
use Data::Dumper;
-use Digest::SHA;
+use PublicInbox::SHA;
-sub new { bless { dig => Digest::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
+sub new { bless { dig => PublicInbox::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
sub add {
$_[0]->{dig}->add($_[1]);
use PublicInbox::MsgIter;
# not sure if less-widely supported hash families are worth bothering with
-use Digest::SHA;
+use PublicInbox::SHA; # faster, but no ->clone
+use Digest::SHA; # we still need this for ->clone
sub digest_addr ($$$) {
my ($dig, $h, $v) = @_;
}
sub content_hash ($) {
- content_digest($_[0])->digest;
+ content_digest($_[0], PublicInbox::SHA->new(256))->digest;
}
+# don't clone the result of this
sub git_sha ($$) {
my ($n, $eml) = @_;
- my $dig = Digest::SHA->new($n);
+ my $dig = PublicInbox::SHA->new($n);
my $bref = ref($eml) eq 'SCALAR' ? $eml : \($eml->as_string);
- $dig->add('blob '.length($$bref)."\0");
- $dig->add($$bref);
+ $dig->add('blob '.length($$bref)."\0", $$bref);
$dig;
}
sub get_fingerprint2 {
my ($git_dir) = @_;
- require Digest::SHA;
+ require PublicInbox::SHA;
my $rd = popen_rd([qw(git show-ref)], undef, { -C => $git_dir });
- Digest::SHA::sha256(do { local $/; <$rd> });
+ PublicInbox::SHA::sha256(do { local $/; <$rd> });
}
sub writable_dir ($) {
use PublicInbox::Tmpfile;
use IO::Poll qw(POLLIN);
use Carp qw(croak carp);
-use Digest::SHA ();
+use PublicInbox::SHA ();
use PublicInbox::DS qw(awaitpid);
our @EXPORT_OK = qw(git_unquote git_quote);
our $PIPE_BUFSIZ = 65536; # Linux default
sub manifest_entry {
my ($self, $epoch, $default_desc) = @_;
my $fh = $self->popen('show-ref');
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
while (read($fh, my $buf, 65536)) {
$dig->add($buf);
}
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
package PublicInbox::LeiDedupe;
use strict;
use v5.10.1;
use PublicInbox::ContentHash qw(content_hash git_sha);
-use Digest::SHA ();
+use PublicInbox::SHA ();
# n.b. mutt sets most of these headers not sure about Bytes
our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
sub smsg_hash ($) {
my ($smsg) = @_;
- my $dig = Digest::SHA->new(256);
+ my $dig = PublicInbox::SHA->new(256);
my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)});
utf8::encode($x);
$dig->add($x);
use PublicInbox::Inbox;
use PublicInbox::LeiCurl;
use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
use POSIX qw(strftime);
our $LIVE; # pid => callback
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# pretends to be like LeiDedupe and also PublicInbox::Inbox
use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
our $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Undocumented hidden command somebody might discover if they're
package PublicInbox::LeiSucks;
use strict;
use v5.10.1;
-use Digest::SHA ();
+use PublicInbox::SHA qw(sha1_hex);
use Config;
use POSIX ();
use PublicInbox::Config;
} else {
push @out, "Xapian not available: $@\n";
}
- my $dig = Digest::SHA->new(1);
push @out, "public-inbox blob OIDs of loaded features:\n";
for my $m (grep(m{^PublicInbox/}, sort keys %INC)) {
my $f = $INC{$m} // next; # lazy require failed (missing dep)
- $dig->add('blob '.(-s $f)."\0");
- $dig->addfile($f);
- push @out, ' '.$dig->hexdigest.' '.$m."\n";
+ open my $fh, '<', $f or do { warn "open($f): $!"; next };
+ my $hex = sha1_hex('blob '.(-s $fh)."\0".
+ (do { local $/; <$fh> } // die("read: $!")));
+ push @out, ' '.$hex.' '.$m."\n";
}
push @out, <<'EOM';
Let us know how it sucks! Please include the above and any other
package PublicInbox::Linkify;
use strict;
use v5.10.1;
-use Digest::SHA qw/sha1_hex/;
+use PublicInbox::SHA qw(sha1_hex);
use PublicInbox::Hval qw(ascii_html mid_href);
use PublicInbox::MID qw($MID_EXTRACT);
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Various Message-ID-related functions.
package PublicInbox::MID;
use strict;
-use warnings;
-use base qw/Exporter/;
+use v5.10.1; # TODO: check unicode_strings compat for v5.12
+use parent qw(Exporter);
our @EXPORT_OK = qw(mid_clean id_compress mid2path mid_escape MID_ESC
mids references mids_for_index mids_in $MID_EXTRACT);
use URI::Escape qw(uri_escape_utf8);
-use Digest::SHA qw/sha1_hex/;
+use PublicInbox::SHA qw(sha1_hex);
require PublicInbox::Address;
use constant {
ID_MAX => 40, # SHA-1 hex length for HTML id anchors
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Maildirs for now, MH eventually
use strict;
use v5.10.1;
use PublicInbox::InboxWritable qw(eml_from_path);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
# returns Maildir flags from a basename ('' for no flags, undef for invalid)
sub maildir_basename_flags {
use PublicInbox::Eml;
use POSIX qw(strftime);
use PublicInbox::DS qw(now);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use Time::Local qw(timegm timelocal);
use PublicInbox::GitAsyncCat;
use PublicInbox::Address;
--- /dev/null
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# OpenSSL exception added in commit 22711f81f4e79da6b796820e37803a05cae14645
+# (README: add OpenSSL exception, 2015-10-05)
+
+# Replaces most uses of Digest::SHA with OpenSSL via Net::SSLeay if
+# possible. OpenSSL SHA-256 is nearly twice as fast as Digest::SHA on
+# x86-64, and SHA-1 is a bit faster as well.
+# I don't think we can implement Digest::SHA->clone with what Net::SSLeay
+# gives us... (maybe EVP_MD_CTX_copy+EVP_MD_CTX_copy_ex need to be added
+# to Net::SSLeay?)
+package PublicInbox::SHA;
+use v5.12;
+require Exporter;
+our @EXPORT_OK = qw(sha1_hex sha256_hex sha256);
+our @ISA;
+
+BEGIN {
+ push @ISA, 'Exporter';
+ unless (eval(<<'EOM')) {
+use Net::SSLeay 1.43;
+my %SHA = (
+ 1 => Net::SSLeay::EVP_get_digestbyname('sha1'),
+ 256 => Net::SSLeay::EVP_get_digestbyname('sha256'),
+);
+
+sub new {
+ my ($cls, $n) = @_;
+ my $mdctx = Net::SSLeay::EVP_MD_CTX_create();
+ Net::SSLeay::EVP_DigestInit($mdctx, $SHA{$n}) or
+ die "EVP_DigestInit $n: $!";
+ bless \$mdctx, $cls;
+}
+
+sub add {
+ my $self = shift;
+ Net::SSLeay::EVP_DigestUpdate($$self, $_) for @_;
+ $self;
+}
+
+sub digest { Net::SSLeay::EVP_DigestFinal(${$_[0]}) };
+sub hexdigest { unpack('H*', Net::SSLeay::EVP_DigestFinal(${$_[0]})) }
+sub DESTROY { Net::SSLeay::EVP_MD_CTX_destroy(${$_[0]}) };
+
+sub sha1_hex { unpack('H*', Net::SSLeay::SHA1($_[0])) };
+sub sha256_hex { unpack('H*', Net::SSLeay::SHA256($_[0])) };
+*sha256 = \&Net::SSLeay::SHA256;
+# end of eval
+EOM
+ require Digest::SHA; # stdlib fallback
+ push @ISA, 'Digest::SHA';
+ *sha1_hex = \&Digest::SHA::sha1_hex;
+ *sha256_hex = \&Digest::SHA::sha256_hex;
+ *sha256 = \&Digest::SHA::sha256;
+}
+
+} # /BEGIN
+1;
use parent 'PublicInbox::GzipFilter';
use POSIX qw(strftime);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use PublicInbox::Address;
use PublicInbox::Hval qw(ascii_html mid_href);
use PublicInbox::MsgTime qw(msg_timestamp);
use PublicInbox::Import;
use File::Temp;
use File::Path qw(remove_tree);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
require_mods(qw(json Plack::Builder HTTP::Date HTTP::Status));
require_git '1.8.5';
require_ok 'PublicInbox::LeiMirror';
# Usage: plackup [OPTIONS] /path/to/this/file
use v5.12;
use Plack::Builder;
-require Digest::SHA;
+require PublicInbox::SHA;
if (defined(my $f = $ENV{TEST_OPEN_FIFO})) {
open my $fh, '>', $f or die "open($f): $!";
say $fh 'hi';
my $h = [ 'Content-Type' => 'text/plain' ];
my $body = [];
if ($path eq '/sha1') {
- my $sha1 = Digest::SHA->new('SHA-1');
+ my $sha1 = PublicInbox::SHA->new(1);
my $buf;
while (1) {
my $r = $in->read($buf, 4096);
use Time::HiRes qw(gettimeofday tv_interval);
use PublicInbox::Spawn qw(spawn popen_rd);
require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use IO::Handle ();
use IO::Socket::UNIX;
use Fcntl qw(:seek);
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
use Fcntl qw(SEEK_SET);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
require_mods(qw(Storable||Sereal));
require_ok 'PublicInbox::IPC';
my ($tmpdir, $for_destroy) = tmpdir();
state $once = eval <<'';
package PublicInbox::IPC;
use strict;
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
sub test_array { qw(test array) }
sub test_scalar { 'scalar' }
sub test_scalarref { \'scalarref' }
use Socket qw(IPPROTO_TCP TCP_NODELAY);
use Sys::Hostname;
use POSIX qw(_exit);
-use Digest::SHA;
+use PublicInbox::SHA;
# t/nntpd-v2.t wraps this for v2
my $version = $ENV{PI_TEST_VERSION} || 1;
my %sums;
for (1..$nart) {
<$s> =~ /\A220 / or _exit(4);
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
while (my $l = <$s>) {
last if $l eq ".\r\n";
$dig->add($l);
--- /dev/null
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::SHA;
+use Test::More;
+
+{
+ my $dig = PublicInbox::SHA->new(1);
+ open my $fh, '<', 'COPYING' or die "open: $!";
+ $dig->add(do { local $/; <$fh> });
+ is($dig->hexdigest, '78e50e186b04c8fe1defaa098f1c192181b3d837',
+ 'AGPL-3 matches');
+}
+
+SKIP: {
+ my $n = $ENV{TEST_LEAK_NR} or skip 'TEST_LEAK_NR unset', 1;
+ for (1..$n) {
+ PublicInbox::SHA->new(1)->add('hello')->digest;
+ PublicInbox::SHA->new(1)->add('hello');
+ PublicInbox::SHA->new(1);
+ }
+}
+
+done_testing;
use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Import;
use IO::Uncompress::Gunzip qw(gunzip);
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
+require_mods(qw(json URI::Escape Plack::Builder HTTP::Tiny));
require PublicInbox::WwwListing;
require PublicInbox::ManifestJsGz;
use PublicInbox::Config;
#!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use Test::More;
use Benchmark qw(:all);
-use Digest::SHA;
+use PublicInbox::SHA;
use PublicInbox::TestCommon;
my $git_dir = $ENV{GIANT_GIT_DIR};
plan 'skip_all' => "GIANT_GIT_DIR not defined for $0" unless defined($git_dir);
my $nr = $ENV{NR} || 1;
diag "NR=$nr";
my $async = timeit($nr, sub {
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $cb = sub {
my ($bref) = @_;
$dig->add($$bref);
});
my $sync = timeit($nr, sub {
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $cat = $git->popen(@cat);
while (<$cat>) {
my ($oid, undef, undef) = split(/ /);
ok(scalar(@dig) >= 2, 'got some digests');
my $ref = shift @dig;
my $exp = $ref->[1];
-isnt($exp, Digest::SHA->new(1)->hexdigest, 'not empty');
+isnt($exp, PublicInbox::SHA->new(1)->hexdigest, 'not empty');
foreach (@dig) {
is($_->[1], $exp, "digest matches $_->[0] <=> $ref->[0]");
}
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Expensive test to validate compression and TLS.
use strict;
use v5.10.1;
use Symbol qw(gensym);
use PublicInbox::DS qw(now);
+use PublicInbox::SHA;
use POSIX qw(_exit);
use PublicInbox::TestCommon;
my $inbox_dir = $ENV{GIANT_INBOX_DIR};
my ($desc, $opt) = @_;
local $SIG{__DIE__} = sub { print STDERR $desc, ': ', @_; _exit(1) };
my $t0 = now();
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $mic = $imap_client->new(%$opt);
$mic->examine($mailbox) or die "examine: $!";
my $uid_base = 1;
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Integration test to validate compression.
use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
use POSIX qw(_exit);
use PublicInbox::TestCommon;
+use PublicInbox::SHA;
my $inbox_dir = $ENV{GIANT_INBOX_DIR};
plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
my $mid = $ENV{TEST_MID};
my ($methods) = @_;
my $desc = join(',', @$methods);
my $t0 = clock_gettime(CLOCK_MONOTONIC);
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $digfh = gensym;
my $tmpfh;
if ($File::Temp::KEEP_ALL) {