# MakeMaker only seems to support manpage sections 1 and 3...
m1 =
m1 += public-inbox-compact
+m1 += public-inbox-convert
m1 += public-inbox-edit
m1 += public-inbox-httpd
m1 += public-inbox-index
$(RSYNC) --chmod=Fugo=r -av $(rsync_docs) $(rsync_xdocs) $(RSYNC_DEST)
clean-doc:
- $(RM) $(man1) $(man5) $(man7) $(gz_docs) $(docs_html) $(mantxt)
+ $(RM) $(man1) $(man5) $(man7) $(man8) $(gz_docs) $(docs_html) $(mantxt)
$(RM) $(gz_xdocs) $(xdocs_html) $(xdocs)
clean :: clean-doc
v2 repositories are described in L<public-inbox-v2-format>.
-=back
-
=head1 ENVIRONMENT
=over 8
pkg: p5-TimeDate
rpm: perl-TimeDate
+* Digest::SHA typically installed with Perl
+ rpm: perl-Digest-SHA
+
* Email::MIME deb: libemail-mime-perl
pkg: p5-Email-MIME
rpm: perl-Email-MIME
(for public-inbox-watch, pulled in by Plack)
- IO::Compress::Gzip deb: perl-modules (or libio-compress-perl)
- rpm: perl-PerlIO-gzip
pkg: perl5
+ rpm: perl-IO-Compress
(for gzipped mbox over HTTP)
Uncommonly needed modules:
pkg: p5-IPC-Run
rpm: perl-IPC-Run
+- Plack::Test deb: libplack-test-perl
+ pkg: p5-Plack
+ rpm: perl-Plack-Test
+
+- Test::Simple deb: perl-modules-5.$MINOR
+ pkg: perl5
+ rpm: perl-Test-Simple
+
- XML::Feed deb: libxml-feed-perl
pkg: p5-XML-Feed
rpm: perl-XML-Feed
-- Test::HTTP::Server::Simple deb: libtest-http-server-simple-perl
- pkg: p5-Test-HTTP-Server-Simple
- rpm: perl-Test-HTTP-Server-Simple
- (XXX is this really needed?)
-
standard MakeMaker installation (Perl)
--------------------------------------
+To use MakeMaker, you need to ensure ExtUtils::MakeMaker is available.
+This is typically installed with Perl, but RPM-based systems will likely
+need to install the `perl-ExtUtils-MakeMaker' package.
+
Once the dependencies are installed, you should be able to build and
install the system (into /usr/local) with:
# We also depend on git.
# Keep this sorted and synced to the INSTALL document
'Date::Parse' => 0,
+
+ # libperl$PERL_VERSION,
+ # `perl5' on FreeBSD
+ # perl-Digest-SHA on RH-based
+ 'Digest::SHA' => 0,
'Email::MIME' => 0,
# the following should be pulled in by Email::MIME:
# We have more test dependencies, but do not force
# users to install them. See INSTALL
+
+ # All Perl installs I know about have these, but RH-based
+ # distros make them separate even though 'perl' pulls them in
+ 'File::Path' => 0,
+ 'File::Temp' => 0,
+ 'Getopt::Long' => 0,
+ 'Exporter' => 0,
+ # ExtUtils::MakeMaker # this file won't run w/o it...
},
MAN3PODS => \%man3,
);
-include config.mak
-include Documentation/include.mk
SCRIPTS := scripts/ssoma-replay
-my_syntax := \$(addsuffix .syntax, $PM_FILES \$(EXE_FILES) \$(SCRIPTS))
-
+syn_files := $PM_FILES \$(EXE_FILES) \$(SCRIPTS)
+my_syntax := \$(addsuffix .syntax, \$(syn_files))
+changed = \$(shell git ls-files -m)
%.syntax ::
- @\$(PERL) -I lib -c \$(subst .syntax,,\$@)
+ @\$(PERL) -w -I lib -c \$(subst .syntax,,\$@)
syntax:: \$(my_syntax)
+dsyn :: \$(addsuffix .syntax, \$(filter \$(changed), \$(syn_files)))
+
check-manifest :: MANIFEST
if git ls-files >\$?.gen 2>&1; then diff -u \$? \$?.gen; fi
my $pkg_fmt = shift;
@ARGV or die $usage, "\n";
+my @test_essential = qw(Test::Simple Plack::Test);
+
# package profiles
my $profiles = {
- # the smallest possible profile
+ # the smallest possible profile for testing
# TODO: trim this, Plack pulls in Filesys::Notify::Simple,
# and we don't need that for mda-only installs
essential => [ qw(
perl
Date::Parse
Devel::Peek
+ Digest::SHA
Email::Simple
Email::MIME
Email::MIME::ContentType
Encode
+ ExtUtils::MakeMaker
Filesys::Notify::Simple
Plack
URI::Escape
- ) ],
+ ), @test_essential ],
# everything optional for normal use
optional => [ qw(
xapian-compact
) ],
- # developer stuff
+ # optional developer stuff
devtest => [ qw(
IPC::Run
- Test::HTTP::Server::Simple
XML::Feed
curl
w3m
deb => 'perl', # libperl5.XX, but the XX varies
pkg => 'perl5',
},
+ 'Digest::SHA' => {
+ deb => 'perl', # libperl5.XX, but the XX varies
+ pkg => 'perl5',
+ },
'Encode' => {
deb => 'perl', # libperl5.XX, but the XX varies
pkg => 'perl5',
rpm => 'perl-Encode',
},
+ 'ExtUtils::MakeMaker' => {
+ deb => 'perl', # perl-modules-5.xx
+ pkg => 'perl5',
+ rpm => 'perl-ExtUtils-MakeMaker',
+ },
'IO::Compress::Gzip' => {
deb => 'perl', # perl-modules-5.xx
pkg => 'perl5',
- rpm => 'perl-PerlIO-gzip',
+ rpm => 'perl-IO-Compress',
},
'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
+ 'Plack::Test' => {
+ deb => 'libplack-perl',
+ pkg => 'p5-Plack',
+ rpm => 'perl-Plack-Test',
+ },
'URI::Escape' => {
deb => 'liburi-perl',
pkg => 'p5-URI',
rpm => 'perl-URI',
},
+ 'Test::Simple' => {
+ deb => 'perl', # perl-modules-5.XX, but the XX varies
+ pkg => 'perl5',
+ rpm => 'perl-Test-Simple',
+ },
'highlight.pm' => {
deb => 'libhighlight-perl',
pkg => [],
fail($self, "Unexpected result from git cat-file: $head");
my $size = $1;
- my $ref_type = $ref ? ref($ref) : '';
-
my $rv;
my $left = $size;
- $$ref = $size if ($ref_type eq 'SCALAR');
- my $cb_err;
-
- if ($ref_type eq 'CODE') {
- $rv = eval { $ref->($in, \$left) };
- $cb_err = $@;
- # drain the rest
- my $max = 8192;
- while ($left > 0) {
- my $r = read($in, my $x, $left > $max ? $max : $left);
- defined($r) or fail($self, "read failed: $!");
- $r == 0 and fail($self, 'exited unexpectedly');
- $left -= $r;
- }
- } else {
- my $offset = 0;
- my $buf = '';
- while ($left > 0) {
- my $r = read($in, $buf, $left, $offset);
- defined($r) or fail($self, "read failed: $!");
- $r == 0 and fail($self, 'exited unexpectedly');
- $left -= $r;
- $offset += $r;
- }
- $rv = \$buf;
+ $$ref = $size if $ref;
+
+ my $offset = 0;
+ my $buf = '';
+ while ($left > 0) {
+ my $r = read($in, $buf, $left, $offset);
+ defined($r) or fail($self, "read failed: $!");
+ $r == 0 and fail($self, 'exited unexpectedly');
+ $left -= $r;
+ $offset += $r;
}
+ $rv = \$buf;
- my $r = read($in, my $buf, 1);
+ my $r = read($in, my $lf, 1);
defined($r) or fail($self, "read failed: $!");
- fail($self, 'newline missing after blob') if ($r != 1 || $buf ne "\n");
- die $cb_err if $cb_err;
+ fail($self, 'newline missing after blob') if ($r != 1 || $lf ne "\n");
$rv;
}
sub set_nntp_headers ($$$$$) {
my ($self, $hdr, $ng, $n, $mid) = @_;
+ # why? leafnode requires a Path: header for some inexplicable
+ # reason. We'll fake the shortest one possible.
+ $hdr->header_set('Path', 'y');
+
+ # leafnode (and maybe other NNTP clients) have trouble dealing
+ # with v2 messages which have multiple Message-IDs (either due
+ # to our own content-based dedupe or buggy git-send-email versions).
+ my @mids = $hdr->header('Message-ID');
+ if (scalar(@mids) > 1) {
+ my $mid0 = "<$mid>";
+ $hdr->header_set('Message-ID', $mid0);
+ my @alt = $hdr->header('X-Alt-Message-ID');
+ my %seen = map { $_ => 1 } (@alt, $mid0);
+ foreach my $m (@mids) {
+ next if $seen{$m}++;
+ push @alt, $m;
+ }
+ $hdr->header_set('X-Alt-Message-ID', @alt);
+ }
+
# clobber some
my $xref = xref($self, $ng, $n, $mid);
$hdr->header_set('Xref', $xref);
my $hdr = $_[0]->header_obj->as_string;
utf8::encode($hdr);
$hdr =~ s/(?<!\r)\n/\r\n/sg;
+
+ # for leafnode compatibility, we need to ensure Message-ID headers
+ # are only a single line. We can't subclass Email::Simple::Header
+ # and override _default_fold_at in here, either; since that won't
+ # affect messages already in the archive.
+ $hdr =~ s/^(Message-ID:)[ \t]*\r\n[ \t]+([^\r]+)\r\n/$1 $2\r\n/igsm;
+
$hdr
}
}
}
return unless defined $flag;
- $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
+ my $xdb = eval { Search::Xapian::WritableDatabase->new($dir, $flag) };
+ if ($@) {
+ die "Failed opening $dir: ", $@;
+ }
+ $self->{xdb} = $xdb;
}
sub add_val ($$$) {
bless { mid => $mid }, $class;
}
-sub get {
- my ($class, $head, $db, $mid) = @_;
- my $doc_id = $head->get_docid;
- load_expand(wrap($class, $mid), $db->get_document($doc_id));
-}
-
sub get_val ($$) {
my ($doc, $col) = @_;
Search::Xapian::sortable_unserialise($doc->get_value($col));
use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
use POSIX qw(dup2);
+use strict;
+use warnings;
sub stream_to_string {
my ($res) = @_;
my $cur_int = ($cur_maj << 24) | ($cur_min << 16);
if ($cur_int < $req_int) {
return 0 if $maybe;
- plan skip_all => "git $req+ required, have $git_ver";
+ plan skip_all => "git $req+ required, have $cur_maj.$cur_min";
}
1;
}
my $raw = $gcf->cat_file($f);
is($x[2], length($$raw), 'length matches');
- {
- my $size;
- my $rv = $gcf->cat_file($f, sub {
- my ($in, $left) = @_;
- $size = $$left;
- 'nothing'
- });
- is($rv, 'nothing', 'returned from callback without reading');
- is($size, $x[2], 'set size for callback correctly');
- }
-
- eval { $gcf->cat_file($f, sub { die 'OMG' }) };
- like($@, qr/\bOMG\b/, 'died in callback propagated');
is(${$gcf->cat_file($f)}, $$raw, 'not broken after failures');
-
- {
- my ($buf, $r);
- my $rv = $gcf->cat_file($f, sub {
- my ($in, $left) = @_;
- $r = read($in, $buf, 2);
- $$left -= $r;
- 'blah'
- });
- is($r, 2, 'only read 2 bytes');
- is($buf, '--', 'partial read succeeded');
- is($rv, 'blah', 'return value propagated');
- }
is(${$gcf->cat_file($f)}, $$raw, 'not broken after partial read');
}
my $gcf = PublicInbox::Git->new($dir);
my $rsize;
- is($gcf->cat_file($buf, sub {
- $rsize = ${$_[1]};
- 'x';
- }), 'x', 'checked input');
- is($rsize, $size, 'got correct size on big file');
-
my $x = $gcf->cat_file($buf, \$rsize);
is($rsize, $size, 'got correct size ref on big file');
is(length($$x), $size, 'read correct number of bytes');
- my $rline;
- $gcf->cat_file($buf, sub {
- my ($in, $left) = @_;
- $rline = <$in>;
- $$left -= length($rline);
- });
- {
- open my $fh, '<', $big_data or die "open failed: $!\n";
- is($rline, <$fh>, 'first line matches');
- };
-
- my $all;
- $gcf->cat_file($buf, sub {
- my ($in, $left) = @_;
- my $x = read($in, $all, $$left);
- $$left -= $x;
- });
- {
- open my $fh, '<', $big_data or die "open failed: $!\n";
- local $/;
- is($all, <$fh>, 'entire read matches');
- };
-
my $ref = $gcf->qx(qw(cat-file blob), $buf);
- is($all, $ref, 'qx read giant single string');
-
my @ref = $gcf->qx(qw(cat-file blob), $buf);
- is($all, join('', @ref), 'qx returned array when wanted');
my $nl = scalar @ref;
ok($nl > 1, "qx returned array length of $nl");
ok($date >= $t0, 'valid date after start');
ok($date <= $t1, 'valid date before stop');
}
+ if ('leafnode interop') {
+ my $for_leafnode = PublicInbox::MIME->new(<<"");
+From: longheader\@example.com
+To: $addr
+Subject: none
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+
+ my $long_hdr = 'for-leafnode-'.('y'x200).'@example.com';
+ $for_leafnode->header_set('Message-ID', "<$long_hdr>");
+ $im->add($for_leafnode);
+ $im->done;
+ if ($version == 1) {
+ my $s = PublicInbox::SearchIdx->new($mainrepo, 1);
+ $s->index_sync;
+ }
+ my $hdr = $n->head("<$long_hdr>");
+ my $expect = qr/\AMessage-ID: /i . qr/\Q<$long_hdr>\E/;
+ ok(scalar(grep(/$expect/, @$hdr)), 'Message-ID not folded');
+ ok(scalar(grep(/^Path:/, @$hdr)), 'Path: header found');
+
+ # it's possible for v2 messages to have 2+ Message-IDs,
+ # but leafnode can't handle it
+ if ($version != 1) {
+ my @mids = ("<$long_hdr>", '<2mid@wtf>');
+ $for_leafnode->header_set('Message-ID', @mids);
+ $for_leafnode->body_set('not-a-dupe');
+ my $warn = '';
+ $SIG{__WARN__} = sub { $warn .= join('', @_) };
+ $im->add($for_leafnode);
+ $im->done;
+ like($warn, qr/reused/, 'warned for reused MID');
+ $hdr = $n->head('<2mid@wtf>');
+ my @hmids = grep(/\AMessage-ID: /i, @$hdr);
+ is(scalar(@hmids), 1, 'Single Message-ID in header');
+ like($hmids[0], qr/: <2mid\@wtf>/, 'got expected mid');
+ }
+ }
# pipelined requests:
{