Clients should use normal git-clone/git-fetch, or NNTP clients
if they want to import mail into their personal inboxes.
+public-inbox is developed on Debian GNU/Linux systems and will
+never depend on packages outside of the "main" component of
+the "stable" distribution, currently Debian 9.x ("stretch")
+
+Most packages are available in other GNU/Linux distributions;
+and FreeBSD support can happen.
+
TODO: this still needs to be documented better,
also see the scripts/ and sa_config/ directories in the source tree
public-inbox requires a number of other packages to access its full
functionality. The core tools are, of course:
-* Git
-* Perl
+* Git (1.8.0+, 2.6+ for writing v2 repositories)
+* Perl 5.8+
* SQLite (needed for Xapian use)
To accept incoming mail into a public inbox, you'll likely want:
Beyond that, there is a long list of Perl modules required, starting with:
-* Date::Parse deb: libdatetime-perl
+* Date::Parse deb: libtimedate-perl
rpm: perl-Time-ParseDate
* Email::MIME deb: libemail-mime-perl
Numerous optional modules are likely to be useful as well:
- - Socket6 deb: libsocket6-perl
- rpm: perl-Socket6
- (for IPv6 support)
-
- Search::Xapian deb: libsearch-xapian-perl
rpm: perl-Search-Xapian
(for NNTP service or gzipped mbox over HTTP)
- Net::Server deb: libnet-server-perl
rpm: perl-Net-Server
- (for HTTP/NNTP servers as standalone daemons)
+ (for HTTP/NNTP servers as standalone daemons,
+ not needed as systemd services)
- Filesys::Notify::Simple deb: libfilesys-notify-simple-perl
rpm: perl-Filesys-Notify-Simple
(for public-inbox-watch)
- Inline::C[7] deb: libinline-c-perl
- (speeds up spawning on Linux
+ (speeds up spawning of git(1) on Linux
(see public-inbox-daemon(8))
- Plack::Middleware::ReverseProxy
rpm: perl-Plack-Middleware-Deflater
(saves bandwidth on responses)
+ - Socket6 deb: libsocket6-perl
+ rpm: perl-Socket6
+ (pulled in by SpamAssassin and Net::Server,
+ only necessary if using IPv6 with
+ Plack::Middleware::AccessLog or similar)
On Fedora systems, you'll probably also end up wanting
perl-Test-HTTP-Server-Simple, perl-Devel-Peek, and perl-IPC-Run to run the
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Used for giving serial numbers to messages. This can be tied to
+# the msgmap for live updates to living lists (see
+# PublicInbox::Filters::RubyLang), or kept separate for imports
+# of defunct NNTP groups (e.g. scripts/xhdr-num2mid)
+#
+# Introducing NEW uses of serial numbers is discouraged because of
+# it leads to reliance on centralization. However, being able
+# to use existing serial numbers is beneficial.
package PublicInbox::AltId;
use strict;
use warnings;
use URI::Escape qw(uri_unescape);
# spec: TYPE:PREFIX:param1=value1¶m2=value2&...
+# The PREFIX will be a searchable boolean prefix in Xapian
# Example: serial:gmane:file=/path/to/altmsgmap.sqlite3
sub new {
my ($class, $inbox, $spec, $writable) = @_;
# Copyright (C) 2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Unstable internal API.
+# Used for on-the-fly duplicate detection in V2 inboxes.
+# This is not stored in any database anywhere and may change
+# as changes in duplicate detection are needed.
+# See L<public-inbox-v2-format(5)> manpage for more details.
package PublicInbox::ContentId;
use strict;
use warnings;
"$host:$port";
}
+sub unpack_ipv6 ($) {
+ my ($addr) = @_;
+
+ # TODO: support IO::Socket::IP which comes with Perl 5.24
+ # (perl-modules-5.24 in Debian)
+
+ # SpamAssassin and Net::Server use Socket6, so it may be installed
+ # on our system, already:
+ eval { require Socket6 } or return ('???-Socket6-missing', 0);
+
+ my ($port, $host) = Socket6::unpack_sockaddr_in6($addr);
+ $host = Socket6::inet_ntop(Socket6::AF_INET6(), $host);
+ ($host, $port);
+}
+
sub host_with_port ($) {
my ($addr) = @_;
my ($port, $host);
# this eval will die on Unix sockets:
eval {
if (length($addr) >= 28) {
- require Socket6;
- ($port, $host) = Socket6::unpack_sockaddr_in6($addr);
- $host = Socket6::inet_ntop(Socket6::AF_INET6(), $host);
+ ($host, $port) = unpack_ipv6($addr);
$host = "[$host]";
} else {
($port, $host) = Socket::sockaddr_in($addr);
# Wrap a pipe or file for PSGI streaming response bodies and calls the
# end callback when the object goes out-of-scope.
# This depends on rpipe being _blocking_ on getline.
+#
+# public-inbox-httpd favors "getline" response bodies to take a
+# "pull"-based approach to feeding slow clients (as opposed to a
+# more common "push" model)
package PublicInbox::GetlineBody;
use strict;
use warnings;
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# when no endpoints match, fallback to this and serve a static file
-# or smart HTTP
+# or smart HTTP. This is our wrapper for git-http-backend(1)
package PublicInbox::GitHTTPBackend;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# wraps a listen socket for HTTP and links it to the PSGI app in
+# public-inbox-httpd
package PublicInbox::HTTPD;
use strict;
use warnings;
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# git fast-import-based ssoma-mda MDA replacement
-# This is only ever run by public-inbox-mda and public-inbox-learn,
-# not the WWW or NNTP code which only requires read-only access.
+# This is only ever run by public-inbox-mda, public-inbox-learn
+# and public-inbox-watch. Not the WWW or NNTP code which only
+# requires read-only access.
package PublicInbox::Import;
use strict;
use warnings;
# Copyright (C) 2015-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Streaming interface for formatting messages as an mboxrd.
-# Used by the web interface
+# Streaming (via getline) interface for formatting messages as an mboxrd.
+# Used by the PSGI web interface.
+#
+# public-inbox-httpd favors "getline" response bodies to take a
+# "pull"-based approach to feeding slow clients (as opposed to a
+# more common "push" model)
package PublicInbox::Mbox;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
+
+# read-only utilities for Email::MIME
package PublicInbox::MsgIter;
use strict;
use warnings;
# Copyright (C) 2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Various date/time-related functions
package PublicInbox::MsgTime;
use strict;
use warnings;
# Copyright (C) 2014-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# For reply instructions and address generation in WWW UI
package PublicInbox::Reply;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Rules useful for running a mailing list mirror. We want to:
+# SpamAssassin rules useful for running a mailing list mirror. We want to:
# * ensure Received: headers are really from the list mail server
# users expect. This is to prevent malicious users from
# injecting spam into mirrors without going through the expected
--raw -r --no-abbrev/, $range);
}
+# --is-ancestor requires git 1.8.0+
sub is_ancestor ($$$) {
my ($git, $cur, $tip) = @_;
return 0 unless $git->check($cur);
# Copyright (C) 2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# used to interface with a single Xapian partition in V2 repos.
+# See L<public-inbox-v2-format(5)> for more info on how we partition Xapian
package PublicInbox::SearchIdxPart;
use strict;
use warnings;
# based on notmuch, but with no concept of folders, files or flags
#
# Wraps a document inside our Xapian search index.
+# There may be many of these objects loaded in memory at once
+# for large threads in our WWW UI.
package PublicInbox::SearchMsg;
use strict;
use warnings;
# Copyright (C) 2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Spamchecking used by -watch and -mda tools
package PublicInbox::Spamcheck;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Default spam filter class for wrapping spamc(1)
package PublicInbox::Spamcheck::Spamc;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Pure-Perl implementation of "spawn". This can't take advantage
+# of vfork, so no speedups under Linux for spawning from large processes.
package PublicInbox::SpawnPP;
use strict;
use warnings;
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# This interface wraps and mimics PublicInbox::Import
+# Used to write to V2 inboxes (see L<public-inbox-v2-format(5)>).
package PublicInbox::V2Writable;
use strict;
use warnings;
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Atom body stream for which yields getline+close methods
+# public-inbox-httpd favors "getline" response bodies to take a
+# "pull"-based approach to feeding slow clients (as opposed to a
+# more common "push" model)
package PublicInbox::WwwAtomStream;
use strict;
use warnings;
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# HTML body stream for which yields getline+close methods
+#
+# public-inbox-httpd favors "getline" response bodies to take a
+# "pull"-based approach to feeding slow clients (as opposed to a
+# more common "push" model)
package PublicInbox::WwwStream;
use strict;
use warnings;
# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# used for displaying help texts and other non-mail content
package PublicInbox::WwwText;
use strict;
use warnings;
use warnings;
use Test::More;
use File::Temp qw/tempdir/;
+require './t/common.perl';
+require_git(2.6);
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
eval "require $mod";
plan skip_all => "$mod missing for altid_v2.t" if $@;
$pid;
}
+sub require_git ($;$) {
+ my ($req, $maybe) = @_;
+ my ($req_maj, $req_min) = split(/\./, $req);
+ my ($cur_maj, $cur_min) = (`git --version` =~ /version (\d+)\.(\d+)/);
+
+ my $req_int = ($req_maj << 24) | ($req_min << 16);
+ my $cur_int = ($cur_maj << 24) | ($cur_min << 16);
+ if ($cur_int < $req_int) {
+ return 0 if $maybe;
+ plan skip_all => "git $req+ required, have $git_ver";
+ }
+ 1;
+}
+
1;
use Test::More;
use File::Temp qw/tempdir/;
use PublicInbox::MIME;
+require './t/common.perl';
+require_git(2.6);
my @mods = qw(DBD::SQLite Search::Xapian);
foreach my $mod (@mods) {
eval "require $mod";
use Fcntl qw(:DEFAULT);
use File::Temp qw/tempdir tempfile/;
my $dir = tempdir('pi-import-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
is(system(qw(git init -q --bare), $dir), 0, 'git init successful');
my $git = PublicInbox::Git->new($dir);
],
body => "hello world\n",
);
+my $v2 = require_git(2.6, 1);
-$im->{want_object_info} = 1 if 'v2';
+$im->{want_object_info} = 1 if $v2;
like($im->add($mime), qr/\A:\d+\z/, 'added one message');
-if ('v2') {
+if ($v2) {
my $info = $im->{last_object};
like($info->[0], qr/\A[a-f0-9]{40}\z/, 'got last object_id');
is($mime->as_string, ${$info->[2]}, 'string matches');
use Test::More;
use PublicInbox::Config;
use File::Temp qw/tempdir/;
+require './t/common.perl';
my $tmpdir = tempdir('pi-init-XXXXXX', TMPDIR => 1, CLEANUP => 1);
use constant pi_init => 'blib/script/public-inbox-init';
use PublicInbox::Import;
eval "require $mod";
skip "$mod missing for v2", 2 if $@;
}
+ require_git(2.6, 1) or skip "git 2.6+ required", 2;
local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
my $cfgfile = "$ENV{PI_DIR}/config";
my @cmd = (pi_init, '-V2', 'v2list', "$tmpdir/v2list",
use File::Temp qw/tempdir/;
use PublicInbox::MIME;
use PublicInbox::Config;
+require './t/common.perl';
+require_git(2.6);
my @mods = qw(DBD::SQLite Search::Xapian IPC::Run);
foreach my $mod (@mods) {
eval "require $mod";
use Sys::Hostname;
require './t/common.perl';
+# FIXME: make easier to test both versions
+my $version = $ENV{PI_VERSION} || 2;
+require_git('2.6') if $version == 2;
+
my $tmpdir = tempdir('pi-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
my $home = "$tmpdir/pi-home";
my $err = "$tmpdir/stderr.log";
use_ok 'PublicInbox::Import';
use_ok 'PublicInbox::Inbox';
use_ok 'PublicInbox::Git';
-use_ok 'PublicInbox::V2Writable';
+SKIP: {
+ skip "git 2.6+ required for V2Writable", 1 if $version == 1;
+ use_ok 'PublicInbox::V2Writable';
+}
-# XXX FIXME: make it easier to test both versions
-my $version = int($ENV{PI_VERSION} || 1);
my %opts = (
LocalAddr => '127.0.0.1',
ReuseAddr => 1,
use strict;
use warnings;
use Test::More;
+require './t/common.perl';
+require_git(2.6);
use File::Temp qw/tempdir/;
use PublicInbox::MIME;
use PublicInbox::Config;
use PublicInbox::ContentId qw(content_digest);
use File::Temp qw/tempdir/;
use File::Path qw(remove_tree);
+require './t/common.perl';
+require_git(2.6);
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
eval "require $mod";
use Test::More;
use PublicInbox::MIME;
use File::Temp qw/tempdir/;
+require './t/common.perl';
+require_git(2.6);
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
eval "require $mod";
use File::Temp qw/tempdir/;
use Fcntl qw(SEEK_SET);
use Cwd;
+require './t/common.perl';
+require_git(2.6);
my $V = 2;
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
use warnings;
use Test::More;
require './t/common.perl';
+require_git(2.6);
# Integration tests for HTTP cloning + mirroring
foreach my $mod (qw(Plack::Util Plack::Builder Danga::Socket
use PublicInbox::ContentId qw(content_digest);
use File::Temp qw/tempdir/;
use File::Path qw(remove_tree);
+require './t/common.perl';
+require_git(2.6);
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
eval "require $mod";
use PublicInbox::ContentId qw(content_digest);
use File::Temp qw/tempdir/;
require './t/common.perl';
+require_git(2.6);
foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
eval "require $mod";
plan skip_all => "$mod missing for nntpd.t" if $@;
};
{
local $ENV{NPROC} = 2;
- my @before = $git0->qx(qw(log --pretty=oneline));
- my $before = $git0->qx(qw(log --pretty=raw --raw -r --no-abbrev));
+ my @log = qw(log --no-decorate --no-abbrev --no-notes --no-color);
+ my @before = $git0->qx(@log, qw(--pretty=oneline));
+ my $before = $git0->qx(@log, qw(--pretty=raw --raw -r));
$im = PublicInbox::V2Writable->new($ibx, 1);
is($im->{partitions}, 1, 'detected single partition from previous');
my $smsg = $im->remove($mime, 'test removal');
$im->done;
- my @after = $git0->qx(qw(log --pretty=oneline));
+ my @after = $git0->qx(@log, qw(--pretty=oneline));
my $tip = shift @after;
like($tip, qr/\A[a-f0-9]+ test removal\n\z/s,
'commit message propagated to git');
my $srch = $ibx->search->reopen;
my $mset = $srch->query('m:'.$smsg->mid, { mset => 1});
is($mset->size, 0, 'no longer found in Xapian');
- my @log1 = qw(log -1 --pretty=raw --raw -r --no-abbrev --no-renames);
+ my @log1 = (@log, qw(-1 --pretty=raw --raw -r --no-renames));
is($srch->{over_ro}->get_art($num), undef,
'removal propagated to Over DB');
use PublicInbox::MIME;
use Cwd;
use PublicInbox::Config;
+require './t/common.perl';
+require_git(2.6);
my @mods = qw(Filesys::Notify::Simple PublicInbox::V2Writable);
foreach my $mod (@mods) {
eval "require $mod";