-# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# git fast-import-based ssoma-mda MDA replacement
-# This is only ever run by public-inbox-mda and public-inbox-learn,
-# not the WWW or NNTP code which only requires read-only access.
+# This is only ever run by public-inbox-mda, public-inbox-learn
+# and public-inbox-watch. Not the WWW or NNTP code which only
+# requires read-only access.
package PublicInbox::Import;
use strict;
use warnings;
use POSIX qw(strftime);
sub new {
+ # we can't change arg order, this is documented in POD
+ # and external projects may rely on it:
my ($class, $git, $name, $email, $ibx) = @_;
my $ref = 'refs/heads/master';
if ($ibx) {
$ref = $ibx->{ref_head} || 'refs/heads/master';
$name ||= $ibx->{name};
$email ||= $ibx->{-primary_address};
+ $git ||= $ibx->git;
}
bless {
git => $git,
ident => "$name <$email>",
mark => 1,
ref => $ref,
- inbox => $ibx,
+ -inbox => $ibx,
path_type => '2/38', # or 'v2'
lock_path => "$git->{git_dir}/ssoma.lock", # v2 changes this
bytes_added => 0,
run_die([@cmd, 'update-server-info'], undef);
($self->{path_type} eq '2/38') and eval {
require PublicInbox::SearchIdx;
- my $inbox = $self->{inbox} || $git_dir;
- my $s = PublicInbox::SearchIdx->new($inbox);
+ my $ibx = $self->{-inbox} || $git_dir;
+ my $s = PublicInbox::SearchIdx->new($ibx);
$s->index_sync({ ref => $self->{ref} });
};
eval { run_die([@cmd, qw(gc --auto)], undef) } if $do_gc;
my $sender = '';
my $from = $mime->header('From');
+ $from ||= '';
my ($email) = PublicInbox::Address::emails($from);
my ($name) = PublicInbox::Address::names($from);
if (!defined($name) || !defined($email)) {
$sender = $mime->header('Sender');
+ $sender ||= '';
if (!defined($name)) {
($name) = PublicInbox::Address::names($sender);
}
my @ct = msg_timestamp($hdr);
my $author_time_raw = git_timestamp(@at);
my $commit_time_raw = git_timestamp(@ct);
+
my $subject = $mime->header('Subject');
$subject = '(no subject)' unless defined $subject;
- my $path_type = $self->{path_type};
+ # Mime decoding can create nulls replace them with spaces to protect git
+ $subject =~ tr/\0/ /;
+ utf8::encode($subject);
+ my $path_type = $self->{path_type};
my $path;
if ($path_type eq '2/38') {
$path = mid2path(v1_mid0($mime));
print $w "reset $ref\n" or wfail;
}
- utf8::encode($subject);
print $w "commit $ref\nmark :$commit\n",
"author $name <$email> $author_time_raw\n",
"committer $self->{ident} $commit_time_raw\n" or wfail;
$? == 0 or die join(' ', @$cmd) . " failed: $?\n";
}
+sub init_bare {
+ my ($dir) = @_;
+ my @cmd = (qw(git init --bare -q), $dir);
+ run_die(\@cmd);
+ # set a reasonable default:
+ @cmd = (qw/git config/, "--file=$dir/config",
+ 'repack.writeBitmaps', 'true');
+ run_die(\@cmd);
+}
+
sub done {
my ($self) = @_;
my $w = delete $self->{out} or return;
sub atfork_child {
my ($self) = @_;
foreach my $f (qw(in out)) {
+ next unless defined($self->{$f});
close $self->{$f} or die "failed to close import[$f]: $!\n";
}
}
my $old = $self->{'ref'};
my $git = $self->{git};
my @export = (qw(fast-export --no-data --use-done-feature), $old);
- my ($rd, $pid) = $git->popen(@export);
+ my $rd = $git->popen(@export);
my ($r, $w) = $self->gfi_start;
my @buf;
my $npurge = 0;
push @buf, $buf;
} elsif (/^M 100644 ([a-f0-9]+) (\w+)/) {
my ($oid, $path) = ($1, $2);
+ $tree->{$path} = 1;
if ($purge->{$oid}) {
push @oids, $oid;
- delete $tree->{$path};
+ my $cmd = "M 100644 inline $path\ndata 0\n\n";
+ push @buf, $cmd;
} else {
- $tree->{$path} = 1;
push @buf, $_;
}
} elsif (/^D (\w+)/) {
push @buf, $_;
}
}
+ close $rd or die "close fast-export failed: $?";
if (@buf) {
$w->print(@buf) or wfail;
}
=head1 NAME
-PublicInbox::Import - message importer for public-inbox
+PublicInbox::Import - message importer for public-inbox v1 inboxes
=head1 VERSION
=head1 DESCRIPTION
An importer and remover for public-inboxes which takes L<Email::MIME>
-messages as input and stores them in a ssoma repository as
-documented in L<https://ssoma.public-inbox.org/ssoma_repository.txt>,
+messages as input and stores them in a git repository as
+documented in L<https://public-inbox.org/public-inbox-v1-format.txt>,
except it does not allow duplicate Message-IDs.
It requires L<git(1)> and L<git-fast-import(1)> to be installed.