v2: support Xapian + SQLite indexing

[public-inbox.git] / lib / PublicInbox / Import.pm

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 845fbb69852e57f465a47e73de844330215ff888..1a2698a75d609aea4162075e312a791f1c9041c0 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -31,6 +31,7 @@ sub new {
                inbox => $ibx,
                path_type => '2/38', # or 'v2'
                ssoma_lock => 1, # disable for v2
+               bytes_added => 0,
        }, $class
 }
 
@@ -233,7 +234,7 @@ sub parse_date ($) {
                warn "bogus TZ offset: $zone, ignoring and assuming +0000\n";
                $zone = '+0000';
        }
-       $ts ||= time;
+       $ts = time unless defined $ts;
        $ts = 0 if $ts < 0; # git uses unsigned times
        "$ts $zone";
 }
@@ -246,11 +247,6 @@ sub add {
        my $from = $mime->header('From');
        my ($email) = PublicInbox::Address::emails($from);
        my ($name) = PublicInbox::Address::names($from);
-       # git gets confused with:
-       #  "'A U Thor <u@example.com>' via foo" <foo@example.com>
-       # ref:
-       # <CAD0k6qSUYANxbjjbE4jTW4EeVwOYgBD=bXkSu=akiYC_CB7Ffw@mail.gmail.com>
-       $name =~ tr/<>//d;
 
        my $date_raw = parse_date($mime);
        my $subject = $mime->header('Subject');
@@ -280,13 +276,16 @@ sub add {
 
        my $blob = $self->{mark}++;
        my $str = $mime->as_string;
-       print $w "blob\nmark :$blob\ndata ", length($str), "\n" or wfail;
+       my $n = length($str);
+       $self->{bytes_added} += $n;
+       print $w "blob\nmark :$blob\ndata ", $n, "\n" or wfail;
        print $w $str, "\n" or wfail;
        $str = undef;
 
        # v2: we need this for Xapian
        if ($self->{want_object_id}) {
                chomp($self->{last_object_id} = $self->get_mark(":$blob"));
+               $self->{last_object_size} = $n;
        }
 
        my $ref = $self->{ref};
@@ -297,10 +296,26 @@ sub add {
                print $w "reset $ref\n" or wfail;
        }
 
-       utf8::encode($email);
-       utf8::encode($name);
+       # quiet down wide character warnings with utf8::encode
+       if (defined $email) {
+               utf8::encode($email);
+       } else {
+               $email = '';
+               warn "no email in From: $from\n";
+       }
+
+       # git gets confused with:
+       #  "'A U Thor <u@example.com>' via foo" <foo@example.com>
+       # ref:
+       # <CAD0k6qSUYANxbjjbE4jTW4EeVwOYgBD=bXkSu=akiYC_CB7Ffw@mail.gmail.com>
+       if (defined $name) {
+               $name =~ tr/<>//d;
+               utf8::encode($name);
+       } else {
+               $name = '';
+               warn "no name in From: $from\n";
+       }
        utf8::encode($subject);
-       # quiet down wide character warnings:
        print $w "commit $ref\nmark :$commit\n",
                "author $name <$email> $date_raw\n",
                "committer $self->{ident} ", now_raw(), "\n" or wfail;
@@ -314,7 +329,7 @@ sub add {
        $self->{tip} = ":$commit";
 }
 
-sub run_die ($$) {
+sub run_die ($;$) {
        my ($cmd, $env) = @_;
        my $pid = spawn($cmd, $env, undef);
        defined $pid or die "spawning ".join(' ', @$cmd)." failed: $!";
@@ -343,7 +358,7 @@ sub done {
        }
        if ($nchg) {
                run_die([@cmd, 'update-server-info'], undef);
-               eval {
+               ($self->{path_type} eq '2/38') and eval {
                        require PublicInbox::SearchIdx;
                        my $inbox = $self->{inbox} || $git_dir;
                        my $s = PublicInbox::SearchIdx->new($inbox);