v2: support Xapian + SQLite indexing

This is too slow, currently.  Working with only 2017 LKML
archives:

         git-only: ~1 minute
     git + SQLite: ~12 minutes
git+Xapian+SQlite: ~45 minutes

So yes, it looks like we'll need to parallelize Xapian indexing,
at least.

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 364ab602917eaa41cec96fb29f6d3ec7cac549e3..1a2698a75d609aea4162075e312a791f1c9041c0 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -285,6 +285,7 @@ sub add {
        # v2: we need this for Xapian
        if ($self->{want_object_id}) {
                chomp($self->{last_object_id} = $self->get_mark(":$blob"));
+               $self->{last_object_size} = $n;
        }
 
        my $ref = $self->{ref};

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 2ec2be69a6f288a134fc13f58cc5bcaee1cbb0b0..e7856e3cd4d20af2aca0eea03142abba727b736b 100644 (file)
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -79,7 +79,9 @@ sub new {
 sub git {
        my ($self) = @_;
        $self->{git} ||= eval {
-               my $g = PublicInbox::Git->new($self->{mainrepo});
+               my $git_dir = $self->{mainrepo};
+               $git_dir .= '/all.git' if (($self->{version} || 1) == 2);
+               my $g = PublicInbox::Git->new($git_dir);
                $g->{-httpbackend_limiter} = $self->{-httpbackend_limiter};
                _cleanup_later($self);
                $g;

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 33a1f2d3f36100e6871bb7a4d7599fa63deb3fa0..eac11bd44f905bd854af6e80cc4de104d26da93a 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -120,15 +120,29 @@ chomp @HELP;
 my $mail_query = Search::Xapian::Query->new('T' . 'mail');
 
 sub xdir {
-       my (undef, $git_dir) = @_;
-       "$git_dir/public-inbox/xapian" . SCHEMA_VERSION;
+       my ($self) = @_;
+       if ($self->{version} == 1) {
+               "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION;
+       } else {
+               "$self->{mainrepo}/xap" . SCHEMA_VERSION;
+       }
 }
 
 sub new {
-       my ($class, $git_dir, $altid) = @_;
-       my $dir = $class->xdir($git_dir);
-       my $db = Search::Xapian::Database->new($dir);
-       bless { xdb => $db, git_dir => $git_dir, altid => $altid }, $class;
+       my ($class, $mainrepo, $altid) = @_;
+       my $version = 1;
+       my $ibx = $mainrepo;
+       if (ref $ibx) {
+               $version = $ibx->{version} || 1;
+               $mainrepo = $ibx->{mainrepo};
+       }
+       my $self = bless {
+               mainrepo => $mainrepo,
+               altid => $altid,
+               version => $version,
+       }, $class;
+       $self->{xdb} = Search::Xapian::Database->new($self->xdir);
+       $self;
 }
 
 sub reopen { $_[0]->{xdb}->reopen }

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 265403a30664ee8a3000d8c64372f98656dd4564..c6c5bd259a014e02a4eed7a3837f68544e6925cb 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -51,26 +51,43 @@ sub git_unquote ($) {
 }
 
 sub new {
-       my ($class, $inbox, $creat) = @_;
-       my $git_dir = $inbox;
-       my $altid;
-       if (ref $inbox) {
-               $git_dir = $inbox->{mainrepo};
-               $altid = $inbox->{altid};
+       my ($class, $ibx, $creat) = @_;
+       my $mainrepo = $ibx; # for "public-inbox-index" w/o entry in config
+       my $git_dir = $mainrepo;
+       my ($altid, $git);
+       my $version = 1;
+       if (ref $ibx) {
+               $mainrepo = $ibx->{mainrepo};
+               $altid = $ibx->{altid};
+               $version = $ibx->{version} || 1;
                if ($altid) {
                        require PublicInbox::AltId;
                        $altid = [ map {
-                               PublicInbox::AltId->new($inbox, $_);
+                               PublicInbox::AltId->new($ibx, $_);
                        } @$altid ];
                }
+               $git = $ibx->git;
+       } else {
+               $git = PublicInbox::Git->new($git_dir); # v1 only
        }
        require Search::Xapian::WritableDatabase;
-       my $self = bless { git_dir => $git_dir, -altid => $altid }, $class;
+       my $self = bless {
+               mainrepo => $mainrepo,
+               git => $git,
+               -altid => $altid,
+               version => $version,
+       }, $class;
        my $perm = $self->_git_config_perm;
        my $umask = _umask_for($perm);
        $self->{umask} = $umask;
-       $self->{lock_path} = "$git_dir/ssoma.lock";
-       $self->{git} = PublicInbox::Git->new($git_dir);
+       if ($version == 1) {
+               $self->{lock_path} = "$mainrepo/ssoma.lock";
+       } elsif ($version == 2) {
+               $self->{lock_path} = "$mainrepo/inbox.lock";
+               $self->{msgmap_path} = "$mainrepo/msgmap.sqlite3";
+       } else {
+               die "unsupported inbox version=$version\n";
+       }
        $self->{creat} = ($creat || 0) == 1;
        $self;
 }
@@ -86,7 +103,7 @@ sub _xdb_release {
 sub _xdb_acquire {
        my ($self) = @_;
        croak 'already acquired' if $self->{xdb};
-       my $dir = PublicInbox::Search->xdir($self->{git_dir});
+       my $dir = $self->xdir;
        my $flag = Search::Xapian::DB_OPEN;
        if ($self->{creat}) {
                require File::Path;
@@ -541,6 +558,7 @@ sub batch_adjust ($$$$) {
        }
 }
 
+# only for v1
 sub rlog {
        my ($self, $log, $add_cb, $del_cb, $batch_cb) = @_;
        my $hex = '[a-f0-9]';
@@ -573,9 +591,14 @@ sub rlog {
 
 sub _msgmap_init {
        my ($self) = @_;
-       $self->{mm} = eval {
+       $self->{mm} ||= eval {
                require PublicInbox::Msgmap;
-               PublicInbox::Msgmap->new($self->{git_dir}, 1);
+               my $msgmap_path = $self->{msgmap_path};
+               if (defined $msgmap_path) { # v2
+                       PublicInbox::Msgmap->new_file($msgmap_path, 1);
+               } else {
+                       PublicInbox::Msgmap->new($self->{mainrepo}, 1);
+               }
        };
 }
 
@@ -712,8 +735,11 @@ sub merge_threads {
 
 sub _read_git_config_perm {
        my ($self) = @_;
-       my @cmd = qw(config core.sharedRepository);
-       my $fh = PublicInbox::Git->new($self->{git_dir})->popen(@cmd);
+       my @cmd = qw(config);
+       if ($self->{version} == 2) {
+               push @cmd, "--file=$self->{mainrepo}/inbox-config";
+       }
+       my $fh = $self->{git}->popen(@cmd, 'core.sharedRepository');
        local $/ = "\n";
        my $perm = <$fh>;
        chomp $perm if defined $perm;

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 9b68e9b1032b53b084b3118e9d5223cb0bceded4..41bfb8d19f2cd505ad0096415c85fb58c1b7e541 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -34,7 +34,7 @@ sub new {
                xap_ro => undef,
 
                # limit each repo to 1GB or so
-               rotate_bytes => int((100 * 1024 * 1024) / $PACKING_FACTOR),
+               rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
        };
        bless $self, $class
 }
@@ -55,11 +55,29 @@ sub add {
        my $cmt = $im->add($mime, $check_cb) or return;
        $cmt = $im->get_mark($cmt);
        my $oid = $im->{last_object_id};
-       $self->index_msg($mime, $existing, $cmt, $oid);
+       my $size = $im->{last_object_size};
+
+       my $idx = $self->search_idx;
+       $idx->index_both($mime, $size, $oid);
+       $idx->{xdb}->set_metadata('last_commit', $cmt);
+       my $n = $self->{transact_bytes} += $size;
+       if ($n > PublicInbox::SearchIdx::BATCH_BYTES) {
+               $self->checkpoint;
+       }
+
        $mime;
 }
 
-sub index_msg {  # TODO
+sub search_idx {
+       my ($self) = @_;
+       $self->{idx} ||= eval {
+               my $idx = PublicInbox::SearchIdx->new($self->{-inbox}, 1);
+               my $mm = $idx->_msgmap_init;
+               $idx->_xdb_acquire->begin_transaction;
+               $self->{transact_bytes} = 0;
+               $mm->{dbh}->begin_work;
+               $idx
+       };
 }
 
 sub remove {
@@ -79,12 +97,25 @@ sub remove {
 
 sub done {
        my ($self) = @_;
-       $self->{im}->done; # PublicInbox::Import::done
+       my $im = $self->{im};
+       $im->done if $im; # PublicInbox::Import::done
+       $self->searchidx_checkpoint;
 }
 
 sub checkpoint {
        my ($self) = @_;
-       $self->{im}->checkpoint; # PublicInbox::Import::checkpoint
+       my $im = $self->{im};
+       $im->checkpoint if $im; # PublicInbox::Import::checkpoint
+       $self->searchidx_checkpoint;
+}
+
+sub searchidx_checkpoint {
+       my ($self) = @_;
+       my $idx = delete $self->{idx} or return;
+
+       $idx->{mm}->{dbh}->commit;
+       $idx->{xdb}->commit_transaction;
+       $idx->_xdb_release;
 }
 
 sub git_init {
@@ -127,6 +158,7 @@ sub importer {
                } else {
                        $self->{im} = undef;
                        $im->done;
+                       $self->searchidx_checkpoint;
                        $im = undef;
                        my $git_dir = $self->git_init(++$self->{max_git});
                        my $git = PublicInbox::Git->new($git_dir);
@@ -156,8 +188,6 @@ sub importer {
                        $self->{max_git} = $max;
                        return $self->import_init($git, $packed_bytes);
                }
-       } else {
-               warn "latest not found in $pfx\n";
        }
        $self->{max_git} = $new;
        $latest = $self->git_init($new);
@@ -168,6 +198,7 @@ sub import_init {
        my ($self, $git, $packed_bytes) = @_;
        my $im = PublicInbox::Import->new($git, undef, undef, $self->{-inbox});
        $im->{bytes_added} = int($packed_bytes / $PACKING_FACTOR);
+       $im->{want_object_id} = 1;
        $im->{ssoma_lock} = 0;
        $im->{path_type} = 'v2';
        $self->{im} = $im;

diff --git a/scripts/import_vger_from_mbox b/scripts/import_vger_from_mbox
index c45dc4ee1a4986f38d15d96ba9cfe2d706d00019..d30e8a3096e10848823e71773bc1bef3be3018aa 100644 (file)
--- a/scripts/import_vger_from_mbox
+++ b/scripts/import_vger_from_mbox
@@ -7,6 +7,7 @@ use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
 use Date::Parse qw/str2time/;
 use Email::MIME;
 $Email::MIME::ContentType::STRICT_PARAMS = 0; # user input is imperfect
+use PublicInbox::Inbox;
 use PublicInbox::V2Writable;
 my $usage = "usage: $0 NAME EMAIL DIR <MBOX\n";
 my $dry_run;
@@ -18,8 +19,10 @@ my $mainrepo = shift or die $usage; # /path/to/v2/repo
 my $v2ibx = {
        mainrepo => $mainrepo,
        name => $name,
+       version => 2,
        -primary_address => $email,
 };
+$v2ibx = PublicInbox::Inbox->new($v2ibx);
 my $im = $dry_run ? undef : PublicInbox::V2Writable->new($v2ibx, 1);
 binmode STDIN;
 my $msg = '';