search: v2: ensure shards are numerically sorted

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 55eee41ca4a88c43594e9e0899c3dba25e77771e..4d02a7c169c202ebd150ccd1e0158ab21939c56a 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -164,15 +164,10 @@ chomp @HELP;
 
 sub xdir ($;$) {
        my ($self, $rdonly) = @_;
-       if ($self->{ibx_ver} == 1) {
-               "$self->{inboxdir}/public-inbox/xapian" . SCHEMA_VERSION;
-       } else {
-               my $dir = "$self->{inboxdir}/xap" . SCHEMA_VERSION;
-               return $dir if $rdonly;
-
-               my $shard = $self->{shard};
-               defined $shard or die "shard not given";
-               $dir .= "/$shard";
+       if ($rdonly || !defined($self->{shard})) {
+               $self->{xpfx};
+       } else { # v2 only:
+               "$self->{xpfx}/$self->{shard}";
        }
 }
 
@@ -182,19 +177,24 @@ sub _xdb ($) {
        my ($xdb, $slow_phrase);
        my $qpf = \($self->{qp_flags} ||= $QP_FLAGS);
        if ($self->{ibx_ver} >= 2) {
-               my $n = 0;
-               foreach my $shard (<$dir/*>) {
-                       -d $shard && $shard =~ m!/[0-9]+\z! or next;
-                       my $sub = $X{Database}->new($shard);
-                       if ($xdb) {
-                               $xdb->add_database($sub);
-                       } else {
-                               $xdb = $sub;
+               my @xdb;
+               opendir(my $dh, $dir) or return; # not initialized yet
+
+               # We need numeric sorting so shard[0] is first for reading
+               # Xapian metadata, if needed
+               for (sort { $a <=> $b } grep(/\A[0-9]+\z/, readdir($dh))) {
+                       my $shard_dir = "$dir/$_";
+                       if (-d $shard_dir && -r _) {
+                               push @xdb, $X{Database}->new($shard_dir);
+                               $slow_phrase ||= -f "$shard_dir/iamchert";
+                       } else { # gaps from missing epochs throw off mdocid()
+                               warn "E: $shard_dir missing or unreadable\n";
+                               return;
                        }
-                       $slow_phrase ||= -f "$shard/iamchert";
-                       ++$n;
                }
-               $self->{nshard} = $n;
+               $self->{nshard} = scalar(@xdb);
+               $xdb = shift @xdb;
+               $xdb->add_database($_) for @xdb;
        } else {
                $slow_phrase = -f "$dir/iamchert";
                $xdb = $X{Database}->new($dir);
@@ -220,14 +220,24 @@ sub xdb ($) {
        };
 }
 
+sub xpfx_init ($) {
+       my ($self) = @_;
+       if ($self->{ibx_ver} == 1) {
+               $self->{xpfx} .= '/public-inbox/xapian' . SCHEMA_VERSION;
+       } else {
+               $self->{xpfx} .= '/xap'.SCHEMA_VERSION;
+       }
+}
+
 sub new {
        my ($class, $ibx) = @_;
        ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
        my $self = bless {
-               inboxdir => $ibx->{inboxdir},
+               xpfx => $ibx->{inboxdir}, # for xpfx_init
                altid => $ibx->{altid},
                ibx_ver => $ibx->version,
        }, $class;
+       xpfx_init($self);
        my $dir = xdir($self, 1);
        $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3");
        $self;