InboxWritable should only set $v2w->{parallel} if the $parallel
flag is defined to 0 or 1. We want indexing a new inbox to
utilize SMP, just like --reindex.
-index once again allows -j0/--jobs=0 to force single-process
use, and we'll be ensuring that works in tests to maintain
performance on small systems.
Fixes: 61a2fff5b34a3e32 ("admin: move index_inbox over")
12 files changed:
die "v2 not supported: $@\n" if $@;
my $opt = $self->{-creat_opt};
my $v2w = PublicInbox::V2Writable->new($self, $opt);
die "v2 not supported: $@\n" if $@;
my $opt = $self->{-creat_opt};
my $v2w = PublicInbox::V2Writable->new($self, $opt);
- $v2w->{parallel} = $parallel;
+ $v2w->{parallel} = $parallel if defined $parallel;
$v2w;
} elsif ($v == 1) {
my @arg = (undef, undef, undef, $self);
$v2w;
} elsif ($v == 1) {
my @arg = (undef, undef, undef, $self);
GetOptions($opt, qw(verbose|v+ reindex compact|c+ jobs|j=i prune
indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s))
or die "bad command-line args\n$usage";
GetOptions($opt, qw(verbose|v+ reindex compact|c+ jobs|j=i prune
indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s))
or die "bad command-line args\n$usage";
-die "--jobs must be positive\n" if defined $opt->{jobs} && $opt->{jobs} <= 0;
+die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;
if ($opt->{compact}) {
require PublicInbox::Xapcmd;
if ($opt->{compact}) {
require PublicInbox::Xapcmd;
use_ok 'PublicInbox::Config';
my $cfg = PublicInbox::Config->new($pi_config);
my $ibx = $cfg->lookup_name('test');
use_ok 'PublicInbox::Config';
my $cfg = PublicInbox::Config->new($pi_config);
my $ibx = $cfg->lookup_name('test');
-my $im = PublicInbox::InboxWritable->new($ibx)->importer;
+my $im = PublicInbox::InboxWritable->new($ibx)->importer(0);
{
local $ENV{HOME} = $home;
{
local $ENV{HOME} = $home;
-$cmd = [ qw(-index --reindex -c), "$tmpdir/v2" ];
+$cmd = [ qw(-index -j0 --reindex -c), "$tmpdir/v2" ];
ok(run_script($cmd, undef, $rdr), '--reindex -c');
like($err, qr/xapian-compact/, 'xapian-compact ran (-c)');
$rdr->{2} = \(my $err2 = '');
ok(run_script($cmd, undef, $rdr), '--reindex -c');
like($err, qr/xapian-compact/, 'xapian-compact ran (-c)');
$rdr->{2} = \(my $err2 = '');
-$cmd = [ qw(-index --reindex -cc), "$tmpdir/v2" ];
+$cmd = [ qw(-index -j0 --reindex -cc), "$tmpdir/v2" ];
ok(run_script($cmd, undef, $rdr), '--reindex -c -c');
like($err2, qr/xapian-compact/, 'xapian-compact ran (-c -c)');
ok(($err2 =~ tr/\n/\n/) > ($err =~ tr/\n/\n/), '-compacted twice');
ok(run_script($cmd, undef, $rdr), '--reindex -c -c');
like($err2, qr/xapian-compact/, 'xapian-compact ran (-c -c)');
ok(($err2 =~ tr/\n/\n/) > ($err =~ tr/\n/\n/), '-compacted twice');
-primary_address => 'test@example.com',
indexlevel => $level,
});
-primary_address => 'test@example.com',
indexlevel => $level,
});
- my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer;
+ my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer(0);
$mime->header_set('Message-ID', '<m@1>');
ok($im->add($mime), 'first message added');
$im->done;
# index master (required for v1)
$mime->header_set('Message-ID', '<m@1>');
ok($im->add($mime), 'first message added');
$im->done;
# index master (required for v1)
- ok(run_script(['-index', $ibx->{inboxdir}, "-L$level"]),
+ ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
'index master OK');
my $ro_master = PublicInbox::Inbox->new({
inboxdir => $ibx->{inboxdir},
'index master OK');
my $ro_master = PublicInbox::Inbox->new({
inboxdir => $ibx->{inboxdir},
ok(run_script(\@cmd), "v$v init OK");
# index mirror
ok(run_script(\@cmd), "v$v init OK");
# index mirror
- ok(run_script(['-index', $mirror]), "v$v index mirror OK");
+ ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror OK");
# read-only access
my $ro_mirror = PublicInbox::Inbox->new({
# read-only access
my $ro_mirror = PublicInbox::Inbox->new({
# mirror updates
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
# mirror updates
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
- ok(run_script(['-index', $mirror]), "v$v index mirror again OK");
+ ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
($nr, $msgs) = $ro_mirror->recent;
is($nr, 2, '2nd message seen in mirror');
is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
['m@1','m@2'], 'got both messages in mirror');
# incremental index master (required for v1)
($nr, $msgs) = $ro_mirror->recent;
is($nr, 2, '2nd message seen in mirror');
is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
['m@1','m@2'], 'got both messages in mirror');
# incremental index master (required for v1)
- ok(run_script(['-index', $ibx->{inboxdir}, "-L$level"]),
+ ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
'index master OK');
($nr, $msgs) = $ro_master->recent;
is($nr, 2, '2nd message seen in master');
'index master OK');
($nr, $msgs) = $ro_master->recent;
is($nr, 2, '2nd message seen in master');
# sync the mirror
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
# sync the mirror
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
- ok(run_script(['-index', $mirror]), "v$v index mirror again OK");
+ ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
($nr, $msgs) = $ro_mirror->recent;
is($nr, 1, '2nd message gone from mirror');
is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
($nr, $msgs) = $ro_mirror->recent;
is($nr, 1, '2nd message gone from mirror');
is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
}
$im->done;
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
}
$im->done;
is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
- ok(run_script(['-index', '--reindex', $mirror]),
+ ok(run_script([qw(-index -j0 --reindex), $mirror]),
"v$v index --reindex mirror OK");
@ro_nums = map { $_->{num} } @{$ro_mirror->over->query_ts(0, 0)};
@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
"v$v index --reindex mirror OK");
@ro_nums = map { $_->{num} } @{$ro_mirror->over->query_ts(0, 0)};
@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
my $cmd = [ '-init', "-$v", $v, $inboxdir,
"http://example.com/$v", $addr ];
ok(run_script($cmd), 'public-inbox-init');
my $cmd = [ '-init', "-$v", $v, $inboxdir,
"http://example.com/$v", $addr ];
ok(run_script($cmd), 'public-inbox-init');
- ok(run_script(['-index', $inboxdir]), 'public-inbox-index');
+ ok(run_script([qw(-index -j0), $inboxdir]), 'public-inbox-index');
is(xsys(@cfg, "$cfgpfx.filter", 'PublicInbox::Filter::RubyLang'), 0);
is(xsys(@cfg, "$cfgpfx.altid",
'serial:alerts:file=msgmap.sqlite3'), 0);
is(xsys(@cfg, "$cfgpfx.filter", 'PublicInbox::Filter::RubyLang'), 0);
is(xsys(@cfg, "$cfgpfx.altid",
'serial:alerts:file=msgmap.sqlite3'), 0);
$cmd = [ '-init', '-Lbasic', '-V2', 'v2c', "$tmpdir/v2-clone",
'http://example.com/v2c', 'v2c@example.com' ];
ok(run_script($cmd, $env), 'init clone');
$cmd = [ '-init', '-Lbasic', '-V2', 'v2c', "$tmpdir/v2-clone",
'http://example.com/v2c', 'v2c@example.com' ];
ok(run_script($cmd, $env), 'init clone');
- $cmd = [ '-index', "$tmpdir/v2-clone" ];
+ $cmd = [ qw(-index -j0), "$tmpdir/v2-clone" ];
sleep($delay) if $delay;
ok(run_script($cmd, $env), 'index the clone');
$ibx->cleanup;
sleep($delay) if $delay;
ok(run_script($cmd, $env), 'index the clone');
$ibx->cleanup;
my $len;
$ibx = PublicInbox::InboxWritable->new($ibx);
my $len;
$ibx = PublicInbox::InboxWritable->new($ibx);
- my $im = $ibx->importer;
+ my $im = $ibx->importer(0);
# ensure successful message delivery
{
# ensure successful message delivery
{
Top secret info about my house in Malibu...
EOF
Top secret info about my house in Malibu...
EOF
- my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer;
+ my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer(0);
# fake a bunch of epochs
$im->{rotate_bytes} = $opt->{rotate_bytes} if $opt->{rotate_bytes};
# fake a bunch of epochs
$im->{rotate_bytes} = $opt->{rotate_bytes} if $opt->{rotate_bytes};
is($smsg->{subject}, 'redacted', 'after subject');
is($smsg->{mid}, 'replace@example.com', 'before MID');
}
is($smsg->{subject}, 'redacted', 'after subject');
is($smsg->{mid}, 'replace@example.com', 'before MID');
}
+ # $git->cleanup; # needed if $im->{parallel};
@warn = ();
is($im->replace($orig, $repl), undef, 'no-op replace returns undef');
is($im->purge($orig), undef, 'no-op purge returns undef');
is_deeply(\@warn, [], 'no warnings on noop');
@warn = ();
is($im->replace($orig, $repl), undef, 'no-op replace returns undef');
is($im->purge($orig), undef, 'no-op purge returns undef');
is_deeply(\@warn, [], 'no warnings on noop');
+ # $im->done; # needed if $im->{parallel}
'alt@example.com');
ok(run_script(\@cmd), 'initialized public-inbox -V2');
'alt@example.com');
ok(run_script(\@cmd), 'initialized public-inbox -V2');
-ok(run_script(['-index', "$tmpdir/m"]), 'indexed');
+ok(run_script([qw(-index -j0), "$tmpdir/m"]), 'indexed');
my $mibx = { inboxdir => "$tmpdir/m", address => 'alt@example.com' };
$mibx = PublicInbox::Inbox->new($mibx);
my $mibx = { inboxdir => "$tmpdir/m", address => 'alt@example.com' };
$mibx = PublicInbox::Inbox->new($mibx);
my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
is(scalar($mset->items), 0, 'new message not found in mirror, yet');
my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
is(scalar($mset->items), 0, 'new message not found in mirror, yet');
-ok(run_script(["-index", "$tmpdir/m"]), 'index updated');
+ok(run_script([qw(-index -j0), "$tmpdir/m"]), 'index updated');
is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax');
$mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
is(scalar($mset->items), 1, 'found message in mirror');
is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax');
$mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
is(scalar($mset->items), 1, 'found message in mirror');
$ibx->cleanup;
PublicInbox::InboxWritable::cleanup($mibx);
$v2w->done;
$ibx->cleanup;
PublicInbox::InboxWritable::cleanup($mibx);
$v2w->done;
- my $cmd = [ '-index', '--prune', "$tmpdir/m" ];
+ my $cmd = [ qw(-index --prune -j0), "$tmpdir/m" ];
my ($out, $err) = ('', '');
my $opt = { 1 => \$out, 2 => \$err };
ok(run_script($cmd, undef, $opt), '-index --prune');
my ($out, $err) = ('', '');
my $opt = { 1 => \$out, 2 => \$err };
ok(run_script($cmd, undef, $opt), '-index --prune');
$fetch_each_epoch->();
PublicInbox::InboxWritable::cleanup($mibx);
$fetch_each_epoch->();
PublicInbox::InboxWritable::cleanup($mibx);
- my $cmd = [ "-index", "$tmpdir/m" ];
+ my $cmd = [ qw(-index -j0), "$tmpdir/m" ];
my ($out, $err) = ('', '');
my $opt = { 1 => \$out, 2 => \$err };
ok(run_script($cmd, undef, $opt), 'index ran');
my ($out, $err) = ('', '');
my $opt = { 1 => \$out, 2 => \$err };
ok(run_script($cmd, undef, $opt), 'index ran');
$ibx->cleanup;
$fetch_each_epoch->();
PublicInbox::InboxWritable::cleanup($mibx);
$ibx->cleanup;
$fetch_each_epoch->();
PublicInbox::InboxWritable::cleanup($mibx);
- my $cmd = ['-index', "$tmpdir/m", "--max-size=$max" ];
+ my $cmd = [qw(-index -j0), "$tmpdir/m", "--max-size=$max" ];
my $opt = { 2 => \(my $err) };
ok(run_script($cmd, undef, $opt), 'indexed with --max-size');
like($err, qr/skipping [a-f0-9]{40,}/, 'warned about skipping message');
my $opt = { 2 => \(my $err) };
ok(run_script($cmd, undef, $opt), 'indexed with --max-size');
like($err, qr/skipping [a-f0-9]{40,}/, 'warned about skipping message');
- $cmd = ['-index', "$tmpdir/m", "--reindex" ];
+ $cmd = [ qw(-index -j0 --reindex), "$tmpdir/m" ];
ok(run_script($cmd, undef, $opt), 'reindexed w/ indexMaxSize in file');
like($err, qr/skipping [a-f0-9]{40,}/, 'warned about skipping message');
$mset = $mibx->search->reopen->query('m:2big@a', {mset =>1});
ok(run_script($cmd, undef, $opt), 'reindexed w/ indexMaxSize in file');
like($err, qr/skipping [a-f0-9]{40,}/, 'warned about skipping message');
$mset = $mibx->search->reopen->query('m:2big@a', {mset =>1});
my %config = %$ibx_config;
my $ibx = PublicInbox::Inbox->new(\%config);
my $im = PublicInbox::V2Writable->new($ibx, {nproc => 1});
my %config = %$ibx_config;
my $ibx = PublicInbox::Inbox->new(\%config);
my $im = PublicInbox::V2Writable->new($ibx, {nproc => 1});
- my $im0 = $im->importer();
+ my $im0 = $im->importer(0);
foreach my $i (1..10) {
$mime->header_set('Message-Id', "<$i\@example.com>");
ok($im->add($mime), "message $i added");
foreach my $i (1..10) {
$mime->header_set('Message-Id', "<$i\@example.com>");
ok($im->add($mime), "message $i added");
my @xcpdb = qw(-xcpdb -q);
my $nproc = 8;
my $ndoc = 13;
my @xcpdb = qw(-xcpdb -q);
my $nproc = 8;
my $ndoc = 13;
-my $im = PublicInbox::InboxWritable->new($ibx, {nproc => $nproc})->importer(1);
+my $im = PublicInbox::InboxWritable->new($ibx, {nproc => $nproc})->importer;
for my $i (1..$ndoc) {
$mime->header_set('Message-ID', "<m$i\@example.com>");
ok($im->add($mime), "message $i added");
for my $i (1..$ndoc) {
$mime->header_set('Message-ID', "<m$i\@example.com>");
ok($im->add($mime), "message $i added");