From ac2cbae7af54527299dbfe927716ebc3c7ec89de Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 9 Jun 2021 07:47:49 +0000 Subject: [PATCH] lei tag: parallelize Maildir access Since Maildir isn't guaranteed to have any sort of order, we can parallelize inputs, here. On a 4-core system, this reduced one of my tag invocations from 5.5 to 1.4s. --- lib/PublicInbox/LeiImport.pm | 8 ++++---- lib/PublicInbox/LeiPmdir.pm | 10 ++-------- lib/PublicInbox/LeiTag.pm | 8 +++++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index cddd5619..e3cb69ca 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -35,13 +35,13 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn die "BUG: $f was not from a Maildir?\n"; my $kw = PublicInbox::MdirReader::flags2kw($fl); substr($folder, 0, 0) = 'maildir:'; # add prefix - my $lms = $self->{-lms_ro}; + my $lse = $self->{lse} //= $self->{lei}->{sto}->search; + my $lms = $self->{-lms_ro} //= $lse->lms; # may be 0 or undef my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef; - my @docids = defined($oidbin) ? - $self->{over}->oidbin_exists($oidbin) : (); + my @docids = defined($oidbin) ? $lse->over->oidbin_exists($oidbin) : (); my $vmd = $self->{-import_kw} ? { kw => $kw } : undef; if (scalar @docids) { - $self->{lse}->kw_changed(undef, $kw, \@docids) or return; + $lse->kw_changed(undef, $kw, \@docids) or return; } if (my $eml = eml_from_path($f)) { $vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync}; diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm index aa9ce713..760f276c 100644 --- a/lib/PublicInbox/LeiPmdir.pm +++ b/lib/PublicInbox/LeiPmdir.pm @@ -30,15 +30,9 @@ sub new { sub ipc_atfork_child { my ($self) = @_; - my $lei = $self->{lei}; - $lei->_lei_atfork_child; my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}'; - $ipt->{lei} = $lei; - $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}'; - $ipt->{lse} = $ipt->{sto}->search; - $ipt->{over} = $ipt->{lse}->over; - $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0' - $self->SUPER::ipc_atfork_child; + $ipt->{lei} = $self->{lei}; + $ipt->ipc_atfork_child; } sub each_mdir_fn { # maildir_each_file callback diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index 4b3ce7d8..e0532653 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -6,6 +6,7 @@ package PublicInbox::LeiTag; use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); +use PublicInbox::InboxWritable qw(eml_from_path); sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh my ($self, $eml) = @_; @@ -24,8 +25,9 @@ sub input_mbox_cb { input_eml_cb($self, $eml); } -sub input_maildir_cb { # maildir_each_eml cb - my ($f, $kw, $eml, $self) = @_; +sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn + my ($self, $f) = @_; + my $eml = eml_from_path($f) or return; input_eml_cb($self, $eml); } @@ -42,12 +44,12 @@ sub lei_tag { # the "lei tag" method $lei->ale; # refresh and prepare my $vmd_mod = $self->vmd_mod_extract(\@argv); return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err}; + $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs $self->prepare_inputs($lei, \@argv) or return; grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or return $lei->fail('no keywords or labels specified'); my $ops = {}; $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; - $self->{vmd_mod} = $vmd_mod; my $j = $self->{-wq_nr_workers} = 1; # locked for now (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); $lei->{wq1} = $self; -- 2.44.0