X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiOverview.pm;h=dcfb9cc7d54a7a7f974a3c2c4c5680ba1218541f;hb=757652fd1ad6843c984610263a2a0b336c974111;hp=a7021b03d9246da2c4b584a0353ec73f58f17b40;hpb=934b7431d93e6e49920e407c02653edad06987d6;p=public-inbox.git diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index a7021b03..dcfb9cc7 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -23,19 +23,31 @@ my $JSONL = 'ldjson|ndjson|jsonl'; # 3 names for the same thing sub _iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) } -# we open this in the parent process before ->wq_do handoff +# we open this in the parent process before ->wq_io_do handoff sub ovv_out_lk_init ($) { my ($self) = @_; - $self->{tmp_lk_id} = "$self.$$"; my $tmp = File::Temp->new("lei-ovv.dst.$$.lock-XXXXXX", TMPDIR => 1, UNLINK => 0); - $self->{lock_path} = $tmp->filename; + $self->{"lk_id.$self.$$"} = $self->{lock_path} = $tmp->filename; } sub ovv_out_lk_cancel ($) { my ($self) = @_; - ($self->{tmp_lk_id}//'') eq "$self.$$" and - unlink(delete($self->{lock_path})); + my $lock_path = delete $self->{"lk_id.$self.$$"} or return; + unlink($lock_path); +} + +sub detect_fmt ($$) { + my ($lei, $dst) = @_; + if ($dst =~ m!\A([:/]+://)!) { + $lei->fail("$1 support not implemented, yet\n"); + } elsif (!-e $dst || -d _) { + 'maildir'; # the default TODO: MH? + } elsif (-f _ || -p _) { + $lei->fail("unable to determine mbox family of $dst\n"); + } else { + $lei->fail("unable to determine format of $dst\n"); + } } sub new { @@ -54,7 +66,7 @@ sub new { } $fmt //= 'json' if $dst eq '/dev/stdout'; - $fmt //= 'maildir'; + $fmt //= detect_fmt($lei, $dst) or return; if (index($dst, '://') < 0) { # not a URL, so assume path $dst = File::Spec->canonpath($dst); @@ -66,10 +78,8 @@ sub new { if ($fmt =~ /\A($JSONL|(?:concat)?json)\z/) { $json = $self->{json} = ref(PublicInbox::Config->json); } - my ($isatty, $seekable); if ($dst eq '/dev/stdout') { - $isatty = -t $lei->{1}; - $lei->start_pager if $isatty; + my $isatty = $lei->{need_pager} = -t $lei->{1}; $opt->{pretty} //= $isatty; if (!$isatty && -f _) { my $fl = fcntl($lei->{1}, F_GETFL, 0) // @@ -78,14 +88,17 @@ sub new { } else { ovv_out_lk_init($self); } + } elsif (!$opt->{quiet}) { + $lei->{-progress} = 1; } - if (!$json) { + if ($json) { + $lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei); + } else { # default to the cheapest sort since MUA usually resorts $lei->{opt}->{'sort'} //= 'docid' if $dst ne '/dev/stdout'; $lei->{l2m} = eval { PublicInbox::LeiToMail->new($lei) }; return $lei->fail($@) if $@; } - $lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei); $self; } @@ -93,28 +106,22 @@ sub new { sub ovv_begin { my ($self, $lei) = @_; if ($self->{fmt} eq 'json') { - print { $lei->{1} } '['; + $lei->out('['); } # TODO HTML/Atom/... } # called once by parent (via PublicInbox::EOFpipe) sub ovv_end { my ($self, $lei) = @_; - my $out = $lei->{1} or return; if ($self->{fmt} eq 'json') { # JSON doesn't allow trailing commas, and preventing # trailing commas is a PITA when parallelizing outputs - print $out "null]\n"; + $lei->out("null]\n"); } elsif ($self->{fmt} eq 'concatjson') { - print $out "\n"; + $lei->out("\n"); } } -sub ovv_atfork_child { - my ($self) = @_; - # reopen dedupe here -} - # prepares an smsg for JSON sub _unbless_smsg { my ($smsg, $mitem) = @_; @@ -122,11 +129,9 @@ sub _unbless_smsg { delete @$smsg{qw(lines bytes num tid)}; $smsg->{rt} = _iso8601(delete $smsg->{ts}); # JMAP receivedAt $smsg->{dt} = _iso8601(delete $smsg->{ds}); # JMAP UTCDate - $smsg->{relevance} = get_pct($mitem) if $mitem; - + $smsg->{pct} = get_pct($mitem) if $mitem; if (my $r = delete $smsg->{references}) { - $smsg->{refs} = [ - map { "<$_>" } ($r =~ m/$MID_EXTRACT/go) ]; + $smsg->{refs} = [ map { "<$_>" } ($r =~ m/$MID_EXTRACT/go) ]; } if (my $m = delete($smsg->{mid})) { $smsg->{'m'} = "<$m>"; @@ -142,23 +147,9 @@ sub _unbless_smsg { sub ovv_atexit_child { my ($self, $lei) = @_; - if (my $l2m = delete $lei->{l2m}) { - # gracefully stop lei2mail processes after all - # ->write_mail work is complete - delete $l2m->{-wq_s1}; - if (my $rd = delete $l2m->{each_smsg_done}) { - read($rd, my $buf, 1); # wait for EOF - } - } - # order matters, git->{-tmp}->DESTROY must not fire until - # {each_smsg_done} hits EOF above - if (my $git = delete $self->{git}) { - $git->async_wait_all; - } if (my $bref = delete $lei->{ovv_buf}) { - my $out = $lei->{1} or return; my $lk = $self->lock_for_scope; - print $out $$bref; + $lei->out($$bref); } } @@ -190,48 +181,37 @@ sub _json_pretty { sub ovv_each_smsg_cb { # runs in wq worker usually my ($self, $lei, $ibxish) = @_; - my $json; - $lei->{1}->autoflush(1); + my ($json, $dedupe); if (my $pkg = $self->{json}) { $json = $pkg->new; $json->utf8->canonical; $json->ascii(1) if $lei->{opt}->{ascii}; } my $l2m = $lei->{l2m}; - if ($l2m && $l2m->{-wq_s1}) { - my ($lei_ipc, @io) = $lei->atfork_parent_wq($l2m); - # n.b. $io[0] = qry_status_wr, $io[1] = mbox|stdout, - # $io[4] becomes a notification pipe that triggers EOF - # in this wq worker when all outstanding ->write_mail - # calls are complete - die "BUG: \$io[4] $io[4] unexpected" if $io[4]; - pipe($l2m->{each_smsg_done}, $io[4]) or die "pipe: $!"; - fcntl($io[4], 1031, 4096) if $^O eq 'linux'; - delete @$lei_ipc{qw(l2m opt mset_opt cmd)}; - my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git - $self->{git} = $git; - my $git_dir = $git->{git_dir}; - sub { - my ($smsg, $mitem) = @_; - my $kw = []; # TODO get from mitem - $l2m->wq_do('write_mail', \@io, $git_dir, - $smsg->{blob}, $lei_ipc, $kw) - } - } elsif ($l2m) { - my $wcb = $l2m->write_cb($lei); - my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git - $self->{git} = $git; # for ovv_atexit_child + if (!$l2m) { + $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing'; + $dedupe->prepare_dedupe; + } + $lei->{ovv_buf} = \(my $buf = '') if !$l2m; + if ($l2m && !$ibxish) { # remote https?:// mboxrd my $g2m = $l2m->can('git_to_mail'); + my $wcb = $l2m->write_cb($lei); sub { - my ($smsg, $mitem) = @_; - my $kw = []; # TODO get from mitem - $git->cat_async($smsg->{blob}, $g2m, [ $wcb, $kw ]); + my ($smsg, undef, $eml) = @_; # no mitem in $_[1] + $wcb->(undef, $smsg, $eml); }; + } elsif ($l2m && $l2m->{-wq_s1}) { + my $git_dir = $ibxish->git->{git_dir}; + sub { + my ($smsg, $mitem) = @_; + $smsg->{pct} = get_pct($mitem) if $mitem; + $l2m->wq_io_do('write_mail', [], $git_dir, $smsg); + } } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) { my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},"; - $lei->{ovv_buf} = \(my $buf = ''); sub { # DIY prettiness :P my ($smsg, $mitem) = @_; + return if $dedupe->is_smsg_dup($smsg); $smsg = _unbless_smsg($smsg, $mitem); $buf .= "{\n"; $buf .= join(",\n", map { @@ -244,30 +224,25 @@ sub ovv_each_smsg_cb { # runs in wq worker usually } } sort keys %$smsg); $buf .= $EOR; - if (length($buf) > 65536) { - my $lk = $self->lock_for_scope; - print { $lei->{1} } $buf; - $buf = ''; - } + return if length($buf) < 65536; + my $lk = $self->lock_for_scope; + $lei->out($buf); + $buf = ''; } } elsif ($json) { my $ORS = $self->{fmt} eq 'json' ? ",\n" : "\n"; # JSONL - $lei->{ovv_buf} = \(my $buf = ''); sub { my ($smsg, $mitem) = @_; - delete @$smsg{qw(tid num)}; + return if $dedupe->is_smsg_dup($smsg); $buf .= $json->encode(_unbless_smsg(@_)) . $ORS; - if (length($buf) > 65536) { - my $lk = $self->lock_for_scope; - print { $lei->{1} } $buf; - $buf = ''; - } - } - } elsif ($self->{fmt} eq 'oid') { - sub { - my ($smsg, $mitem) = @_; + return if length($buf) < 65536; + my $lk = $self->lock_for_scope; + $lei->out($buf); + $buf = ''; } - } # else { ... + } else { + die "TODO: unhandled case $self->{fmt}" + } } no warnings 'once';