X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiOverview.pm;h=b4d8132855f58f0b6626cc38236b4f22837fb323;hb=43c43f785aa53607a0dd050989da5d7fd0dcfff4;hp=ef5f27c1309bd78f697adf558e654b00cf372601;hpb=f49d6a9ac8ff542c625e909798ef0947df45f34e;p=public-inbox.git diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index ef5f27c1..b4d81328 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -15,58 +15,77 @@ use PublicInbox::MID qw($MID_EXTRACT); use PublicInbox::Address qw(pairs); use PublicInbox::Config; use PublicInbox::Search qw(get_pct); +use PublicInbox::LeiDedupe; +use PublicInbox::LeiToMail; # cf. https://en.wikipedia.org/wiki/JSON_streaming my $JSONL = 'ldjson|ndjson|jsonl'; # 3 names for the same thing sub _iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) } -# we open this in the parent process before ->wq_do handoff +# we open this in the parent process before ->wq_io_do handoff sub ovv_out_lk_init ($) { my ($self) = @_; - $self->{tmp_lk_id} = "$self.$$"; - my $tmp = File::Temp->new("lei-ovv.out.$$.lock-XXXXXX", + my $tmp = File::Temp->new("lei-ovv.dst.$$.lock-XXXXXX", TMPDIR => 1, UNLINK => 0); - $self->{lock_path} = $tmp->filename; + $self->{"lk_id.$self.$$"} = $self->{lock_path} = $tmp->filename; } sub ovv_out_lk_cancel ($) { my ($self) = @_; - ($self->{tmp_lk_id}//'') eq "$self.$$" and - unlink(delete($self->{lock_path})); + my $lock_path = delete $self->{"lk_id.$self.$$"} or return; + unlink($lock_path); +} + +sub detect_fmt ($$) { + my ($lei, $dst) = @_; + if ($dst =~ m!\A([:/]+://)!) { + $lei->fail("$1 support not implemented, yet\n"); + } elsif (!-e $dst || -d _) { + 'maildir'; # the default TODO: MH? + } elsif (-f _ || -p _) { + $lei->fail("unable to determine mbox family of $dst\n"); + } else { + $lei->fail("unable to determine format of $dst\n"); + } } sub new { - my ($class, $lei) = @_; + my ($class, $lei, $ofmt_key) = @_; my $opt = $lei->{opt}; - my $out = $opt->{output} // '-'; - $out = '/dev/stdout' if $out eq '-'; + my $dst = $opt->{output} // '-'; + $dst = '/dev/stdout' if $dst eq '-'; + $ofmt_key //= 'format'; - my $fmt = $opt->{'format'}; + my $fmt = $opt->{$ofmt_key}; $fmt = lc($fmt) if defined $fmt; - if ($out =~ s/\A([a-z]+)://is) { # e.g. Maildir:/home/user/Mail/ + if ($dst =~ m!\A([a-z0-9\+]+)://!is) { + defined($fmt) and return $lei->fail(<<""); +--$ofmt_key=$fmt invalid with URL $dst + + $fmt = lc $1; + } elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/ my $ofmt = lc $1; $fmt //= $ofmt; return $lei->fail(<<"") if $fmt ne $ofmt; ---format=$fmt and --output=$ofmt conflict +--$ofmt_key=$fmt and --output=$ofmt conflict } - $fmt //= 'json' if $out eq '/dev/stdout'; - $fmt //= 'maildir'; # TODO + $fmt //= 'json' if $dst eq '/dev/stdout'; + $fmt //= detect_fmt($lei, $dst) or return; - if (index($out, '://') < 0) { # not a URL, so assume path - $out = File::Spec->canonpath($out); + if (index($dst, '://') < 0) { # not a URL, so assume path + $dst = File::Spec->canonpath($dst); } # else URL - my $self = bless { fmt => $fmt, out => $out }, $class; + my $self = bless { fmt => $fmt, dst => $dst }, $class; + $lei->{ovv} = $self; my $json; if ($fmt =~ /\A($JSONL|(?:concat)?json)\z/) { $json = $self->{json} = ref(PublicInbox::Config->json); } - my ($isatty, $seekable); - if ($out eq '/dev/stdout') { - $isatty = -t $lei->{1}; - $lei->start_pager if $isatty; + if ($dst eq '/dev/stdout') { + my $isatty = $lei->{need_pager} = -t $lei->{1}; $opt->{pretty} //= $isatty; if (!$isatty && -f _) { my $fl = fcntl($lei->{1}, F_GETFL, 0) // @@ -75,10 +94,20 @@ sub new { } else { ovv_out_lk_init($self); } - } elsif ($json) { - return $lei->fail('JSON formats only output to stdout'); + } elsif (!$opt->{quiet}) { + $lei->{-progress} = 1; + } + if ($json) { + $lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei); } else { - return $lei->fail("TODO: $out -f $fmt"); + # default to the cheapest sort since MUA usually resorts + $opt->{'sort'} //= 'docid' if $dst ne '/dev/stdout'; + $lei->{l2m} = eval { PublicInbox::LeiToMail->new($lei) }; + return $lei->fail($@) if $@; + if ($opt->{mua} && $lei->{l2m}->lock_free) { + $lei->{early_mua} = 1; + $opt->{alert} //= [ ':WINCH,:bell' ] if -t $lei->{1}; + } } $self; } @@ -87,7 +116,7 @@ sub new { sub ovv_begin { my ($self, $lei) = @_; if ($self->{fmt} eq 'json') { - print { $lei->{1} } '['; + $lei->out('['); } # TODO HTML/Atom/... } @@ -97,47 +126,43 @@ sub ovv_end { if ($self->{fmt} eq 'json') { # JSON doesn't allow trailing commas, and preventing # trailing commas is a PITA when parallelizing outputs - print { $lei->{1} } "null]\n"; + $lei->out("null]\n"); } elsif ($self->{fmt} eq 'concatjson') { - print { $lei->{1} } "\n"; + $lei->out("\n"); } } -sub ovv_atfork_child { - my ($self) = @_; - # reopen dedupe here -} - # prepares an smsg for JSON sub _unbless_smsg { my ($smsg, $mitem) = @_; - delete @$smsg{qw(lines bytes num tid)}; + # TODO: make configurable + # num/tid are nonsensical with multi-inbox search, + # lines/bytes are not generally useful + delete @$smsg{qw(num tid lines bytes)}; $smsg->{rt} = _iso8601(delete $smsg->{ts}); # JMAP receivedAt $smsg->{dt} = _iso8601(delete $smsg->{ds}); # JMAP UTCDate - $smsg->{relevance} = get_pct($mitem) if $mitem; - + $smsg->{pct} = get_pct($mitem) if $mitem; if (my $r = delete $smsg->{references}) { - $smsg->{refs} = [ - map { "<$_>" } ($r =~ m/$MID_EXTRACT/go) ]; + $smsg->{refs} = [ map { $_ } ($r =~ m/$MID_EXTRACT/go) ]; } if (my $m = delete($smsg->{mid})) { - $smsg->{'m'} = "<$m>"; + $smsg->{'m'} = $m; } for my $f (qw(from to cc)) { my $v = delete $smsg->{$f} or next; $smsg->{substr($f, 0, 1)} = pairs($v); } $smsg->{'s'} = delete $smsg->{subject}; - # can we be bothered to parse From/To/Cc into arrays? - scalar { %$smsg }; # unbless + my $kw = delete($smsg->{kw}); + scalar { %$smsg, ($kw && scalar(@$kw) ? (kw => $kw) : ()) }; # unbless } sub ovv_atexit_child { my ($self, $lei) = @_; if (my $bref = delete $lei->{ovv_buf}) { my $lk = $self->lock_for_scope; - print { $lei->{1} } $$bref; + $lei->out($$bref); } } @@ -155,7 +180,10 @@ sub _json_pretty { $pair =~ s/(null|"),"/$1, "/g; $pair; } @$v) . ']'; - } else { # references + } elsif ($k eq 'kw') { # keywords are short, one-line + $v = $json->encode($v); + $v =~ s/","/", "/g; + } else { # refs, labels, ... $v = '[' . join($sep, map { substr($json->encode([$_]), 1, -1); } @$v) . ']'; @@ -167,20 +195,39 @@ sub _json_pretty { qq{ "$k": }.$v; } -sub ovv_each_smsg_cb { - my ($self, $lei) = @_; - $lei->{ovv_buf} = \(my $buf = ''); - delete(@$self{qw(lock_path tmp_lk_id)}) unless $lei->{-parallel}; - my $json = $self->{json}->new; - $lei->{1}->autoflush(1); - if ($json) { +sub ovv_each_smsg_cb { # runs in wq worker usually + my ($self, $lei, $ibxish) = @_; + my ($json, $dedupe); + if (my $pkg = $self->{json}) { + $json = $pkg->new; $json->utf8->canonical; $json->ascii(1) if $lei->{opt}->{ascii}; } - if ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) { + my $l2m = $lei->{l2m}; + if (!$l2m) { + $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing'; + $dedupe->prepare_dedupe; + } + $lei->{ovv_buf} = \(my $buf = '') if !$l2m; + if ($l2m && !$ibxish) { # remote https?:// mboxrd + my $wcb = $l2m->write_cb($lei); + sub { + my ($smsg, undef, $eml) = @_; # no mitem in $_[1] + $wcb->(undef, $smsg, $eml); + }; + } elsif ($l2m && $l2m->{-wq_s1}) { + sub { + my ($smsg, $mitem) = @_; + $smsg->{pct} = get_pct($mitem) if $mitem; + $l2m->wq_io_do('write_mail', [], $smsg); + } + } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) { my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},"; + my $lse = $lei->{sto}->search; sub { # DIY prettiness :P my ($smsg, $mitem) = @_; + return if $dedupe->is_smsg_dup($smsg); + $lse->xsmsg_vmd($smsg, $smsg->{L} ? undef : 1); $smsg = _unbless_smsg($smsg, $mitem); $buf .= "{\n"; $buf .= join(",\n", map { @@ -193,29 +240,27 @@ sub ovv_each_smsg_cb { } } sort keys %$smsg); $buf .= $EOR; - if (length($buf) > 65536) { - my $lk = $self->lock_for_scope; - print { $lei->{1} } $buf; - $buf = ''; - } + return if length($buf) < 65536; + my $lk = $self->lock_for_scope; + $lei->out($buf); + $buf = ''; } } elsif ($json) { my $ORS = $self->{fmt} eq 'json' ? ",\n" : "\n"; # JSONL + my $lse = $lei->{sto}->search; sub { my ($smsg, $mitem) = @_; - delete @$smsg{qw(tid num)}; + return if $dedupe->is_smsg_dup($smsg); + $lse->xsmsg_vmd($smsg, $smsg->{L} ? undef : 1); $buf .= $json->encode(_unbless_smsg(@_)) . $ORS; - if (length($buf) > 65536) { - my $lk = $self->lock_for_scope; - print { $lei->{1} } $buf; - $buf = ''; - } - } - } elsif ($self->{fmt} eq 'oid') { - sub { - my ($smsg, $mitem) = @_; + return if length($buf) < 65536; + my $lk = $self->lock_for_scope; + $lei->out($buf); + $buf = ''; } - } # else { ... + } else { + die "TODO: unhandled case $self->{fmt}" + } } no warnings 'once';