X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2FPublicInbox%2FView.pm;h=81e83d97738256511a530bc4093bfd2793267eb8;hb=956ede734d7c2e8d0a3003c6e2d554114586643e;hp=ed3c96e80d6ec5297abbae0d1c523e515a42963d;hpb=95a438465b81a89b687ac0c2cabce54a300140a9;p=public-inbox.git diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index ed3c96e8..81e83d97 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -6,14 +6,14 @@ package PublicInbox::View; use strict; use warnings; -use URI::Escape qw/uri_escape_utf8/; use Date::Parse qw/str2time/; -use PublicInbox::Hval qw/ascii_html/; +use PublicInbox::Hval qw/ascii_html obfuscate_addrs/; use PublicInbox::Linkify; -use PublicInbox::MID qw/mid_clean id_compress mid_mime/; +use PublicInbox::MID qw/mid_clean id_compress mid_mime mid_escape/; use PublicInbox::MsgIter; use PublicInbox::Address; use PublicInbox::WwwStream; +use PublicInbox::Reply; require POSIX; use constant INDENT => ' '; @@ -24,11 +24,14 @@ sub th_pfx ($) { $_[0] == 0 ? '' : TCHILD }; sub msg_html { my ($ctx, $mime) = @_; my $hdr = $mime->header_obj; - my $tip = _msg_html_prepare($hdr, $ctx); + my $ibx = $ctx->{-inbox}; + my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef; + my $tip = _msg_html_prepare($hdr, $ctx, $obfs_ibx); PublicInbox::WwwStream->response($ctx, 200, sub { my ($nr, undef) = @_; if ($nr == 1) { - $tip . multipart_text_as_html($mime, '') . '
'; @@ -347,50 +367,49 @@ sub thread_html { $ctx->{prev_attr} = ''; $ctx->{prev_level} = 0; $ctx->{root_anchor} = anchor_for($mid); - $ctx->{seen} = {}; $ctx->{mapping} = {}; $ctx->{s_nr} = "$nr+ messages in thread"; - my $th = thread_results($msgs); - walk_thread($th, $ctx, *pre_thread); - $skel .= ''; - return stream_thread($th, $ctx) unless $ctx->{flat}; + my $rootset = thread_results($msgs, $srch); - # flat display: lazy load the full message from mini_mime: + # reduce hash lookups in pre_thread->skel_dump my $inbox = $ctx->{-inbox}; + $ctx->{-obfs_ibx} = $inbox->{obfuscate} ? $inbox : undef; + walk_thread($rootset, $ctx, *pre_thread); + + $skel .= ''; + return stream_thread($rootset, $ctx) unless $ctx->{flat}; + + # flat display: lazy load the full message from smsg my $mime; while ($mime = shift @$msgs) { - $mime = $inbox->msg_by_mid(mid_clean(mid_mime($mime))) and last; + $mime = $inbox->msg_by_smsg($mime) and last; } return missing_thread($ctx) unless $mime; - $mime = Email::MIME->new($mime); + $mime = PublicInbox::MIME->new($mime); $ctx->{-title_html} = ascii_html($mime->header('Subject')); $ctx->{-html_tip} = '
'.index_entry($mime, $ctx, scalar @$msgs); $mime = undef; PublicInbox::WwwStream->response($ctx, 200, sub { return unless $msgs; while ($mime = shift @$msgs) { - $mid = mid_clean(mid_mime($mime)); - $mime = $inbox->msg_by_mid($mid) and last; + $mime = $inbox->msg_by_smsg($mime) and last; } if ($mime) { - $mime = Email::MIME->new($mime); + $mime = PublicInbox::MIME->new($mime); return index_entry($mime, $ctx, scalar @$msgs); } $msgs = undef; - ''.$skel; + $skel; }); } sub multipart_text_as_html { - my ($mime, $upfx) = @_; + my ($mime, $upfx, $obfs_ibx) = @_; my $rv = ""; # scan through all parts, looking for displayable text - msg_iter($mime, sub { - my ($p) = @_; - $rv .= add_text_body($upfx, $p); - }); + msg_iter($mime, sub { $rv .= add_text_body($upfx, $obfs_ibx, $_[0]) }); $rv; } @@ -408,14 +427,16 @@ sub flush_quote { $$s .= qq() . $rv . '' } -sub attach_link ($$$$) { - my ($upfx, $ct, $p, $fn) = @_; +sub attach_link ($$$$;$) { + my ($upfx, $ct, $p, $fn, $err) = @_; my ($part, $depth, @idx) = @$p; my $nl = $idx[-1] > 1 ? "\n" : ''; my $idx = join('.', @idx); my $size = bytes::length($part->body); - $ct ||= 'text/plain'; - $ct =~ s/;.*//; # no attributes + + # hide attributes normally, unless we want to aid users in + # spotting MUA problems: + $ct =~ s/;.*// unless $err; $ct = ascii_html($ct); my $desc = $part->header('Content-Description'); $desc = $fn unless defined $desc; @@ -428,62 +449,85 @@ sub attach_link ($$$$) { } else { $sfn = 'a.bin'; } - my @ret = qq($nl[-- Attachment #$idx: ); + my $ret = qq($nl); + if ($err) { + $ret .= +"[-- Warning: decoded text below may be mangled --]\n"; + } + $ret .= "[-- Attachment #$idx: "; my $ts = "Type: $ct, Size: $size bytes"; - push(@ret, ($desc eq '') ? "$ts --]" : "$desc --]\n[-- $ts --]"); - join('', @ret, "\n"); + $desc = ascii_html($desc); + $ret .= ($desc eq '') ? "$ts --]" : "$desc --]\n[-- $ts --]"; + $ret .= "\n"; } sub add_text_body { - my ($upfx, $p) = @_; # from msg_iter: [ Email::MIME, depth, @idx ] - my ($part, $depth, @idx) = @$p; - my $ct = $part->content_type; + my ($upfx, $obfs_ibx, $p) = @_; + # $p - from msg_iter: [ Email::MIME, depth, @idx ] + my ($part, $depth) = @$p; # attachment @idx is unused + my $ct = $part->content_type || 'text/plain'; my $fn = $part->filename; - if (defined $ct && $ct =~ m!\btext/x?html\b!i) { + if ($ct =~ m!\btext/x?html\b!i) { return attach_link($upfx, $ct, $p, $fn); } my $s = eval { $part->body_str }; # badly-encoded message? tell the world about it! - return attach_link($upfx, $ct, $p, $fn) if $@; + my $err = $@; + if ($err) { + if ($ct =~ m!\btext/plain\b!i) { + # Try to assume UTF-8 because Alpine seems to + # do wacky things and set charset=X-UNKNOWN + $part->charset_set('UTF-8'); + $s = eval { $part->body_str }; + + # If forcing charset=UTF-8 failed, + # attach_link will warn further down... + $s = $part->body if $@; + } else { + return attach_link($upfx, $ct, $p, $fn); + } + } my @lines = split(/^/m, $s); $s = ''; - if (defined($fn) || $depth > 0) { - $s .= attach_link($upfx, $ct, $p, $fn); + if (defined($fn) || $depth > 0 || $err) { + $s .= attach_link($upfx, $ct, $p, $fn, $err); $s .= "\n"; } my @quot; my $l = PublicInbox::Linkify->new; - while (defined(my $cur = shift @lines)) { + foreach my $cur (@lines) { if ($cur !~ /^>/) { # show the previously buffered quote inline flush_quote(\$s, $l, \@quot) if @quot; # regular line, OK - $cur = $l->linkify_1($cur); - $cur = ascii_html($cur); - $s .= $l->linkify_2($cur); + $l->linkify_1($cur); + $s .= $l->linkify_2(ascii_html($cur)); } else { push @quot, $cur; } } - my $end = "\n"; - if (@quot) { - $end = ''; + if (@quot) { # ugh, top posted flush_quote(\$s, $l, \@quot); + obfuscate_addrs($obfs_ibx, $s) if $obfs_ibx; + $s; + } else { + obfuscate_addrs($obfs_ibx, $s) if $obfs_ibx; + if ($s =~ /\n\z/s) { # common, last line ends with a newline + $s; + } else { # some editors don't do newlines... + $s .= "\n"; + } } - $s =~ s/[ \t]+$//sgm; # kill per-line trailing whitespace - $s =~ s/\A\n+//s; # kill leading blank lines - $s =~ s/\s+\z//s; # kill all trailing spaces - $s .= $end; } sub _msg_html_prepare { - my ($hdr, $ctx) = @_; + my ($hdr, $ctx, $obfs_ibx) = @_; my $srch = $ctx->{srch} if $ctx; my $atom = ''; my $rv = "
"; # anchor for body start @@ -492,7 +536,7 @@ sub _msg_html_prepare { $ctx->{-upfx} = '../'; } my @title; - my $mid = $hdr->header_raw('Message-ID'); + my $mid = mid_clean($hdr->header_raw('Message-ID')); $mid = PublicInbox::Hval->new_msgid($mid); foreach my $h (qw(From To Cc Subject Date)) { my $v = $hdr->header($h); @@ -502,6 +546,7 @@ sub _msg_html_prepare { if ($h eq 'From') { my @n = PublicInbox::Address::names($v->raw); $title[1] = ascii_html(join(', ', @n)); + obfuscate_addrs($obfs_ibx, $title[1]) if $obfs_ibx; } elsif ($h eq 'Subject') { $title[0] = $v->as_html; if ($srch) { @@ -511,7 +556,7 @@ sub _msg_html_prepare { } } $v = $v->as_html; - $v =~ s/(\@[^,]+,) /$1\n\t/g if ($h eq 'Cc' || $h eq 'To'); + obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx; $rv .= "$h: $v\n"; } @@ -550,13 +595,20 @@ sub thread_skel { $$dst .= "$nr+ messages in thread ($expand"; $$dst .= qq! / [top])\n!; - my $subj = $srch->subject_path($hdr->header('Subject')); - $ctx->{seen} = { $subj => 1 }; + my $subj = $hdr->header('Subject'); + defined $subj or $subj = ''; + $ctx->{prev_subj} = [ split(/ /, $srch->subject_normalized($subj)) ]; $ctx->{cur} = $mid; $ctx->{prev_attr} = ''; $ctx->{prev_level} = 0; $ctx->{dst} = $dst; - walk_thread(thread_results(load_results($sres)), $ctx, *skel_dump); + $sres = load_results($srch, $sres); + + # reduce hash lookups in skel_dump + my $ibx = $ctx->{-inbox}; + $ctx->{-obfs_ibx} = $ibx->{obfuscate} ? $ibx : undef; + walk_thread(thread_results($sres, $srch), $ctx, *skel_dump); + $ctx->{parent_msg} = $parent; } @@ -568,7 +620,7 @@ sub _parent_headers { if (defined $irt) { my $v = PublicInbox::Hval->new_msgid($irt); my $html = $v->as_html; - my $href = $v->as_href; + my $href = $v->{href}; $rv .= "In-Reply-To: <"; $rv .= "$html>\n"; } @@ -597,50 +649,6 @@ sub _parent_headers { $rv; } -sub squote_maybe ($) { - my ($val) = @_; - if ($val =~ m{([^\w@\./,\%\+\-])}) { - $val =~ s/(['!])/'\\$1'/g; # '!' for csh - return "'$val'"; - } - $val; -} - -sub mailto_arg_link { - my ($hdr) = @_; - my %cc; # everyone else - my $to; # this is the From address - - foreach my $h (qw(From To Cc)) { - my $v = $hdr->header($h); - defined($v) && ($v ne '') or next; - my @addrs = PublicInbox::Address::emails($v); - foreach my $address (@addrs) { - my $dst = lc($address); - $cc{$dst} ||= $address; - $to ||= $dst; - } - } - my @arg; - - my $subj = $hdr->header('Subject') || ''; - $subj = "Re: $subj" unless $subj =~ /\bRe:/i; - my $mid = $hdr->header_raw('Message-ID'); - push @arg, '--in-reply-to='.squote_maybe(mid_clean($mid)); - my $irt = uri_escape_utf8($mid); - delete $cc{$to}; - push @arg, "--to=$to"; - $to = uri_escape_utf8($to); - $subj = uri_escape_utf8($subj); - my @cc = sort values %cc; - push(@arg, map { "--cc=$_" } @cc); - my $cc = uri_escape_utf8(join(',', @cc)); - my $href = "mailto:$to?In-Reply-To=$irt&Cc=${cc}&Subject=$subj"; - $href =~ s/%20/+/g; - - (\@arg, ascii_html($href)); -} - sub html_footer { my ($hdr, $standalone, $ctx, $rhref) = @_; @@ -657,17 +665,17 @@ sub html_footer { $next = $prev = ' '; if (my $n = $ctx->{next_msg}) { - $n = PublicInbox::Hval->new_msgid($n)->as_href; + $n = PublicInbox::Hval->new_msgid($n)->{href}; $next = "next"; } my $u; my $par = $ctx->{parent_msg}; if ($par) { - $u = PublicInbox::Hval->new_msgid($par)->as_href; + $u = PublicInbox::Hval->new_msgid($par)->{href}; $u = "$upfx$u/"; } if (my $p = $ctx->{prev_msg}) { - $prev = PublicInbox::Hval->new_msgid($p)->as_href; + $prev = PublicInbox::Hval->new_msgid($p)->{href}; if ($p && $par && $p eq $par) { $prev = "prev parent'; @@ -692,7 +700,7 @@ sub html_footer { sub linkify_ref_nosrch { my $v = PublicInbox::Hval->new_msgid($_[0]); my $html = $v->as_html; - my $href = $v->as_href; + my $href = $v->{href}; "<$html>"; } @@ -703,24 +711,22 @@ sub anchor_for { sub ghost_parent { my ($upfx, $mid) = @_; - # 'subject dummy' is used internally by Mail::Thread - return '[no common parent]' if ($mid eq 'subject dummy'); $mid = PublicInbox::Hval->new_msgid($mid); - my $href = $mid->as_href; + my $href = $mid->{href}; my $html = $mid->as_html; qq{[parent not found: <$html>]}; } sub indent_for { my ($level) = @_; - INDENT x ($level - 1); + $level ? INDENT x ($level - 1) : ''; } sub load_results { - my ($sres) = @_; - - [ map { $_->mini_mime } @{delete $sres->{msgs}} ]; + my ($srch, $sres) = @_; + my $msgs = delete $sres->{msgs}; + $srch->retry_reopen(sub { [ map { $_->mid; $_ } @$msgs ] }); } sub msg_timestamp { @@ -730,12 +736,9 @@ sub msg_timestamp { } sub thread_results { - my ($msgs) = @_; - require PublicInbox::Thread; - my $th = PublicInbox::Thread->new(@$msgs); - $th->thread; - $th->order(*sort_ts); - $th + my ($msgs, $srch) = @_; + require PublicInbox::SearchThread; + PublicInbox::SearchThread::thread($msgs, *sort_ts, $srch); } sub missing_thread { @@ -752,14 +755,38 @@ sub _msg_date { sub fmt_ts { POSIX::strftime('%Y-%m-%d %k:%M', gmtime($_[0])) } -sub _skel_header { - my ($ctx, $hdr, $level) = @_; +sub dedupe_subject { + my ($prev_subj, $subj, $val) = @_; + + my $omit = ''; # '"' denotes identical text omitted + my (@prev_pop, @curr_pop); + while (@$prev_subj && @$subj && $subj->[-1] eq $prev_subj->[-1]) { + push(@prev_pop, pop(@$prev_subj)); + push(@curr_pop, pop(@$subj)); + $omit ||= $val; + } + pop @$subj if @$subj && $subj->[-1] =~ /^re:\s*/i; + if (scalar(@curr_pop) == 1) { + $omit = ''; + push @$prev_subj, @prev_pop; + push @$subj, @curr_pop; + } + $omit; +} + +sub skel_dump { + my ($ctx, $level, $node) = @_; + my $smsg = $node->{smsg} or return _skel_ghost($ctx, $level, $node); my $dst = $ctx->{dst}; my $cur = $ctx->{cur}; - my $mid = mid_clean($hdr->header_raw('Message-ID')); - my $f = ascii_html($hdr->header('X-PI-From')); - my $d = _msg_date($hdr) . ' ' . indent_for($level) . th_pfx($level); + my $mid = $smsg->{mid}; + + my $f = ascii_html($smsg->from_name); + my $obfs_ibx = $ctx->{-obfs_ibx}; + obfuscate_addrs($obfs_ibx, $f) if $obfs_ibx; + + my $d = fmt_ts($smsg->{ts}) . ' ' . indent_for($level) . th_pfx($level); my $attr = $f; $ctx->{first_level} ||= $level; @@ -784,76 +811,67 @@ sub _skel_header { # Subject is never undef, this mail was loaded from # our Xapian which would've resulted in '' if it were # really missing (and Filter rejects empty subjects) - my $s = $hdr->header('Subject'); - my $h = $ctx->{srch}->subject_path($s); - if ($ctx->{seen}->{$h}) { - $s = undef; + my @subj = split(/ /, $ctx->{srch}->subject_normalized($smsg->subject)); + + # remove common suffixes from the subject if it matches the previous, + # so we do not show redundant text at the end. + my $prev_subj = $ctx->{prev_subj} || []; + $ctx->{prev_subj} = [ @subj ]; + my $omit = dedupe_subject($prev_subj, \@subj, '" '); + my $end; + if (@subj) { + my $subj = join(' ', @subj); + $subj = ascii_html($subj); + obfuscate_addrs($obfs_ibx, $subj) if $obfs_ibx; + $end = "$subj $omit$f\n" } else { - $ctx->{seen}->{$h} = 1; - $s = PublicInbox::Hval->new($s); - $s = $s->as_html; + $end = "$f\n"; } - my $m = PublicInbox::Hval->new_msgid($mid); + my $m; my $id = ''; my $mapping = $ctx->{mapping}; - my $end = defined($s) ? "$s $f\n" : "$f\n"; if ($mapping) { my $map = $mapping->{$mid}; $id = id_compress($mid, 1); $m = '#m'.$id; - $map->[1] = "$d$end"; + $map->[0] = "$d$end"; $id = "\nid=r".$id; } else { - $m = $ctx->{-upfx}.$m->as_href.'/'; + $m = $ctx->{-upfx}.mid_escape($mid).'/'; } $$dst .= $d . "" . $end; } -sub skel_dump { +sub _skel_ghost { my ($ctx, $level, $node) = @_; - if (my $mime = $node->message) { - _skel_header($ctx, $mime->header_obj, $level); + + my $mid = $node->{id}; + my $d = $ctx->{pct} ? ' [irrelevant] ' # search result + : ' [not found] '; + $d .= indent_for($level) . th_pfx($level); + my $upfx = $ctx->{-upfx}; + my $m = PublicInbox::Hval->new_msgid($mid); + my $href = $upfx . $m->{href} . '/'; + my $html = $m->as_html; + + my $mapping = $ctx->{mapping}; + my $map = $mapping->{$mid} if $mapping; + if ($map) { + my $id = id_compress($mid, 1); + $map->[0] = $d . qq{<$html>\n}; + $d .= qq{<$html>\n}; } else { - my $mid = $node->messageid; - my $dst = $ctx->{dst}; - my $mapping = $ctx->{mapping}; - my $map = $mapping->{$mid} if $mapping; - if ($mid eq 'subject dummy') { - my $ncp = "\t[no common parent]\n"; - $map->[1] = $ncp if $map; - $$dst .= $ncp; - return; - } - my $d = $ctx->{pct} ? ' [irrelevant] ' # search result - : ' [not found] '; - $d .= indent_for($level) . th_pfx($level); - my $upfx = $ctx->{-upfx}; - my $m = PublicInbox::Hval->new_msgid($mid); - my $href = $upfx . $m->as_href . '/'; - my $html = $m->as_html; - - if ($map) { - my $id = id_compress($mid, 1); - $map->[1] = $d . qq{<$html>\n}; - $d .= qq{<$html>\n}; - } else { - $d .= qq{<$html>\n}; - } - $$dst .= $d; + $d .= qq{<$html>\n}; } + my $dst = $ctx->{dst}; + $$dst .= $d; } sub sort_ts { - sort { - (eval { $a->topmost->message->header('X-PI-TS') } || 0) <=> - (eval { $b->topmost->message->header('X-PI-TS') } || 0) - } @_; -} - -sub _tryload_ghost ($$) { - my ($srch, $mid) = @_; - my $smsg = $srch->lookup_mail($mid) or return; - $smsg->mini_mime; + [ sort { + (eval { $a->topmost->{smsg}->ts } || 0) <=> + (eval { $b->topmost->{smsg}->ts } || 0) + } @{$_[0]} ]; } # accumulate recent topics if search is supported @@ -861,15 +879,14 @@ sub _tryload_ghost ($$) { sub acc_topic { my ($ctx, $level, $node) = @_; my $srch = $ctx->{srch}; - my $mid = $node->messageid; - my $x = $node->message || _tryload_ghost($srch, $mid); + my $mid = $node->{id}; + my $x = $node->{smsg} || $srch->lookup_mail($mid); my ($subj, $ts); my $topic; if ($x) { - $x = $x->header_obj; - $subj = $x->header('Subject') || ''; + $subj = $x->subject; $subj = $srch->subject_normalized($subj); - $ts = $x->header('X-PI-TS'); + $ts = $x->ts; if ($level == 0) { $topic = [ $ts, 1, { $subj => $mid }, $subj ]; $ctx->{-cur_topic} = $topic; @@ -904,6 +921,9 @@ sub dump_topics { } my @out; + my $ibx = $ctx->{-inbox}; + my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef; + my $srch = $ctx->{srch}; # sort by recency, this allows new posts to "bump" old topics... foreach my $topic (sort { $b->[0] <=> $a->[0] } @$order) { @@ -911,7 +931,8 @@ sub dump_topics { @$topic = (); next unless defined $top; # ghost topic my $mid = delete $seen->{$top}; - my $href = PublicInbox::Hval->new_msgid($mid)->as_href; + my $href = mid_escape($mid); + my $prev_subj = [ split(/ /, $top) ]; $top = PublicInbox::Hval->new($top)->as_html; $ts = fmt_ts($ts); @@ -932,12 +953,17 @@ sub dump_topics { " $ts UTC $n - $mbox / $atom\n"; for (my $i = 0; $i < scalar(@ex); $i += 2) { my $level = $ex[$i]; - my $sub = $ex[$i + 1]; - $mid = delete $seen->{$sub}; - $sub = PublicInbox::Hval->new($sub)->as_html; - $href = PublicInbox::Hval->new_msgid($mid)->as_href; + my $subj = $ex[$i + 1]; + $mid = delete $seen->{$subj}; + my @subj = split(/ /, $srch->subject_normalized($subj)); + my @next_prev = @subj; # full copy + my $omit = dedupe_subject($prev_subj, \@subj, ' "'); + $prev_subj = \@next_prev; + $subj = ascii_html(join(' ', @subj)); + obfuscate_addrs($obfs_ibx, $subj) if $obfs_ibx; + $href = mid_escape($mid); $s .= indent_for($level) . TCHILD; - $s .= "$sub\n"; + $s .= "$subj$omit\n"; } push @out, $s; } @@ -975,11 +1001,12 @@ sub index_topics { my $opts = { offset => $off, limit => 200 }; $ctx->{order} = []; - my $sres = $ctx->{srch}->query('', $opts); + my $srch = $ctx->{srch}; + my $sres = $srch->query('', $opts); my $nr = scalar @{$sres->{msgs}}; if ($nr) { - $sres = load_results($sres); - walk_thread(thread_results($sres), $ctx, *acc_topic); + $sres = load_results($srch, $sres); + walk_thread(thread_results($sres, $srch), $ctx, *acc_topic); } $ctx->{-next_o} = $off+ $nr; $ctx->{-cur_o} = $off; @@ -1011,9 +1038,10 @@ sub thread_adj_level { } sub ghost_index_entry { - my ($ctx, $level, $mid) = @_; + my ($ctx, $level, $node) = @_; my ($beg, $end) = thread_adj_level($ctx, $level); - $beg . ''. ghost_parent($ctx->{-upfx}, $mid) . '' . $end; + $beg . ''. ghost_parent($ctx->{-upfx}, $node->{id}) + . '' . $end; } 1;