smsg: handle wide characters in raw mail headers

There may be messages in the wild with wide characters in
headers which aren't non-RFC2047 encoded.  Assume UTF-8 so
those fields can round trip through over.sqlite3.

This doesn't affect docdata.glass in Xapian, but it does
affect how over.sqlite3 stores the same deflated info.

diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index aaf88f355e60952291226aefba9ae970e369a609..62cb951eb9603e92376e6bd3318a922b9f830d4e 100644 (file)
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -105,6 +105,9 @@ sub populate {
                # to protect git and NNTP clients
                $val =~ tr/\0\t\n/   /;
 
+               # rare: in case headers have wide chars (not RFC2047-encoded)
+               utf8::decode($val);
+
                # lower-case fields for read-only stuff
                $self->{lc($f)} = $val;
 

diff --git a/t/psgi_search.t b/t/psgi_search.t
index 2d12ba6a2c6a886e7db9019024a2dd13a47c26d5..5d537363d3aa46f80ae49a38d1f6fe38debb6cc0 100644 (file)
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -28,8 +28,10 @@ my $im = $ibx->importer(0);
 my $digits = '10010260936330';
 my $ua = 'Pine.LNX.4.10';
 my $mid = "$ua.$digits.2460-100000\@penguin.transmeta.com";
+
+# n.b. these headers are not properly RFC2047-encoded
 my $mime = PublicInbox::Eml->new(<<EOF);
-Subject: test
+Subject: test Ævar
 Message-ID: <$mid>
 From: Ævar Arnfjörð Bjarmason <avarab\@example>
 To: git\@vger.kernel.org
@@ -102,6 +104,8 @@ test_psgi(sub { $www->call(@_) }, sub {
                'subject-less message linked from "/$INBOX/"');
        like($html, qr/\bhref="blank-subject[^>]+>\(no subject\)</,
                'blank subject message linked from "/$INBOX/"');
+       like($html, qr/test &#198;var/,
+               "displayed Ævar's name properly in topic view");
 
        $res = $cb->(GET('/test/?q=tc:git'));
        like($html, qr/\bhref="no-subject-at-all[^>]+>\(no subject\)</,