lib/PublicInbox/Search.pm | 3 --- lib/PublicInbox/SearchIdx.pm | 26 ++++++++++++++++++++++---- lib/PublicInbox/Smsg.pm | 18 ------------------ t/search.t | 4 ++-- diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index fb35b74752c88531fe3ad0c2c8774dcf598d1b38..0321ca9352944d720a0962df8e3d1097ca9de68f 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -90,9 +90,6 @@ # just an integer: $ENQ_ASCENDING = $x eq 'Xapian' ? 1 : Search::Xapian::ENQ_ASCENDING(); - # for Smsg: - *PublicInbox::Smsg::sortable_unserialise = - $Xap.'::sortable_unserialise'; # n.b. FLAG_PURE_NOT is expensive not suitable for a public # website as it could become a denial-of-service vector # FLAG_PHRASE also seems to cause performance problems chert diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index eb620f44fe990e4c38514d9375fa6328ae30e80d..803494f508291baffe932f0671e3adabb76f0b20 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -17,6 +17,7 @@ use PublicInbox::MsgIter; use PublicInbox::IdxStack; use Carp qw(croak); use POSIX qw(strftime); +use Time::Local qw(timegm); use PublicInbox::OverIdx; use PublicInbox::Spawn qw(spawn nodatacow_dir); use PublicInbox::Git qw(git_unquote); @@ -104,6 +105,7 @@ $X->{$_} = $xap.'::'.$_; } eval 'require '.$X->{WritableDatabase} or die; *sortable_serialise = $xap.'::sortable_serialise'; + *sortable_unserialise = $xap.'::sortable_unserialise'; $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()'); $DB_OPEN = eval($xap.'::DB_OPEN()'); my $ver = (eval($xap.'::major_version()') << 16) | @@ -434,6 +436,23 @@ } $smsg->{num}; } +sub get_val ($$) { + my ($doc, $col) = @_; + sortable_unserialise($doc->get_value($col)); +} + +sub smsg_from_doc ($) { + my ($doc) = @_; + my $data = $doc->get_data or return; + my $smsg = bless {}, 'PublicInbox::Smsg'; + $smsg->{ts} = get_val($doc, PublicInbox::Search::TS()); + my $dt = get_val($doc, PublicInbox::Search::DT()); + my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt); + $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy); + $smsg->load_from_data($data); + $smsg; +} + sub xdb_remove { my ($self, $oid, @removed) = @_; my $xdb = $self->{xdb} or return; @@ -444,10 +463,9 @@ warn "E: $@\n" if $@; warn "E: #$num $oid missing in Xapian\n"; next; } - my $smsg = bless {}, 'PublicInbox::Smsg'; - $smsg->load_expand($doc); - my $blob = $smsg->{blob} // '(unset)'; - if ($blob eq $oid) { + my $smsg = smsg_from_doc($doc); + my $blob = $smsg->{blob}; # may be undef if --skip-docdata + if (!defined($blob) || $blob eq $oid) { $xdb->delete_document($num); } else { warn "E: #$num $oid != $blob in Xapian\n"; diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm index 171e0a00908e4b7bdf51ba05beda6dcc498aa08c..140865389bce65102b9a00ee2c739780d4caefb5 100644 --- a/lib/PublicInbox/Smsg.pm +++ b/lib/PublicInbox/Smsg.pm @@ -15,13 +15,6 @@ our @EXPORT_OK = qw(subject_normalized); use PublicInbox::MID qw(mids); use PublicInbox::Address; use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); -use Time::Local qw(timegm); - -sub get_val ($$) { - my ($doc, $col) = @_; - # sortable_unserialise is defined by PublicInbox::Search::load_xapian() - sortable_unserialise($doc->get_value($col)); -} sub to_doc_data { my ($self) = @_; @@ -59,17 +52,6 @@ # NNTP only $self->{bytes}, $self->{lines} ) = split(/\n/, $_[1]); -} - -sub load_expand { - my ($self, $doc) = @_; - my $data = $doc->get_data or return; - $self->{ts} = get_val($doc, PublicInbox::Search::TS()); - my $dt = get_val($doc, PublicInbox::Search::DT()); - my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt); - $self->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy); - load_from_data($self, $data); - $self; } sub psgi_cull ($) { diff --git a/t/search.t b/t/search.t index 8df8a202be6ff2abccf9891c3d5b65042aabb1fa..a66aec36598d887fd6eb67de4de8b03f800d6d32 100644 --- a/t/search.t +++ b/t/search.t @@ -332,13 +332,13 @@ my $smsg = $ibx->over->get_art($m->get_docid); like($smsg->{to}, qr/\blist\@example\.com\b/, 'to appears'); my $doc = $m->get_document; my $col = PublicInbox::Search::BYTES(); - my $bytes = PublicInbox::Smsg::get_val($doc, $col); + my $bytes = PublicInbox::SearchIdx::get_val($doc, $col); like($bytes, qr/\A[0-9]+\z/, '$bytes stored as digit'); ok($bytes > 0, '$bytes is > 0'); is($bytes, $smsg->{bytes}, 'bytes Xapian value matches Over'); $col = PublicInbox::Search::UID(); - my $uid = PublicInbox::Smsg::get_val($doc, $col); + my $uid = PublicInbox::SearchIdx::get_val($doc, $col); is($uid, $smsg->{num}, 'UID column matches {num}'); is($uid, $m->get_docid, 'UID column matches docid'); }