index: account for CRLF conversion when storing bytes

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 1a824531f3cb2df71a8bdd015f891ac6dd6edcfd..913794316330bbc261d6f236f0e5a355c5b11ec5 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -19,7 +19,6 @@ use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::SearchIdx;
-use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::MultiMidQueue;
 use IO::Handle; # ->autoflush
 use File::Temp qw(tempfile);
@@ -156,12 +155,12 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$) {
        my ($self, $msgref, $mime, $smsg) = @_;
-       $smsg->{ds} //= msg_datestamp($mime->header_obj, $self->{autime});
-       $smsg->{ts} //= msg_timestamp($mime->header_obj, $self->{cotime});
+       $smsg->{bytes} = $smsg->{raw_bytes} +
+                       PublicInbox::SearchIdx::crlf_adjust($$msgref);
        $self->{over}->add_overview($mime, $smsg);
        my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
        $idx->index_raw($msgref, $mime, $smsg);
-       my $n = $self->{transact_bytes} += $smsg->{bytes};
+       my $n = $self->{transact_bytes} += $smsg->{raw_bytes};
        $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
 }
 
@@ -571,10 +570,12 @@ W: $list
        for my $smsg (@$need_reindex) {
                my $new_smsg = bless {
                        blob => $blob,
-                       bytes => $bytes,
+                       raw_bytes => $bytes,
                        num => $smsg->{num},
                        mid => $smsg->{mid},
                }, 'PublicInbox::Smsg';
+               my $v2w = { autime => $smsg->{ds}, cotime => $smsg->{ts} };
+               $new_smsg->populate($new_mime, $v2w);
                do_idx($self, \$raw, $new_mime, $new_smsg);
        }
        $rewritten->{rewrites};
@@ -963,11 +964,12 @@ sub reindex_oid_m ($$$$;$) {
        }
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1054,11 +1056,12 @@ sub reindex_oid ($$$$) {
                die "failed to delete <$mid0> for article #$num\n";
        $sync->{nr}++;
        my $smsg = bless {
-               bytes => $len,
+               raw_bytes => $len,
                num => $num,
                blob => $oid,
                mid => $mid0,
        }, 'PublicInbox::Smsg';
+       $smsg->populate($mime, $self);
        if (do_idx($self, $msgref, $mime, $smsg)) {
                reindex_checkpoint($self, $sync, $git);
        }
@@ -1298,7 +1301,7 @@ sub index_epoch ($$$) {
                }
        }
        close $fh or die "git log failed: \$?=$?";
-       delete $self->{reindex_pipe};
+       delete @$self{qw(reindex_pipe autime cotime)};
        update_last_commit($self, $git, $i, $cmt) if defined $cmt;
 }