From 11fe4b66ba5cdc8fc8c6d708e505ef6e38ad7703 Mon Sep 17 00:00:00 2001 From: Yegappan Lakshmanan Date: Sun, 4 Jun 2023 23:39:58 -0700 Subject: [PATCH] Properly support multibyte characters with composing characters --- autoload/lsp/capabilities.vim | 1 + autoload/lsp/completion.vim | 5 +- autoload/lsp/diag.vim | 27 +-- autoload/lsp/inlayhints.vim | 11 +- autoload/lsp/lspserver.vim | 4 +- autoload/lsp/symbol.vim | 17 +- autoload/lsp/textedit.vim | 8 +- autoload/lsp/util.vim | 87 ++++++++-- test/clangd_tests.vim | 311 ++++++++++++++++++++++++++++++++++ 9 files changed, 426 insertions(+), 45 deletions(-) diff --git a/autoload/lsp/capabilities.vim b/autoload/lsp/capabilities.vim index 2e2db54..28a1215 100644 --- a/autoload/lsp/capabilities.vim +++ b/autoload/lsp/capabilities.vim @@ -5,6 +5,7 @@ vim9script import './options.vim' as opt # Process the server capabilities +# interface ServerCapabilities export def ProcessServerCaps(lspserver: dict, caps: dict) # completionProvider if lspserver.caps->has_key('completionProvider') diff --git a/autoload/lsp/completion.vim b/autoload/lsp/completion.vim index f935119..5e7710d 100644 --- a/autoload/lsp/completion.vim +++ b/autoload/lsp/completion.vim @@ -264,7 +264,8 @@ export def CompletionReply(lspserver: dict, cItems: any) start_charcol = chcol endif var textEdit = item.textEdit - var textEditStartCol = textEdit.range.start.character + var textEditStartCol = + util.GetCharIdxWithoutCompChar(bufnr(), textEdit.range.start) if textEditStartCol != start_charcol var offset = start_charcol - textEditStartCol - 1 d.word = textEdit.newText[offset : ] @@ -532,7 +533,7 @@ def g:LspOmniFunc(findstart: number, base: string): any endif if opt.lspOptions.completionMatcher == 'icase' - return res->filter((i, v) => v.word->tolower()->stridx(prefix) == 0) + return res->filter((i, v) => v.word->tolower()->stridx(prefix->tolower()) == 0) endif return res->filter((i, v) => v.word->stridx(prefix) == 0) diff --git a/autoload/lsp/diag.vim b/autoload/lsp/diag.vim index ec3f971..e3f8fa5 100644 --- a/autoload/lsp/diag.vim +++ b/autoload/lsp/diag.vim @@ -213,9 +213,10 @@ def DiagsRefresh(bnr: number) padding = 3 symbol = DiagSevToSymbolText(diag.severity) else - padding = diag.range.start.character + var charIdx = util.GetCharIdxWithoutCompChar(bnr, diag.range.start) + padding = charIdx if padding > 0 - padding = strdisplaywidth(getline(diag.range.start.line + 1)[ : diag.range.start.character - 1]) + padding = strdisplaywidth(getline(diag.range.start.line + 1)[ : charIdx - 1]) endif endif @@ -449,7 +450,7 @@ enddef def ShowDiagInPopup(diag: dict) var dlnum = diag.range.start.line + 1 var ltext = dlnum->getline() - var dlcol = ltext->byteidx(diag.range.start.character + 1) + var dlcol = ltext->byteidxcomp(diag.range.start.character) + 1 var lastline = line('$') if dlnum > lastline @@ -538,11 +539,13 @@ export def GetDiagByPos(bnr: number, lnum: number, col: number, var diags_in_line = GetDiagsByLine(bnr, lnum) for diag in diags_in_line + var startCharIdx = util.GetCharIdxWithoutCompChar(bnr, diag.range.start) + var endCharIdx = util.GetCharIdxWithoutCompChar(bnr, diag.range.end) if atPos - if col >= diag.range.start.character + 1 && col < diag.range.end.character + 1 + if col >= startCharIdx + 1 && col < endCharIdx + 1 return diag endif - elseif col <= diag.range.start.character + 1 + elseif col <= startCharIdx + 1 return diag endif endfor @@ -587,11 +590,13 @@ enddef # Utility function to do the actual jump def JumpDiag(diag: dict) - setcursorcharpos(diag.range.start.line + 1, diag.range.start.character + 1) - if !opt.lspOptions.showDiagWithVirtualText - :redraw - DisplayDiag(diag) - endif + var startPos: dict = diag.range.start + setcursorcharpos(startPos.line + 1, + util.GetCharIdxWithoutCompChar(bufnr(), startPos) + 1) + if !opt.lspOptions.showDiagWithVirtualText + :redraw + DisplayDiag(diag) + endif enddef # jump to the next/previous/first diagnostic message in the current buffer @@ -627,7 +632,7 @@ export def LspDiagsJump(which: string, a_count: number = 0): void for diag in (which == 'next' || which == 'here') ? diags : diags->copy()->reverse() var lnum = diag.range.start.line + 1 - var col = diag.range.start.character + 1 + var col = util.GetCharIdxWithoutCompChar(bnr, diag.range.start) + 1 if (which == 'next' && (lnum > curlnum || lnum == curlnum && col > curcol)) || (which == 'prev' && (lnum < curlnum || lnum == curlnum && col < curcol)) diff --git a/autoload/lsp/inlayhints.vim b/autoload/lsp/inlayhints.vim index b03e1b7..07c0a1b 100644 --- a/autoload/lsp/inlayhints.vim +++ b/autoload/lsp/inlayhints.vim @@ -34,7 +34,7 @@ export def InlayHintsReply(lspserver: dict, inlayHints: any) return endif - var bufnum = bufnr('%') + var bnr = bufnr('%') for hint in inlayHints var label = '' if hint.label->type() == v:t_list @@ -45,12 +45,13 @@ export def InlayHintsReply(lspserver: dict, inlayHints: any) var kind = hint->has_key('kind') ? hint.kind->string() : '1' try + var byteIdx = util.GetLineByteFromPos(bnr, hint.position) if kind == "'type'" || kind == '1' - prop_add(hint.position.line + 1, hint.position.character + 1, - {type: 'LspInlayHintsType', text: label, bufnr: bufnum}) + prop_add(hint.position.line + 1, byteIdx + 1, + {type: 'LspInlayHintsType', text: label, bufnr: bnr}) elseif kind == "'parameter'" || kind == '2' - prop_add(hint.position.line + 1, hint.position.character + 1, - {type: 'LspInlayHintsParam', text: label, bufnr: bufnum}) + prop_add(hint.position.line + 1, byteIdx + 1, + {type: 'LspInlayHintsParam', text: label, bufnr: bnr}) endif catch /E966\|E964/ # Invalid lnum | Invalid col # Inlay hints replies arrive asynchronously and the document might have diff --git a/autoload/lsp/lspserver.vim b/autoload/lsp/lspserver.vim index f6fa2b6..201c097 100644 --- a/autoload/lsp/lspserver.vim +++ b/autoload/lsp/lspserver.vim @@ -597,7 +597,9 @@ def GetLspPosition(find_ident: bool): dict endwhile endif - return {line: lnum, character: col} + # Compute character index counting composing characters as separate + # characters + return {line: lnum, character: util.GetCharIdxWithCompChar(line, col)} enddef # Return the current file name and current cursor position as a LSP diff --git a/autoload/lsp/symbol.vim b/autoload/lsp/symbol.vim index 0cf0edb..e7ac232 100644 --- a/autoload/lsp/symbol.vim +++ b/autoload/lsp/symbol.vim @@ -116,7 +116,8 @@ def JumpToWorkspaceSymbol(popupID: number, result: number): void # used so that the current location is added to the jump list. :normal m' setcursorcharpos(symTbl[result - 1].pos.line + 1, - symTbl[result - 1].pos.character + 1) + util.GetCharIdxWithoutCompChar(bufnum, + symTbl[result - 1].pos) + 1) catch # ignore exceptions endtry @@ -345,9 +346,7 @@ def PeekLocations(lspserver: dict, locations: list>, if bnr == -1 bnr = fname->bufadd() endif - if !bnr->bufloaded() - bnr->bufload() - endif + bnr->bufload() var lnum = range.start.line + 1 var text: string = util.GetBufOneLine(bnr, lnum) @@ -389,9 +388,7 @@ export def ShowLocations(lspserver: dict, locations: list>, if bnr == -1 bnr = fname->bufadd() endif - if !bnr->bufloaded() - bnr->bufload() - endif + bnr->bufload() var text: string = util.GetBufOneLine(bnr, range.start.line + 1)->trim("\t ", 1) qflist->add({filename: fname, lnum: range.start.line + 1, @@ -504,8 +501,10 @@ export def TagFunc(lspserver: dict, tagitem.name = pat var [uri, range] = util.LspLocationParse(tagloc) - tagitem.filename = util.LspUriToFile(uri) - tagitem.cmd = $"/\\%{range.start.line + 1}l\\%{range.start.character + 1}c" + var fname = util.LspUriToFile(uri) + tagitem.filename = fname + var startByteIdx = util.GetLineByteFromPos(fname->bufnr(), range.start) + tagitem.cmd = $"/\\%{range.start.line + 1}l\\%{startByteIdx + 1}c" retval->add(tagitem) endfor diff --git a/autoload/lsp/textedit.vim b/autoload/lsp/textedit.vim index 92cd742..8471ed0 100644 --- a/autoload/lsp/textedit.vim +++ b/autoload/lsp/textedit.vim @@ -104,9 +104,7 @@ export def ApplyTextEdits(bnr: number, text_edits: list>): void endif # if the buffer is not loaded, load it and make it a listed buffer - if !bnr->bufloaded() - bnr->bufload() - endif + bnr->bufload() setbufvar(bnr, '&buflisted', true) var start_line: number = 4294967295 # 2 ^ 32 @@ -122,9 +120,9 @@ export def ApplyTextEdits(bnr: number, text_edits: list>): void for e in text_edits # Adjust the start and end columns for multibyte characters start_row = e.range.start.line - start_col = util.GetLineByteFromPos(bnr, e.range.start) + start_col = util.GetCharIdxWithoutCompChar(bnr, e.range.start) end_row = e.range.end.line - end_col = util.GetLineByteFromPos(bnr, e.range.end) + end_col = util.GetCharIdxWithoutCompChar(bnr, e.range.end) start_line = [e.range.start.line, start_line]->min() finish_line = [e.range.end.line, finish_line]->max() diff --git a/autoload/lsp/util.vim b/autoload/lsp/util.vim index 2000ba2..c956ee8 100644 --- a/autoload/lsp/util.vim +++ b/autoload/lsp/util.vim @@ -162,24 +162,86 @@ export def GetLineByteFromPos(bnr: number, pos: dict): number var col: number = pos.character # When on the first character, we can ignore the difference between byte and # character - if col > 0 - # Need a loaded buffer to read the line and compute the offset - if !bnr->bufloaded() - bnr->bufload() - endif + if col <= 0 + return col + endif - var ltext: list = bnr->getbufline(pos.line + 1) - if !ltext->empty() - var bidx = ltext[0]->byteidx(col) - if bidx != -1 - return bidx - endif + # Need a loaded buffer to read the line and compute the offset + bnr->bufload() + + var ltext: string = GetBufOneLine(bnr, pos.line + 1) + if ltext->empty() + return col + endif + + var byteIdx = ltext->byteidxcomp(col) + if byteIdx != -1 + return byteIdx + endif + + return col +enddef + +# Get the index of the character at [pos.line, pos.character] in buffer "bnr" +# without counting the composing characters. The LSP server counts composing +# characters as separate characters whereas Vim string indexing ignores the +# composing characters. +export def GetCharIdxWithoutCompChar(bnr: number, pos: dict): number + var col: number = pos.character + # When on the first character, nothing to do. + if col <= 0 + return col + endif + + # Need a loaded buffer to read the line and compute the offset + bnr->bufload() + + var ltext: string = GetBufOneLine(bnr, pos.line + 1) + if ltext->empty() + return col + endif + + # Convert the character index that includes composing characters as separate + # characters to a byte index and then back to a character index ignoring the + # composing characters. + var byteIdx = ltext->byteidxcomp(col) + if byteIdx != -1 + if byteIdx == ltext->strlen() + # Byte index points to the byte after the last byte. + return ltext->strcharlen() + else + return ltext->charidx(byteIdx, v:false) endif endif return col enddef +# Get the index of the character at [pos.line, pos.character] in buffer "bnr" +# counting the composing characters as separate characters. The LSP server +# counts composing characters as separate characters whereas Vim string +# indexing ignores the composing characters. +export def GetCharIdxWithCompChar(ltext: string, charIdx: number): number + # When on the first character, nothing to do. + if charIdx <= 0 || ltext->empty() + return charIdx + endif + + # Convert the character index that doesn't include composing characters as + # separate characters to a byte index and then back to a character index + # that includes the composing characters as separate characters + var byteIdx = ltext->byteidx(charIdx) + if byteIdx != -1 + if byteIdx == ltext->strlen() + return ltext->strchars() + else + return ltext->charidx(byteIdx, v:true) + endif + endif + + return charIdx +enddef + # push the current location on to the tag stack export def PushCursorToTagStack() settagstack(winnr(), {items: [ @@ -234,7 +296,8 @@ export def JumpToLspLocation(location: dict, cmdmods: string) else exe $'{cmdmods} split {fname}' endif - setcursorcharpos(range.start.line + 1, range.start.character + 1) + setcursorcharpos(range.start.line + 1, + GetCharIdxWithoutCompChar(bufnr(), range.start) + 1) enddef # 'indexof' is to new to use it, use this instead. diff --git a/test/clangd_tests.vim b/test/clangd_tests.vim index 2d58fc7..dcd418c 100644 --- a/test/clangd_tests.vim +++ b/test/clangd_tests.vim @@ -131,6 +131,33 @@ def g:Test_LspFormat() :%bw! enddef +# Test for :LspFormat when using composing characters +def g:Test_LspFormat_ComposingChars() + :silent! edit Xtest.c + sleep 200m + var lines =<< trim END + void fn(int aVar) + { + int 😊😊😊😊 = aVar + 1; + int áb́áb́ = aVar + 1; + int ą́ą́ą́ą́ = aVar + 1; + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + :redraw! + :LspFormat + var expected =<< trim END + void fn(int aVar) { + int 😊😊😊😊 = aVar + 1; + int áb́áb́ = aVar + 1; + int ą́ą́ą́ą́ = aVar + 1; + } + END + assert_equal(expected, getline(1, '$')) + :%bw! +enddef + # Test for formatting a file using 'formatexpr' def g:Test_LspFormatExpr() :silent! edit Xtest.c @@ -256,6 +283,36 @@ def g:Test_LspShowReferences() :%bw! enddef +# Test for :LspShowReferences when using composing characters +def g:Test_LspShowReferences_ComposingChars() + :silent! edit Xtest.c + sleep 200m + var lines: list =<< trim END + #include + void fn(int aVar) + { + printf("aVar = %d\n", aVar); + printf("😊😊😊😊 = %d\n", aVar); + printf("áb́áb́ = %d\n", aVar); + printf("ą́ą́ą́ą́ = %d\n", aVar); + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + redraw! + cursor(4, 27) + :LspShowReferences + var qfl: list> = getloclist(0) + assert_equal([2, 13], [qfl[0].lnum, qfl[0].col]) + assert_equal([4, 27], [qfl[1].lnum, qfl[1].col]) + assert_equal([5, 39], [qfl[2].lnum, qfl[2].col]) + assert_equal([6, 35], [qfl[3].lnum, qfl[3].col]) + assert_equal([7, 43], [qfl[4].lnum, qfl[4].col]) + :lclose + + :%bw! +enddef + # Test for LSP diagnostics def g:Test_LspDiag() :silent! edit Xtest.c @@ -340,6 +397,32 @@ def g:Test_LspDiag() :%bw! enddef +# Test for :LspDiagShow when using composing characters +def g:Test_LspDiagShow_ComposingChars() + :silent! edit Xtest.c + sleep 200m + var lines =<< trim END + #include + void fn(int aVar) + { + printf("aVar = %d\n", aVar); + printf("😊😊😊😊 = %d\n". aVar); + printf("áb́áb́ = %d\n". aVar); + printf("ą́ą́ą́ą́ = %d\n". aVar); + } + END + setline(1, lines) + g:WaitForServerFileLoad(3) + :redraw! + :LspDiagShow + var qfl: list> = getloclist(0) + assert_equal([5, 37], [qfl[0].lnum, qfl[0].col]) + assert_equal([6, 33], [qfl[1].lnum, qfl[1].col]) + assert_equal([7, 41], [qfl[2].lnum, qfl[2].col]) + :lclose + :%bw! +enddef + # Test for LSP diagnostics handler def g:Test_LspProcessDiagHandler() g:LSPTest_modifyDiags = true @@ -675,6 +758,39 @@ def g:Test_LspCodeAction() :%bw! enddef +# Test for :LspCodeAction with symbols containing composing characters +def g:Test_LspCodeAction_ComposingChars() + silent! edit Xtest.c + sleep 200m + var lines =<< trim END + #include + void fn(int aVar) + { + printf("aVar = %d\n", aVar); + printf("😊😊😊😊 = %d\n", aVar): + printf("áb́áb́ = %d\n", aVar): + printf("ą́ą́ą́ą́ = %d\n", aVar): + } + END + setline(1, lines) + g:WaitForServerFileLoad(3) + :redraw! + cursor(5, 5) + redraw! + :LspCodeAction 1 + assert_equal(' printf("😊😊😊😊 = %d\n", aVar);', getline(5)) + cursor(6, 5) + redraw! + :LspCodeAction 1 + assert_equal(' printf("áb́áb́ = %d\n", aVar);', getline(6)) + cursor(7, 5) + redraw! + :LspCodeAction 1 + assert_equal(' printf("ą́ą́ą́ą́ = %d\n", aVar);', getline(7)) + + :bw! +enddef + # Test for :LspRename def g:Test_LspRename() silent! edit Xtest.c @@ -745,6 +861,40 @@ def g:Test_LspRename() :%bw! enddef +# Test for :LspRename with composing characters +def g:Test_LspRename_ComposingChars() + silent! edit Xtest.c + sleep 200m + var lines: list =<< trim END + #include + void fn(int aVar) + { + printf("aVar = %d\n", aVar); + printf("😊😊😊😊 = %d\n", aVar); + printf("áb́áb́ = %d\n", aVar); + printf("ą́ą́ą́ą́ = %d\n", aVar); + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + redraw! + cursor(2, 12) + :LspRename bVar + redraw! + var expected: list =<< trim END + #include + void fn(int bVar) + { + printf("aVar = %d\n", bVar); + printf("😊😊😊😊 = %d\n", bVar); + printf("áb́áb́ = %d\n", bVar); + printf("ą́ą́ą́ą́ = %d\n", bVar); + } + END + assert_equal(expected, getline(1, '$')) + :%bw! +enddef + # Test for :LspSelectionExpand and :LspSelectionShrink def g:Test_LspSelection() silent! edit Xtest.c @@ -982,6 +1132,64 @@ def g:Test_LspGotoSymbol() :%bw! enddef +# Test for :LspGotoDefinition when using composing characters +def g:Test_LspGotoDefinition_With_ComposingCharacters() + :silent! edit Xtest.c + sleep 200m + var lines: list =<< trim END + #include + void fn(int aVar) + { + printf("aVar = %d\n", aVar); + printf("😊😊😊😊 = %d\n", aVar); + printf("áb́áb́ = %d\n", aVar); + printf("ą́ą́ą́ą́ = %d\n", aVar); + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + redraw! + + for [lnum, colnr] in [[4, 27], [5, 39], [6, 35], [7, 43]] + cursor(lnum, colnr) + :LspGotoDefinition + assert_equal([2, 13], [line('.'), col('.')]) + endfor + + :%bw! +enddef + +# Test for :LspGotoDefinition when using composing characters +def g:Test_LspGotoDefinition_After_ComposingCharacters() + :silent! edit Xtest.c + sleep 200m + var lines =<< trim END + void fn(int aVar) + { + int 😊😊😊😊, bVar; + int áb́áb́, cVar; + int ą́ą́ą́ą́, dVar; + bVar = 10; + cVar = 10; + dVar = 10; + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + :redraw! + cursor(6, 5) + :LspGotoDefinition + assert_equal([3, 27], [line('.'), col('.')]) + cursor(7, 5) + :LspGotoDefinition + assert_equal([4, 23], [line('.'), col('.')]) + cursor(8, 5) + :LspGotoDefinition + assert_equal([5, 31], [line('.'), col('.')]) + + :%bw! +enddef + # Test for :LspHighlight def g:Test_LspHighlight() silent! edit Xtest.c @@ -1153,6 +1361,43 @@ def g:Test_LspSymbolSearch() :%bw! enddef +# Test for :LspSymbolSearch when using composing characters +def g:Test_LspSymbolSearch_ComposingChars() + silent! edit Xtest.c + sleep 200m + var lines: list =<< trim END + typedef void 😊😊😊😊; + typedef void áb́áb́; + typedef void ą́ą́ą́ą́; + + 😊😊😊😊 Func1() + { + } + + áb́áb́ Func2() + { + } + + ą́ą́ą́ą́ Func3() + { + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + + cursor(1, 1) + feedkeys(":LspSymbolSearch Func1\\", "xt") + assert_equal([5, 18], [line('.'), col('.')]) + cursor(1, 1) + feedkeys(":LspSymbolSearch Func2\\", "xt") + assert_equal([9, 14], [line('.'), col('.')]) + cursor(1, 1) + feedkeys(":LspSymbolSearch Func3\\", "xt") + assert_equal([13, 22], [line('.'), col('.')]) + + :%bw! +enddef + # Test for :LspIncomingCalls def g:Test_LspIncomingCalls() silent! edit Xtest.c @@ -1290,6 +1535,37 @@ def g:Test_LspTagFunc() delete('Xtest.c') enddef +# Test for setting the 'tagfunc' with composing characters in symbols +def g:Test_LspTagFunc_ComposingChars() + var lines =<< trim END + void fn(int aVar) + { + int 😊😊😊😊, bVar; + int áb́áb́, cVar; + int ą́ą́ą́ą́, dVar; + bVar = 10; + cVar = 10; + dVar = 10; + } + END + writefile(lines, 'Xtest.c', 'D') + :silent! edit! Xtest.c + g:WaitForServerFileLoad(0) + :setlocal tagfunc=lsp#lsp#TagFunc + cursor(6, 5) + :exe "normal \" + assert_equal([3, 27], [line('.'), col('.')]) + cursor(7, 5) + :exe "normal \" + assert_equal([4, 23], [line('.'), col('.')]) + cursor(8, 5) + :exe "normal \" + assert_equal([5, 31], [line('.'), col('.')]) + :set tagfunc& + + :%bw! +enddef + # Test for the LspDiagsUpdated autocmd def g:Test_LspDiagsUpdated_Autocmd() g:LspAutoCmd = 0 @@ -1448,6 +1724,41 @@ def g:Test_OmniComplete_Struct() :bw! enddef +# Test for doing omni completion for symbols with composing characters +def g:Test_OmniComplete_ComposingChars() + :silent! edit Xtest.c + sleep 200m + var lines: list =<< trim END + void Func1(void) + { + int 😊😊😊😊, aVar; + int áb́áb́, bVar; + int ą́ą́ą́ą́, cVar; + + + + } + END + setline(1, lines) + g:WaitForServerFileLoad(0) + redraw! + + cursor(6, 4) + feedkeys("aaV\\ = 😊😊\\;", 'xt') + assert_equal(' aVar = 😊😊😊😊;', getline('.')) + cursor(7, 4) + feedkeys("abV\\ = áb́\\;", 'xt') + assert_equal(' bVar = áb́áb́;', getline('.')) + cursor(8, 4) + feedkeys("acV\\ = ą́ą́\\;", 'xt') + assert_equal(' cVar = ą́ą́ą́ą́;', getline('.')) + feedkeys("oáb́\\ = ą́ą́\\;", 'xt') + assert_equal(' áb́áb́ = ą́ą́ą́ą́;', getline('.')) + feedkeys("oą́ą́\\ = áb́\\;", 'xt') + assert_equal(' ą́ą́ą́ą́ = áb́áb́;', getline('.')) + :%bw! +enddef + # Test for the :LspServer command. def g:Test_LspServer() new a.raku -- 2.48.1