🐛 fix(text_coordinates): convert LSP UTF-16 positions

lsp-server/src/manager/parser.cppm

@@ -1,6 +1,5 @@
 module;
 
-
 export module lsp.manager.parser;
 import tree_sitter;
 import spdlog;
@@ -123,15 +122,15 @@ namespace lsp::manager
         if (!tree_)
             return;
 
-        protocol::uinteger start_offset = utils::text_coordinates::ToOffset(change.range.start, content);
-        protocol::uinteger end_offset = utils::text_coordinates::ToOffset(change.range.end, content);
+        const auto start = utils::text_coordinates::ToBytePosition(change.range.start, content);
+        const auto old_end = utils::text_coordinates::ToBytePosition(change.range.end, content);
 
         TSInputEdit edit{};
-        edit.start_byte = start_offset;
-        edit.old_end_byte = end_offset;
-        edit.new_end_byte = start_offset + change.text.length();
-        edit.start_point = utils::text_coordinates::ToPoint(change.range.start);
-        edit.old_end_point = utils::text_coordinates::ToPoint(change.range.end);
+        edit.start_byte = start.offset;
+        edit.old_end_byte = old_end.offset;
+        edit.new_end_byte = start.offset + change.text.length();
+        edit.start_point = start.point;
+        edit.old_end_point = old_end.point;
         edit.new_end_point = utils::text_coordinates::CalculateEndPoint(change.text, edit.start_point);
 
         ts_tree_edit(tree_.get(), &edit);

lsp-server/src/utils/text_coordinates.cppm

@@ -9,59 +9,122 @@ import lsp.protocol;
 
 export namespace lsp::utils::text_coordinates
 {
+    struct BytePosition
+    {
+        protocol::uinteger offset;
+        TSPoint point;
+    };
+
+    BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content);
     protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content);
-    TSPoint ToPoint(const protocol::Position& position);
     TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start);
 }
 
 namespace lsp::utils::text_coordinates
 {
-    protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
+    namespace
     {
-        protocol::uinteger offset = 0;
-        protocol::uinteger current_line = 0;
-
-        while (offset < content.length() && current_line < position.line)
+        struct DecodedCharacter
         {
-            if (content[offset] == '\n')
-                current_line++;
-            offset++;
-        }
+            std::size_t byte_count;
+            protocol::uinteger utf16_units;
+        };
 
-        if (offset >= content.length())
-            return content.length();
-
-        protocol::uinteger current_char = 0;
-        while (offset < content.length() && current_char < position.character)
+        DecodedCharacter DecodeCharacter(std::string_view content, std::size_t offset)
         {
-            if (content[offset] == '\n')
-                break;
+            const auto lead = static_cast<unsigned char>(content[offset]);
+            if ((lead & 0x80U) == 0)
+                return { 1, 1 };
 
-            unsigned char ch = static_cast<unsigned char>(content[offset]);
-            if ((ch & 0x80) == 0)
-                offset += 1;
-            else if ((ch & 0xE0) == 0xC0)
-                offset += 2;
-            else if ((ch & 0xF0) == 0xE0)
-                offset += 3;
-            else if ((ch & 0xF8) == 0xF0)
-                offset += 4;
+            std::size_t byte_count = 0;
+            std::uint32_t code_point = 0;
+            std::uint32_t minimum = 0;
+            if ((lead & 0xE0U) == 0xC0U)
+            {
+                byte_count = 2;
+                code_point = lead & 0x1FU;
+                minimum = 0x80U;
+            }
+            else if ((lead & 0xF0U) == 0xE0U)
+            {
+                byte_count = 3;
+                code_point = lead & 0x0FU;
+                minimum = 0x800U;
+            }
+            else if ((lead & 0xF8U) == 0xF0U)
+            {
+                byte_count = 4;
+                code_point = lead & 0x07U;
+                minimum = 0x10000U;
+            }
             else
-                offset += 1;
-            current_char++;
-        }
+            {
+                return { 1, 1 };
+            }
 
-        return std::min(offset, static_cast<protocol::uinteger>(content.length()));
+            if (byte_count > content.size() - offset)
+                return { 1, 1 };
+
+            for (std::size_t index = 1; index < byte_count; ++index)
+            {
+                const auto continuation = static_cast<unsigned char>(content[offset + index]);
+                if ((continuation & 0xC0U) != 0x80U)
+                    return { 1, 1 };
+                code_point = (code_point << 6U) | (continuation & 0x3FU);
+            }
+
+            if (code_point < minimum || code_point > 0x10FFFFU ||
+                (code_point >= 0xD800U && code_point <= 0xDFFFU))
+            {
+                return { 1, 1 };
+            }
+
+            return { byte_count, code_point >= 0x10000U ? 2U : 1U };
+        }
     }
 
-    TSPoint ToPoint(const protocol::Position& position)
+    BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content)
     {
-        return TSPoint{
-            .row = static_cast<uint32_t>(position.line),
-            .column = static_cast<uint32_t>(position.character)
+        std::size_t offset = 0;
+        std::size_t line_start = 0;
+        std::uint32_t row = 0;
+
+        while (offset < content.size() && row < position.line)
+        {
+            if (content[offset++] == '\n')
+            {
+                ++row;
+                line_start = offset;
+            }
+        }
+
+        protocol::uinteger utf16_units = 0;
+        while (offset < content.size() && content[offset] != '\n' &&
+               utf16_units < position.character)
+        {
+            const auto decoded = DecodeCharacter(content, offset);
+            const auto remaining = position.character - utf16_units;
+            if (decoded.utf16_units > remaining)
+                break;
+
+            offset += decoded.byte_count;
+            utf16_units += decoded.utf16_units;
+        }
+
+        return {
+            .offset = static_cast<protocol::uinteger>(offset),
+            .point = TSPoint{
+                .row = row,
+                .column = static_cast<std::uint32_t>(offset - line_start),
+            },
         };
     }
 
+    protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
+    {
+        return ToBytePosition(position, content).offset;
+    }
+
     TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start)
     {
         TSPoint end = start;