🐛 fix(text_coordinates): convert LSP UTF-16 positions

lsp-server/src/manager/parser.cppm

@@ -1,6 +1,5 @@
 module;
 
-
 export module lsp.manager.parser;
 import tree_sitter;
 import spdlog;
@@ -123,15 +122,15 @@ namespace lsp::manager
         if (!tree_)
             return;
 
-        protocol::uinteger start_offset = utils::text_coordinates::ToOffset(change.range.start, content);
+        const auto start = utils::text_coordinates::ToBytePosition(change.range.start, content);
-        protocol::uinteger end_offset = utils::text_coordinates::ToOffset(change.range.end, content);
+        const auto old_end = utils::text_coordinates::ToBytePosition(change.range.end, content);
 
         TSInputEdit edit{};
-        edit.start_byte = start_offset;
+        edit.start_byte = start.offset;
-        edit.old_end_byte = end_offset;
+        edit.old_end_byte = old_end.offset;
-        edit.new_end_byte = start_offset + change.text.length();
+        edit.new_end_byte = start.offset + change.text.length();
-        edit.start_point = utils::text_coordinates::ToPoint(change.range.start);
+        edit.start_point = start.point;
-        edit.old_end_point = utils::text_coordinates::ToPoint(change.range.end);
+        edit.old_end_point = old_end.point;
         edit.new_end_point = utils::text_coordinates::CalculateEndPoint(change.text, edit.start_point);
 
         ts_tree_edit(tree_.get(), &edit);

lsp-server/src/utils/text_coordinates.cppm

@@ -9,59 +9,122 @@ import lsp.protocol;
 
 export namespace lsp::utils::text_coordinates
 {
+    struct BytePosition
+    {
+        protocol::uinteger offset;
+        TSPoint point;
+    };
+
+    BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content);
     protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content);
-    TSPoint ToPoint(const protocol::Position& position);
     TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start);
 }
 
 namespace lsp::utils::text_coordinates
 {
-    protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
+    namespace
     {
-        protocol::uinteger offset = 0;
+        struct DecodedCharacter
-        protocol::uinteger current_line = 0;
+        {
+            std::size_t byte_count;
+            protocol::uinteger utf16_units;
+        };
 
-        while (offset < content.length() && current_line < position.line)
+        DecodedCharacter DecodeCharacter(std::string_view content, std::size_t offset)
         {
-            if (content[offset] == '\n')
+            const auto lead = static_cast<unsigned char>(content[offset]);
-                current_line++;
+            if ((lead & 0x80U) == 0)
-            offset++;
+                return { 1, 1 };
+
+            std::size_t byte_count = 0;
+            std::uint32_t code_point = 0;
+            std::uint32_t minimum = 0;
+            if ((lead & 0xE0U) == 0xC0U)
+            {
+                byte_count = 2;
+                code_point = lead & 0x1FU;
+                minimum = 0x80U;
+            }
+            else if ((lead & 0xF0U) == 0xE0U)
+            {
+                byte_count = 3;
+                code_point = lead & 0x0FU;
+                minimum = 0x800U;
+            }
+            else if ((lead & 0xF8U) == 0xF0U)
+            {
+                byte_count = 4;
+                code_point = lead & 0x07U;
+                minimum = 0x10000U;
+            }
+            else
+            {
+                return { 1, 1 };
             }
 
-        if (offset >= content.length())
+            if (byte_count > content.size() - offset)
-            return content.length();
+                return { 1, 1 };
 
-        protocol::uinteger current_char = 0;
+            for (std::size_t index = 1; index < byte_count; ++index)
-        while (offset < content.length() && current_char < position.character)
             {
-            if (content[offset] == '\n')
+                const auto continuation = static_cast<unsigned char>(content[offset + index]);
+                if ((continuation & 0xC0U) != 0x80U)
+                    return { 1, 1 };
+                code_point = (code_point << 6U) | (continuation & 0x3FU);
+            }
+
+            if (code_point < minimum || code_point > 0x10FFFFU ||
+                (code_point >= 0xD800U && code_point <= 0xDFFFU))
+            {
+                return { 1, 1 };
+            }
+
+            return { byte_count, code_point >= 0x10000U ? 2U : 1U };
+        }
+    }
+
+    BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content)
+    {
+        std::size_t offset = 0;
+        std::size_t line_start = 0;
+        std::uint32_t row = 0;
+
+        while (offset < content.size() && row < position.line)
+        {
+            if (content[offset++] == '\n')
+            {
+                ++row;
+                line_start = offset;
+            }
+        }
+
+        protocol::uinteger utf16_units = 0;
+        while (offset < content.size() && content[offset] != '\n' &&
+               utf16_units < position.character)
+        {
+            const auto decoded = DecodeCharacter(content, offset);
+            const auto remaining = position.character - utf16_units;
+            if (decoded.utf16_units > remaining)
                 break;
 
-            unsigned char ch = static_cast<unsigned char>(content[offset]);
+            offset += decoded.byte_count;
-            if ((ch & 0x80) == 0)
+            utf16_units += decoded.utf16_units;
-                offset += 1;
-            else if ((ch & 0xE0) == 0xC0)
-                offset += 2;
-            else if ((ch & 0xF0) == 0xE0)
-                offset += 3;
-            else if ((ch & 0xF8) == 0xF0)
-                offset += 4;
-            else
-                offset += 1;
-            current_char++;
         }
 
-        return std::min(offset, static_cast<protocol::uinteger>(content.length()));
+        return {
-    }
+            .offset = static_cast<protocol::uinteger>(offset),
-
+            .point = TSPoint{
-    TSPoint ToPoint(const protocol::Position& position)
+                .row = row,
-    {
+                .column = static_cast<std::uint32_t>(offset - line_start),
-        return TSPoint{
+            },
-            .row = static_cast<uint32_t>(position.line),
-            .column = static_cast<uint32_t>(position.character)
         };
     }
 
+    protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
+    {
+        return ToBytePosition(position, content).offset;
+    }
+
     TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start)
     {
         TSPoint end = start;

lsp-server/test/test_provider/CMakeLists.txt

@@ -28,6 +28,7 @@ set(SOURCES
     interpreter_test.cppm
     provider_misc_test.cppm
     provider_surface_test.cppm
+    text_coordinates_test.cppm
     ../../src/tree-sitter/parser.c
     ../../src/tree-sitter/scanner.c)
 
@@ -56,6 +57,7 @@ target_sources(
                              ${CMAKE_CURRENT_SOURCE_DIR}/interpreter_test.cppm
                              ${CMAKE_CURRENT_SOURCE_DIR}/provider_misc_test.cppm
                              ${CMAKE_CURRENT_SOURCE_DIR}/provider_surface_test.cppm
+                             ${CMAKE_CURRENT_SOURCE_DIR}/text_coordinates_test.cppm
                              ../../src/bridge/glaze.cppm
                              ../../src/bridge/spdlog.cppm
                              ../../src/bridge/taskflow.cppm

lsp-server/test/test_provider/test_main.cppm

@@ -12,6 +12,7 @@ import lsp.test.provider.json_flow;
 import lsp.test.provider.json_provider_coverage;
 import lsp.test.provider.misc;
 import lsp.test.provider.surface;
+import lsp.test.provider.text_coordinates;
 import lsp.test.provider.fixtures;
 
 export int Run(int argc, char** argv)
@@ -59,6 +60,8 @@ export int Run(int argc, char** argv)
     lsp::test::provider::ProviderMiscTests::Register(runner);
     std::cout << "  - Provider surface tests" << std::endl;
     lsp::test::provider::ProviderSurfaceTests::Register(runner);
+    std::cout << "  - Text coordinate tests" << std::endl;
+    lsp::test::provider::TextCoordinatesTests::Register(runner);
 
     runner.runAllTests();
     return runner.getFailedCount();

lsp-server/test/test_provider/text_coordinates_test.cppm

@@ -0,0 +1,96 @@
+module;
+
+export module lsp.test.provider.text_coordinates;
+
+import std;
+
+import lsp.protocol;
+import lsp.test.framework;
+import lsp.utils.text_coordinates;
+
+export namespace lsp::test::provider
+{
+    class TextCoordinatesTests
+    {
+    public:
+        static void Register(TestRunner& runner);
+
+    private:
+        static TestResult TestAsciiAndBmpPositions();
+        static TestResult TestSupplementaryPlanePositions();
+        static TestResult TestMultilineAndClampedPositions();
+        static TestResult TestMalformedUtf8IsBounded();
+        static TestResult TestCalculateEndPointUsesByteColumns();
+    };
+}
+
+namespace lsp::test::provider
+{
+    namespace
+    {
+        void ExpectPosition(const utils::text_coordinates::BytePosition& actual,
+                            protocol::uinteger offset,
+                            std::uint32_t row,
+                            std::uint32_t column)
+        {
+            assertEqual(offset, actual.offset, "byte offset should match");
+            assertEqual(row, actual.point.row, "Tree-sitter row should match");
+            assertEqual(column, actual.point.column, "Tree-sitter byte column should match");
+        }
+    }
+
+    void TextCoordinatesTests::Register(TestRunner& runner)
+    {
+        runner.addTest("text coordinates convert ASCII and BMP positions", TestAsciiAndBmpPositions);
+        runner.addTest("text coordinates convert supplementary-plane positions", TestSupplementaryPlanePositions);
+        runner.addTest("text coordinates clamp multiline positions", TestMultilineAndClampedPositions);
+        runner.addTest("text coordinates bound malformed UTF-8", TestMalformedUtf8IsBounded);
+        runner.addTest("text coordinates calculate byte end points", TestCalculateEndPointUsesByteColumns);
+    }
+
+    TestResult TextCoordinatesTests::TestAsciiAndBmpPositions()
+    {
+        const protocol::string content = "A中Z";
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 0 }, content), 0U, 0U, 0U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 4U, 0U, 4U);
+        assertEqual(4U, utils::text_coordinates::ToOffset({ 0, 2 }, content), "ToOffset should use the paired conversion");
+        return { "", true, "ok" };
+    }
+
+    TestResult TextCoordinatesTests::TestSupplementaryPlanePositions()
+    {
+        const protocol::string content = "A😀Z";
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 1U, 0U, 1U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 3 }, content), 5U, 0U, 5U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 4 }, content), 6U, 0U, 6U);
+        return { "", true, "ok" };
+    }
+
+    TestResult TextCoordinatesTests::TestMultilineAndClampedPositions()
+    {
+        const protocol::string content = "中x\n😀y";
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 0 }, content), 5U, 1U, 0U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 2 }, content), 9U, 1U, 4U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 100 }, content), 10U, 1U, 5U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 9, 0 }, content), 10U, 1U, 5U);
+        return { "", true, "ok" };
+    }
+
+    TestResult TextCoordinatesTests::TestMalformedUtf8IsBounded()
+    {
+        const protocol::string content{ 'A', static_cast<char>(0xF0), static_cast<char>(0x9F), 'Z' };
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 2U, 0U, 2U);
+        ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 100 }, content), 4U, 0U, 4U);
+        return { "", true, "ok" };
+    }
+
+    TestResult TextCoordinatesTests::TestCalculateEndPointUsesByteColumns()
+    {
+        const auto end = utils::text_coordinates::CalculateEndPoint("中\n😀x", { 2U, 3U });
+        assertEqual(3U, end.row, "newline should advance the Tree-sitter row");
+        assertEqual(5U, end.column, "multibyte text should advance the byte column");
+        return { "", true, "ok" };
+    }
+}