diff --git a/lsp-server/src/manager/parser.cppm b/lsp-server/src/manager/parser.cppm index 054a8b5..38f5767 100644 --- a/lsp-server/src/manager/parser.cppm +++ b/lsp-server/src/manager/parser.cppm @@ -1,6 +1,5 @@ module; - export module lsp.manager.parser; import tree_sitter; import spdlog; @@ -123,15 +122,15 @@ namespace lsp::manager if (!tree_) return; - protocol::uinteger start_offset = utils::text_coordinates::ToOffset(change.range.start, content); - protocol::uinteger end_offset = utils::text_coordinates::ToOffset(change.range.end, content); + const auto start = utils::text_coordinates::ToBytePosition(change.range.start, content); + const auto old_end = utils::text_coordinates::ToBytePosition(change.range.end, content); TSInputEdit edit{}; - edit.start_byte = start_offset; - edit.old_end_byte = end_offset; - edit.new_end_byte = start_offset + change.text.length(); - edit.start_point = utils::text_coordinates::ToPoint(change.range.start); - edit.old_end_point = utils::text_coordinates::ToPoint(change.range.end); + edit.start_byte = start.offset; + edit.old_end_byte = old_end.offset; + edit.new_end_byte = start.offset + change.text.length(); + edit.start_point = start.point; + edit.old_end_point = old_end.point; edit.new_end_point = utils::text_coordinates::CalculateEndPoint(change.text, edit.start_point); ts_tree_edit(tree_.get(), &edit); diff --git a/lsp-server/src/utils/text_coordinates.cppm b/lsp-server/src/utils/text_coordinates.cppm index ba2454f..3c0efe6 100644 --- a/lsp-server/src/utils/text_coordinates.cppm +++ b/lsp-server/src/utils/text_coordinates.cppm @@ -9,59 +9,122 @@ import lsp.protocol; export namespace lsp::utils::text_coordinates { + struct BytePosition + { + protocol::uinteger offset; + TSPoint point; + }; + + BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content); protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content); - TSPoint ToPoint(const protocol::Position& position); TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start); } namespace lsp::utils::text_coordinates { - protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content) + namespace { - protocol::uinteger offset = 0; - protocol::uinteger current_line = 0; - - while (offset < content.length() && current_line < position.line) + struct DecodedCharacter { - if (content[offset] == '\n') - current_line++; - offset++; - } + std::size_t byte_count; + protocol::uinteger utf16_units; + }; - if (offset >= content.length()) - return content.length(); - - protocol::uinteger current_char = 0; - while (offset < content.length() && current_char < position.character) + DecodedCharacter DecodeCharacter(std::string_view content, std::size_t offset) { - if (content[offset] == '\n') - break; + const auto lead = static_cast(content[offset]); + if ((lead & 0x80U) == 0) + return { 1, 1 }; - unsigned char ch = static_cast(content[offset]); - if ((ch & 0x80) == 0) - offset += 1; - else if ((ch & 0xE0) == 0xC0) - offset += 2; - else if ((ch & 0xF0) == 0xE0) - offset += 3; - else if ((ch & 0xF8) == 0xF0) - offset += 4; + std::size_t byte_count = 0; + std::uint32_t code_point = 0; + std::uint32_t minimum = 0; + if ((lead & 0xE0U) == 0xC0U) + { + byte_count = 2; + code_point = lead & 0x1FU; + minimum = 0x80U; + } + else if ((lead & 0xF0U) == 0xE0U) + { + byte_count = 3; + code_point = lead & 0x0FU; + minimum = 0x800U; + } + else if ((lead & 0xF8U) == 0xF0U) + { + byte_count = 4; + code_point = lead & 0x07U; + minimum = 0x10000U; + } else - offset += 1; - current_char++; - } + { + return { 1, 1 }; + } - return std::min(offset, static_cast(content.length())); + if (byte_count > content.size() - offset) + return { 1, 1 }; + + for (std::size_t index = 1; index < byte_count; ++index) + { + const auto continuation = static_cast(content[offset + index]); + if ((continuation & 0xC0U) != 0x80U) + return { 1, 1 }; + code_point = (code_point << 6U) | (continuation & 0x3FU); + } + + if (code_point < minimum || code_point > 0x10FFFFU || + (code_point >= 0xD800U && code_point <= 0xDFFFU)) + { + return { 1, 1 }; + } + + return { byte_count, code_point >= 0x10000U ? 2U : 1U }; + } } - TSPoint ToPoint(const protocol::Position& position) + BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content) { - return TSPoint{ - .row = static_cast(position.line), - .column = static_cast(position.character) + std::size_t offset = 0; + std::size_t line_start = 0; + std::uint32_t row = 0; + + while (offset < content.size() && row < position.line) + { + if (content[offset++] == '\n') + { + ++row; + line_start = offset; + } + } + + protocol::uinteger utf16_units = 0; + while (offset < content.size() && content[offset] != '\n' && + utf16_units < position.character) + { + const auto decoded = DecodeCharacter(content, offset); + const auto remaining = position.character - utf16_units; + if (decoded.utf16_units > remaining) + break; + + offset += decoded.byte_count; + utf16_units += decoded.utf16_units; + } + + return { + .offset = static_cast(offset), + .point = TSPoint{ + .row = row, + .column = static_cast(offset - line_start), + }, }; } + protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content) + { + return ToBytePosition(position, content).offset; + } + TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start) { TSPoint end = start; diff --git a/lsp-server/test/test_provider/CMakeLists.txt b/lsp-server/test/test_provider/CMakeLists.txt index 8479989..15f2665 100644 --- a/lsp-server/test/test_provider/CMakeLists.txt +++ b/lsp-server/test/test_provider/CMakeLists.txt @@ -28,6 +28,7 @@ set(SOURCES interpreter_test.cppm provider_misc_test.cppm provider_surface_test.cppm + text_coordinates_test.cppm ../../src/tree-sitter/parser.c ../../src/tree-sitter/scanner.c) @@ -56,6 +57,7 @@ target_sources( ${CMAKE_CURRENT_SOURCE_DIR}/interpreter_test.cppm ${CMAKE_CURRENT_SOURCE_DIR}/provider_misc_test.cppm ${CMAKE_CURRENT_SOURCE_DIR}/provider_surface_test.cppm + ${CMAKE_CURRENT_SOURCE_DIR}/text_coordinates_test.cppm ../../src/bridge/glaze.cppm ../../src/bridge/spdlog.cppm ../../src/bridge/taskflow.cppm diff --git a/lsp-server/test/test_provider/test_main.cppm b/lsp-server/test/test_provider/test_main.cppm index 5f72a53..4d19e76 100644 --- a/lsp-server/test/test_provider/test_main.cppm +++ b/lsp-server/test/test_provider/test_main.cppm @@ -12,6 +12,7 @@ import lsp.test.provider.json_flow; import lsp.test.provider.json_provider_coverage; import lsp.test.provider.misc; import lsp.test.provider.surface; +import lsp.test.provider.text_coordinates; import lsp.test.provider.fixtures; export int Run(int argc, char** argv) @@ -59,6 +60,8 @@ export int Run(int argc, char** argv) lsp::test::provider::ProviderMiscTests::Register(runner); std::cout << " - Provider surface tests" << std::endl; lsp::test::provider::ProviderSurfaceTests::Register(runner); + std::cout << " - Text coordinate tests" << std::endl; + lsp::test::provider::TextCoordinatesTests::Register(runner); runner.runAllTests(); return runner.getFailedCount(); diff --git a/lsp-server/test/test_provider/text_coordinates_test.cppm b/lsp-server/test/test_provider/text_coordinates_test.cppm new file mode 100644 index 0000000..f3c737c --- /dev/null +++ b/lsp-server/test/test_provider/text_coordinates_test.cppm @@ -0,0 +1,96 @@ +module; + +export module lsp.test.provider.text_coordinates; + +import std; + +import lsp.protocol; +import lsp.test.framework; +import lsp.utils.text_coordinates; + +export namespace lsp::test::provider +{ + class TextCoordinatesTests + { + public: + static void Register(TestRunner& runner); + + private: + static TestResult TestAsciiAndBmpPositions(); + static TestResult TestSupplementaryPlanePositions(); + static TestResult TestMultilineAndClampedPositions(); + static TestResult TestMalformedUtf8IsBounded(); + static TestResult TestCalculateEndPointUsesByteColumns(); + }; +} + +namespace lsp::test::provider +{ + namespace + { + void ExpectPosition(const utils::text_coordinates::BytePosition& actual, + protocol::uinteger offset, + std::uint32_t row, + std::uint32_t column) + { + assertEqual(offset, actual.offset, "byte offset should match"); + assertEqual(row, actual.point.row, "Tree-sitter row should match"); + assertEqual(column, actual.point.column, "Tree-sitter byte column should match"); + } + } + + void TextCoordinatesTests::Register(TestRunner& runner) + { + runner.addTest("text coordinates convert ASCII and BMP positions", TestAsciiAndBmpPositions); + runner.addTest("text coordinates convert supplementary-plane positions", TestSupplementaryPlanePositions); + runner.addTest("text coordinates clamp multiline positions", TestMultilineAndClampedPositions); + runner.addTest("text coordinates bound malformed UTF-8", TestMalformedUtf8IsBounded); + runner.addTest("text coordinates calculate byte end points", TestCalculateEndPointUsesByteColumns); + } + + TestResult TextCoordinatesTests::TestAsciiAndBmpPositions() + { + const protocol::string content = "A中Z"; + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 0 }, content), 0U, 0U, 0U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 4U, 0U, 4U); + assertEqual(4U, utils::text_coordinates::ToOffset({ 0, 2 }, content), "ToOffset should use the paired conversion"); + return { "", true, "ok" }; + } + + TestResult TextCoordinatesTests::TestSupplementaryPlanePositions() + { + const protocol::string content = "A😀Z"; + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 1U, 0U, 1U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 3 }, content), 5U, 0U, 5U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 4 }, content), 6U, 0U, 6U); + return { "", true, "ok" }; + } + + TestResult TextCoordinatesTests::TestMultilineAndClampedPositions() + { + const protocol::string content = "中x\n😀y"; + ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 0 }, content), 5U, 1U, 0U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 2 }, content), 9U, 1U, 4U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 100 }, content), 10U, 1U, 5U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 9, 0 }, content), 10U, 1U, 5U); + return { "", true, "ok" }; + } + + TestResult TextCoordinatesTests::TestMalformedUtf8IsBounded() + { + const protocol::string content{ 'A', static_cast(0xF0), static_cast(0x9F), 'Z' }; + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 2U, 0U, 2U); + ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 100 }, content), 4U, 0U, 4U); + return { "", true, "ok" }; + } + + TestResult TextCoordinatesTests::TestCalculateEndPointUsesByteColumns() + { + const auto end = utils::text_coordinates::CalculateEndPoint("中\n😀x", { 2U, 3U }); + assertEqual(3U, end.row, "newline should advance the Tree-sitter row"); + assertEqual(5U, end.column, "multibyte text should advance the byte column"); + return { "", true, "ok" }; + } +}