csh/tsl-devkit
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛 fix(text_coordinates): convert LSP UTF-16 positions

Browse Source
This commit is contained in:
csh
2026-07-12 12:15:46 +08:00
parent 16986281a7
commit 61d16d0147
5 changed files with 205 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lsp-server/src/manager/parser.cppm
+7 -8
View File
@@ -1,6 +1,5 @@
module;
export module lsp.manager.parser;
import tree_sitter;
import spdlog;
@@ -123,15 +122,15 @@ namespace lsp::manager
if (!tree_)
return;
protocol::uinteger start_offset = utils::text_coordinates::ToOffset(change.range.start, content);
protocol::uinteger end_offset = utils::text_coordinates::ToOffset(change.range.end, content);
const auto start = utils::text_coordinates::ToBytePosition(change.range.start, content);
const auto old_end = utils::text_coordinates::ToBytePosition(change.range.end, content);
TSInputEdit edit{};
edit.start_byte = start_offset;
edit.old_end_byte = end_offset;
edit.new_end_byte = start_offset + change.text.length();
edit.start_point = utils::text_coordinates::ToPoint(change.range.start);
edit.old_end_point = utils::text_coordinates::ToPoint(change.range.end);
edit.start_byte = start.offset;
edit.old_end_byte = old_end.offset;
edit.new_end_byte = start.offset + change.text.length();
edit.start_point = start.point;
edit.old_end_point = old_end.point;
edit.new_end_point = utils::text_coordinates::CalculateEndPoint(change.text, edit.start_point);
ts_tree_edit(tree_.get(), &edit);
lsp-server/src/utils/text_coordinates.cppm
+97 -34
View File
@@ -9,59 +9,122 @@ import lsp.protocol;
export namespace lsp::utils::text_coordinates
{
struct BytePosition
{
protocol::uinteger offset;
TSPoint point;
};
BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content);
protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content);
TSPoint ToPoint(const protocol::Position& position);
TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start);
}
namespace lsp::utils::text_coordinates
{
protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
namespace
{
protocol::uinteger offset = 0;
protocol::uinteger current_line = 0;
while (offset < content.length() && current_line < position.line)
struct DecodedCharacter
{
if (content[offset] == '\n')
current_line++;
offset++;
}
std::size_t byte_count;
protocol::uinteger utf16_units;
};
if (offset >= content.length())
return content.length();
protocol::uinteger current_char = 0;
while (offset < content.length() && current_char < position.character)
DecodedCharacter DecodeCharacter(std::string_view content, std::size_t offset)
{
if (content[offset] == '\n')
break;
const auto lead = static_cast<unsigned char>(content[offset]);
if ((lead & 0x80U) == 0)
return { 1, 1 };
unsigned char ch = static_cast<unsigned char>(content[offset]);
if ((ch & 0x80) == 0)
offset += 1;
else if ((ch & 0xE0) == 0xC0)
offset += 2;
else if ((ch & 0xF0) == 0xE0)
offset += 3;
else if ((ch & 0xF8) == 0xF0)
offset += 4;
std::size_t byte_count = 0;
std::uint32_t code_point = 0;
std::uint32_t minimum = 0;
if ((lead & 0xE0U) == 0xC0U)
{
byte_count = 2;
code_point = lead & 0x1FU;
minimum = 0x80U;
}
else if ((lead & 0xF0U) == 0xE0U)
{
byte_count = 3;
code_point = lead & 0x0FU;
minimum = 0x800U;
}
else if ((lead & 0xF8U) == 0xF0U)
{
byte_count = 4;
code_point = lead & 0x07U;
minimum = 0x10000U;
}
else
offset += 1;
current_char++;
}
{
return { 1, 1 };
}
return std::min(offset, static_cast<protocol::uinteger>(content.length()));
if (byte_count > content.size() - offset)
return { 1, 1 };
for (std::size_t index = 1; index < byte_count; ++index)
{
const auto continuation = static_cast<unsigned char>(content[offset + index]);
if ((continuation & 0xC0U) != 0x80U)
return { 1, 1 };
code_point = (code_point << 6U) | (continuation & 0x3FU);
}
if (code_point < minimum || code_point > 0x10FFFFU ||
(code_point >= 0xD800U && code_point <= 0xDFFFU))
{
return { 1, 1 };
}
return { byte_count, code_point >= 0x10000U ? 2U : 1U };
}
}
TSPoint ToPoint(const protocol::Position& position)
BytePosition ToBytePosition(const protocol::Position& position, const protocol::string& content)
{
return TSPoint{
.row = static_cast<uint32_t>(position.line),
.column = static_cast<uint32_t>(position.character)
std::size_t offset = 0;
std::size_t line_start = 0;
std::uint32_t row = 0;
while (offset < content.size() && row < position.line)
{
if (content[offset++] == '\n')
{
++row;
line_start = offset;
}
}
protocol::uinteger utf16_units = 0;
while (offset < content.size() && content[offset] != '\n' &&
utf16_units < position.character)
{
const auto decoded = DecodeCharacter(content, offset);
const auto remaining = position.character - utf16_units;
if (decoded.utf16_units > remaining)
break;
offset += decoded.byte_count;
utf16_units += decoded.utf16_units;
}
return {
.offset = static_cast<protocol::uinteger>(offset),
.point = TSPoint{
.row = row,
.column = static_cast<std::uint32_t>(offset - line_start),
},
};
}
protocol::uinteger ToOffset(const protocol::Position& position, const protocol::string& content)
{
return ToBytePosition(position, content).offset;
}
TSPoint CalculateEndPoint(const protocol::string& text, TSPoint start)
{
TSPoint end = start;
lsp-server/test/test_provider/CMakeLists.txt
+2
View File
@@ -28,6 +28,7 @@ set(SOURCES
interpreter_test.cppm
provider_misc_test.cppm
provider_surface_test.cppm
text_coordinates_test.cppm
../../src/tree-sitter/parser.c
../../src/tree-sitter/scanner.c)
@@ -56,6 +57,7 @@ target_sources(
${CMAKE_CURRENT_SOURCE_DIR}/interpreter_test.cppm
${CMAKE_CURRENT_SOURCE_DIR}/provider_misc_test.cppm
${CMAKE_CURRENT_SOURCE_DIR}/provider_surface_test.cppm
${CMAKE_CURRENT_SOURCE_DIR}/text_coordinates_test.cppm
../../src/bridge/glaze.cppm
../../src/bridge/spdlog.cppm
../../src/bridge/taskflow.cppm
lsp-server/test/test_provider/test_main.cppm
+3
View File
@@ -12,6 +12,7 @@ import lsp.test.provider.json_flow;
import lsp.test.provider.json_provider_coverage;
import lsp.test.provider.misc;
import lsp.test.provider.surface;
import lsp.test.provider.text_coordinates;
import lsp.test.provider.fixtures;
export int Run(int argc, char** argv)
@@ -59,6 +60,8 @@ export int Run(int argc, char** argv)
lsp::test::provider::ProviderMiscTests::Register(runner);
std::cout << " - Provider surface tests" << std::endl;
lsp::test::provider::ProviderSurfaceTests::Register(runner);
std::cout << " - Text coordinate tests" << std::endl;
lsp::test::provider::TextCoordinatesTests::Register(runner);
runner.runAllTests();
return runner.getFailedCount();
lsp-server/test/test_provider/text_coordinates_test.cppm
+96
View File
@@ -0,0 +1,96 @@
module;
export module lsp.test.provider.text_coordinates;
import std;
import lsp.protocol;
import lsp.test.framework;
import lsp.utils.text_coordinates;
export namespace lsp::test::provider
{
class TextCoordinatesTests
{
public:
static void Register(TestRunner& runner);
private:
static TestResult TestAsciiAndBmpPositions();
static TestResult TestSupplementaryPlanePositions();
static TestResult TestMultilineAndClampedPositions();
static TestResult TestMalformedUtf8IsBounded();
static TestResult TestCalculateEndPointUsesByteColumns();
};
}
namespace lsp::test::provider
{
namespace
{
void ExpectPosition(const utils::text_coordinates::BytePosition& actual,
protocol::uinteger offset,
std::uint32_t row,
std::uint32_t column)
{
assertEqual(offset, actual.offset, "byte offset should match");
assertEqual(row, actual.point.row, "Tree-sitter row should match");
assertEqual(column, actual.point.column, "Tree-sitter byte column should match");
}
}
void TextCoordinatesTests::Register(TestRunner& runner)
{
runner.addTest("text coordinates convert ASCII and BMP positions", TestAsciiAndBmpPositions);
runner.addTest("text coordinates convert supplementary-plane positions", TestSupplementaryPlanePositions);
runner.addTest("text coordinates clamp multiline positions", TestMultilineAndClampedPositions);
runner.addTest("text coordinates bound malformed UTF-8", TestMalformedUtf8IsBounded);
runner.addTest("text coordinates calculate byte end points", TestCalculateEndPointUsesByteColumns);
}
TestResult TextCoordinatesTests::TestAsciiAndBmpPositions()
{
const protocol::string content = "A中Z";
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 0 }, content), 0U, 0U, 0U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 4U, 0U, 4U);
assertEqual(4U, utils::text_coordinates::ToOffset({ 0, 2 }, content), "ToOffset should use the paired conversion");
return { "", true, "ok" };
}
TestResult TextCoordinatesTests::TestSupplementaryPlanePositions()
{
const protocol::string content = "A😀Z";
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 1 }, content), 1U, 0U, 1U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 1U, 0U, 1U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 3 }, content), 5U, 0U, 5U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 4 }, content), 6U, 0U, 6U);
return { "", true, "ok" };
}
TestResult TextCoordinatesTests::TestMultilineAndClampedPositions()
{
const protocol::string content = "中x\n😀y";
ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 0 }, content), 5U, 1U, 0U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 2 }, content), 9U, 1U, 4U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 1, 100 }, content), 10U, 1U, 5U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 9, 0 }, content), 10U, 1U, 5U);
return { "", true, "ok" };
}
TestResult TextCoordinatesTests::TestMalformedUtf8IsBounded()
{
const protocol::string content{ 'A', static_cast<char>(0xF0), static_cast<char>(0x9F), 'Z' };
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 2 }, content), 2U, 0U, 2U);
ExpectPosition(utils::text_coordinates::ToBytePosition({ 0, 100 }, content), 4U, 0U, 4U);
return { "", true, "ok" };
}
TestResult TextCoordinatesTests::TestCalculateEndPointUsesByteColumns()
{
const auto end = utils::text_coordinates::CalculateEndPoint("中\n😀x", { 2U, 3U });
assertEqual(3U, end.row, "newline should advance the Tree-sitter row");
assertEqual(5U, end.column, "multibyte text should advance the byte column");
return { "", true, "ok" };
}
}