重构语法树/符号表

This commit is contained in:
csh
2025-10-15 20:31:00 +08:00
parent f52768e385
commit 1f7045140e
113 changed files with 229450 additions and 221041 deletions
+101
View File
@@ -0,0 +1,101 @@
#include "./cache.hpp"
#include "./tree_sitter_utils.hpp"
namespace lsp::language::ast
{
NodeCache::NodeCache(size_t max_size) :
max_size_(max_size)
{
}
const ASTNode* NodeCache::Find(const NodeKey& key)
{
auto it = cache_.find(key);
if (it != cache_.end())
{
// 缓存命中,更新LRU
Touch(key, it->second.lru_iter);
++hit_count_;
return &it->second.node;
}
++miss_count_;
return nullptr;
}
void NodeCache::Insert(const NodeKey& key, const ASTNode& node)
{
auto it = cache_.find(key);
if (it != cache_.end())
{
// 键已存在,更新值和LRU
it->second.node = node;
Touch(key, it->second.lru_iter);
return;
}
// 检查是否需要淘汰
if (cache_.size() >= max_size_)
{
EvictLRU();
}
// 插入新条目
lru_list_.push_front(key);
cache_[key] = CacheEntry{ node, lru_list_.begin() };
}
void NodeCache::Clear()
{
cache_.clear();
lru_list_.clear();
ResetStats();
}
void NodeCache::EvictLRU()
{
if (lru_list_.empty())
return;
// 移除最久未使用的(列表尾部)
const NodeKey& evict_key = lru_list_.back();
cache_.erase(evict_key);
lru_list_.pop_back();
}
void NodeCache::Touch([[maybe_unused]] const NodeKey& key, KeyIter iter)
{
// 将元素移到列表前面(最近使用)
lru_list_.splice(lru_list_.begin(), lru_list_, iter);
}
NodeKey NodeCache::MakeKey(const ASTNode& node)
{
return std::visit([](auto&& arg) -> NodeKey {
using T = std::decay_t<decltype(arg)>;
if constexpr (std::is_same_v<T, std::monostate>)
{
return NodeKey{ 0, 0, "" };
}
else
{
return NodeKey{
arg.location.start_byte,
arg.location.end_byte,
arg.node_type
};
}
},
node);
}
NodeKey NodeCache::MakeKey(TSNode ts_node)
{
return NodeKey{
ts_node_start_byte(ts_node),
ts_node_end_byte(ts_node),
std::string(ts::Type(ts_node))
};
}
}
+94
View File
@@ -0,0 +1,94 @@
#pragma once
#include <unordered_map>
#include <list>
#include <cstddef>
#include "./types.hpp"
extern "C" {
#include <tree_sitter/api.h>
}
namespace lsp::language::ast
{
// ==================== 缓存键 ====================
struct NodeKey
{
uint32_t start_byte;
uint32_t end_byte;
std::string node_type;
inline bool operator==(const NodeKey& other) const
{
return start_byte == other.start_byte &&
end_byte == other.end_byte &&
node_type == other.node_type;
}
};
struct NodeKeyHash
{
inline size_t operator()(const NodeKey& key) const
{
return std::hash<uint32_t>()(key.start_byte) ^
(std::hash<uint32_t>()(key.end_byte) << 1) ^
(std::hash<std::string>()(key.node_type) << 2);
}
};
// ==================== LRU 缓存(重构 1====================
class NodeCache
{
public:
explicit NodeCache(size_t max_size = 1000);
const ASTNode* Find(const NodeKey& key);
void Insert(const NodeKey& key, const ASTNode& node);
void Clear();
inline size_t Size() const { return cache_.size(); }
inline void SetMaxSize(size_t max_size) { max_size_ = max_size; }
inline size_t GetMaxSize() const { return max_size_; }
// 缓存统计
inline size_t HitCount() const { return hit_count_; }
inline size_t MissCount() const { return miss_count_; }
inline double HitRate() const
{
size_t total = hit_count_ + miss_count_;
return total > 0 ? static_cast<double>(hit_count_) / total : 0.0;
}
// 重置统计
inline void ResetStats()
{
hit_count_ = 0;
miss_count_ = 0;
}
static NodeKey MakeKey(const ASTNode& node);
static NodeKey MakeKey(TSNode ts_node);
private:
// LRU 列表:最近使用的在前面
using KeyList = std::list<NodeKey>;
using KeyIter = KeyList::iterator;
struct CacheEntry
{
ASTNode node;
KeyIter lru_iter;
};
void EvictLRU();
void Touch(const NodeKey& key, KeyIter iter);
size_t max_size_;
KeyList lru_list_;
std::unordered_map<NodeKey, CacheEntry, NodeKeyHash> cache_;
// 统计信息
size_t hit_count_ = 0;
size_t miss_count_ = 0;
};
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,133 @@
#pragma once
#include <functional>
#include <unordered_map>
#include <memory>
#include "./types.hpp"
#include "./tree_sitter_utils.hpp"
#include "./cache.hpp"
namespace lsp::language::ast
{
// ==================== 增量解析结果 ====================
struct IncrementalParseResult
{
ParseResult result;
std::vector<size_t> changed_indices;
std::vector<size_t> reused_indices;
inline size_t ChangedCount() const { return changed_indices.size(); }
inline size_t ReusedCount() const { return reused_indices.size(); }
inline size_t TotalCount() const { return result.nodes.size(); }
inline double ReuseRate() const
{
return TotalCount() > 0 ? static_cast<double>(ReusedCount()) / TotalCount() : 0.0;
}
};
class Deserializer
{
public:
Deserializer();
~Deserializer();
// 基础解析(不使用缓存)
ParseResult Parse(TSNode root, const std::string& source);
// 增量解析(自动复用缓存)
IncrementalParseResult ParseIncremental(TSNode root, const std::string& source);
// 缓存管理
void ClearCache();
size_t CacheSize() const;
void SetCacheMaxSize(size_t max_size);
// 缓存统计
double CacheHitRate() const;
void ResetCacheStats();
private:
std::unique_ptr<NodeCache> cache_;
void ParseChildWithCache(TSNode child, const std::string& source, IncrementalParseResult& result);
};
namespace detail
{
using ParseFunc = std::function<Result<ASTNode>(TSNode, const std::string&)>;
// 获取解析函数映射表
const std::unordered_map<std::string, ParseFunc>& GetParseFuncMap();
// 辅助函数
template<typename T>
inline void InitNode(T& node, TSNode ts_node)
{
node.location = ts::NodeLocation(ts_node);
node.node_type = std::string(ts::Type(ts_node));
}
template<typename T>
inline bool ExtractNameAndLocation(T& node, TSNode ts_node, const std::string& source)
{
TSNode name_node = ts::FieldChild(ts_node, "name");
if (ts::IsNull(name_node))
return false;
node.name = ts::Text(name_node, source);
node.name_location = ts::NodeLocation(name_node);
return !node.name.empty();
}
Result<std::string> GetRequiredField(TSNode node, std::string_view field, const std::string& source, const std::string& error_msg);
Result<std::string> ParseType(TSNode node, const std::string& source);
Result<Signature> ParseSignature(TSNode node, const std::string& source);
Result<std::vector<Parameter>> ParseParameters(TSNode node, const std::string& source);
Result<Block> ParseBlock(TSNode node, const std::string& source);
Access ParseAccess(std::string_view text);
ParseError MakeError(TSNode node, const std::string& message, ErrorSeverity severity = ErrorSeverity::Error);
template<typename T>
inline std::vector<Result<T>> SplitVariableDeclaration(
TSNode node,
const std::string& source,
std::function<Result<T>(TSNode, const std::string&, const std::string&, TSNode)> parse_func)
{
std::vector<Result<T>> results;
auto names = ts::FieldSummaries(node, "name", source);
for (const auto& name_summary : names)
{
results.push_back(parse_func(node, source, name_summary.text, name_summary.node));
}
return results;
}
void ParseClassMembers(TSNode body_node, const std::string& source, ClassDefinition& def);
void ParseClassVariableMember(TSNode node, const std::string& source, Access current_access, ClassDefinition& def);
void ParseClassMethodMember(TSNode node, const std::string& source, Access current_access, ClassDefinition& def);
void ParseClassPropertyMember(TSNode node, const std::string& source, Access current_access, ClassDefinition& def);
ParseResult ParseStatements(TSNode node, const std::string& source);
ParseResult ParseChildren(const std::vector<TSNode>& children, const std::string& source);
}
// ==================== 顶层解析 API ====================
Result<ASTNode> ParseNode(TSNode node, const std::string& source);
ParseResult ParseRoot(TSNode root, const std::string& source);
// ==================== 具体类型解析函数 ====================
Result<VarDeclaration> ParseVarDeclaration(TSNode node, const std::string& source, const std::string& var_name, TSNode name_node);
Result<StaticDeclaration> ParseStaticDeclaration(TSNode node, const std::string& source, const std::string& var_name, TSNode name_node);
Result<GlobalDeclaration> ParseGlobalDeclaration(TSNode node, const std::string& source, const std::string& var_name, TSNode name_node);
Result<ConstDeclaration> ParseConstDeclaration(TSNode node, const std::string& source);
Result<AssignmentStatement> ParseAssignmentStatement(TSNode node, const std::string& source);
Result<FunctionDefinition> ParseFunctionDefinition(TSNode node, const std::string& source);
Result<ClassDefinition> ParseClassDefinition(TSNode node, const std::string& source);
Result<Method> ParseMethod(TSNode node, const std::string& source);
Result<ExternalMethod> ParseExternalMethod(TSNode node, const std::string& source);
Result<Property> ParseProperty(TSNode node, const std::string& source);
Result<UnitDefinition> ParseUnitDefinition(TSNode node, const std::string& source);
Result<UsesClause> ParseUsesClause(TSNode node, const std::string& source);
}
@@ -0,0 +1,140 @@
#include "./tree_sitter_utils.hpp"
namespace lsp::language::ast::ts
{
// ==================== 字符串池实现 ====================
StringPool& StringPool::Instance()
{
static StringPool instance;
return instance;
}
std::string_view StringPool::Intern(std::string_view str)
{
auto it = pool_.find(std::string(str));
if (it != pool_.end())
{
return *it;
}
auto [inserted_it, _] = pool_.insert(std::string(str));
return *inserted_it;
}
void StringPool::Clear()
{
pool_.clear();
}
// ==================== 文本提取 ====================
std::string Text(TSNode node, std::string_view source)
{
uint32_t start = ts_node_start_byte(node);
uint32_t end = ts_node_end_byte(node);
if (start >= source.length() || end > source.length() || start >= end)
return "";
return std::string(source.substr(start, end - start));
}
std::string Text(TSNode node, const std::string& source)
{
return Text(node, std::string_view(source));
}
Location NodeLocation(TSNode node)
{
TSPoint start = ts_node_start_point(node);
TSPoint end = ts_node_end_point(node);
return Location{
static_cast<uint32_t>(start.row),
static_cast<uint32_t>(start.column),
static_cast<uint32_t>(end.row),
static_cast<uint32_t>(end.column),
ts_node_start_byte(node),
ts_node_end_byte(node)
};
}
std::string FieldText(TSNode node, std::string_view field_name, std::string_view source)
{
TSNode field = FieldChild(node, field_name);
return IsNull(field) ? "" : Text(field, source);
}
std::string FieldText(TSNode node, std::string_view field_name, const std::string& source)
{
return FieldText(node, field_name, std::string_view(source));
}
std::vector<TSNode> Children(TSNode node)
{
std::vector<TSNode> children;
uint32_t count = ts_node_child_count(node);
children.reserve(count);
for (uint32_t i = 0; i < count; i++)
children.push_back(ts_node_child(node, i));
return children;
}
std::vector<TSNode> FieldChildren(TSNode node, std::string_view field_name)
{
std::vector<TSNode> result;
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; i++)
{
const char* field = ts_node_field_name_for_child(node, i);
if (field && field_name == field)
result.push_back(ts_node_child(node, i));
}
return result;
}
std::vector<NodeSummary> ChildSummaries(TSNode node, std::string_view source)
{
std::vector<NodeSummary> result;
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; i++)
{
TSNode child = ts_node_child(node, i);
const char* field_name = ts_node_field_name_for_child(node, i);
result.push_back(NodeSummary{
.node = child,
.field = field_name ? field_name : "",
.type = std::string(Type(child)),
.text = Text(child, source) });
}
return result;
}
std::vector<NodeSummary> ChildSummaries(TSNode node, const std::string& source)
{
return ChildSummaries(node, std::string_view(source));
}
std::vector<NodeSummary> FieldSummaries(TSNode node, std::string_view field_name, std::string_view source)
{
std::vector<NodeSummary> result;
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; i++)
{
TSNode child = ts_node_child(node, i);
const char* field = ts_node_field_name_for_child(node, i);
if (field && field_name == field)
{
result.push_back(NodeSummary{
.node = child,
.field = field,
.type = std::string(Type(child)),
.text = Text(child, source) });
}
}
return result;
}
std::vector<NodeSummary> FieldSummaries(TSNode node, std::string_view field_name, const std::string& source)
{
return FieldSummaries(node, field_name, std::string_view(source));
}
}
@@ -0,0 +1,104 @@
#pragma once
#include <string>
#include <string_view>
#include <vector>
#include <unordered_set>
#include "./types.hpp"
extern "C" {
#include <tree_sitter/api.h>
}
namespace lsp::language::ast::ts
{
class StringPool
{
public:
static StringPool& Instance();
// 缓存字符串,返回池中的引用
std::string_view Intern(std::string_view str);
void Clear();
size_t Size() const { return pool_.size(); }
private:
StringPool() = default;
StringPool(const StringPool&) = delete;
StringPool& operator=(const StringPool&) = delete;
private:
std::unordered_set<std::string> pool_;
};
// ==================== 节点摘要 ====================
struct NodeSummary
{
TSNode node;
std::string field;
std::string type;
std::string text;
};
// ==================== 基础查询 ====================
// 获取节点文本(使用 string_view 优化)
std::string Text(TSNode node, std::string_view source);
std::string Text(TSNode node, const std::string& source);
// 获取节点类型(使用字符串池)
inline std::string_view Type(TSNode node);
// 获取节点位置
Location NodeLocation(TSNode node);
// 节点状态检查
inline bool IsNull(TSNode node);
inline bool IsComment(TSNode node);
// ==================== 字段操作 ====================
inline TSNode FieldChild(TSNode node, std::string_view field_name);
std::string FieldText(TSNode node, std::string_view field_name, std::string_view source);
std::string FieldText(TSNode node, std::string_view field_name, const std::string& source);
inline bool HasField(TSNode node, std::string_view field_name);
// ==================== 子节点操作 ====================
std::vector<TSNode> Children(TSNode node);
std::vector<TSNode> FieldChildren(TSNode node, std::string_view field_name);
std::vector<NodeSummary> ChildSummaries(TSNode node, std::string_view source);
std::vector<NodeSummary> ChildSummaries(TSNode node, const std::string& source);
std::vector<NodeSummary> FieldSummaries(TSNode node, std::string_view field_name, std::string_view source);
std::vector<NodeSummary> FieldSummaries(TSNode node, std::string_view field_name, const std::string& source);
// ==================== Inline 实现 ====================
inline std::string_view Type(TSNode node)
{
const char* type_str = ts_node_type(node);
return StringPool::Instance().Intern(type_str);
}
inline bool IsNull(TSNode node)
{
return ts_node_is_null(node);
}
inline bool IsComment(TSNode node)
{
std::string_view type = Type(node);
return type == "line_comment" || type == "block_comment" || type == "nested_comment";
}
inline TSNode FieldChild(TSNode node, std::string_view field_name)
{
return ts_node_child_by_field_name(node, field_name.data(), field_name.length());
}
inline bool HasField(TSNode node, std::string_view field_name)
{
return !IsNull(FieldChild(node, field_name));
}
}
+334
View File
@@ -0,0 +1,334 @@
#pragma once
#include <string>
#include <vector>
#include <optional>
#include <variant>
#include <cstdint>
#include <stdexcept>
namespace lsp::language::ast
{
// ==================== 基础类型 ====================
struct Location
{
uint32_t start_line = 0;
uint32_t start_column = 0;
uint32_t end_line = 0;
uint32_t end_column = 0;
uint32_t start_byte = 0;
uint32_t end_byte = 0;
};
struct Node
{
Location location;
std::string node_type;
};
struct Expression
{
Location location;
std::string node_type;
std::string text;
};
struct Parameter
{
Location location;
Location name_location;
std::string name;
std::optional<std::string> type_name;
std::optional<std::string> default_value;
bool is_var = false;
bool is_out = false;
};
struct Signature
{
std::vector<Parameter> parameters;
std::optional<std::string> return_type;
};
// ==================== 前向声明 ====================
struct VarDeclaration;
struct StaticDeclaration;
struct GlobalDeclaration;
struct ConstDeclaration;
struct AssignmentStatement;
struct FunctionDefinition;
struct ClassDefinition;
struct Method;
struct ExternalMethod;
struct ConstructorDeclaration;
struct DestructorDeclaration;
struct Property;
struct UsesClause;
struct Block;
struct UnitDefinition;
using ASTNode = std::variant<
std::monostate,
FunctionDefinition,
ClassDefinition,
VarDeclaration,
StaticDeclaration,
GlobalDeclaration,
ConstDeclaration,
AssignmentStatement,
Method,
ExternalMethod,
Property,
UsesClause,
UnitDefinition,
Expression,
Block>;
// ==================== AST 节点定义 ====================
struct Block
{
Location location;
std::string node_type;
std::vector<ASTNode> statements;
};
struct VarDeclaration
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<std::string> type_name;
std::optional<Expression> value;
};
struct StaticDeclaration
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<std::string> type_name;
std::optional<Expression> value;
};
struct GlobalDeclaration
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<std::string> type_name;
std::optional<Expression> value;
};
struct ConstDeclaration
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<std::string> type_name;
Expression value;
};
struct AssignmentStatement
{
Location location;
Location name_location;
std::string node_type;
std::string name;
Expression value;
};
struct FunctionDefinition
{
Location location;
Location name_location;
std::string node_type;
std::string name;
bool is_overload = false;
Signature signature;
Block body;
};
enum class Access
{
kPublic,
kProtected,
kPrivate
};
struct Property
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<std::string> type_name;
std::optional<std::string> getter;
std::optional<std::string> setter;
};
enum class Modifier
{
kNone,
kVirtual,
kOverride,
kOverload
};
struct Method
{
Location location;
Location name_location;
std::string node_type;
std::string name;
Signature signature;
bool is_class_method = false;
Modifier modifier = Modifier::kNone;
std::optional<Block> body;
};
struct ExternalMethod
{
Location location;
Location name_location;
std::string node_type;
std::string class_name;
std::string method_name;
Signature signature;
bool is_class_method = false;
bool is_operator = false;
std::string operator_symbol;
Modifier modifier = Modifier::kNone;
Block body;
};
struct ConstructorDeclaration
{
Location location;
Location name_location;
std::string node_type;
Signature signature;
std::optional<Block> body;
};
struct DestructorDeclaration
{
Location location;
Location name_location;
std::string node_type;
std::optional<Block> body;
};
struct ClassMember
{
Access access = Access::kPublic;
std::variant<
std::monostate,
VarDeclaration,
StaticDeclaration,
Method,
ConstructorDeclaration,
DestructorDeclaration,
Property>
content;
};
struct ClassDefinition
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::vector<std::string> parents;
std::vector<ClassMember> members;
};
struct UsesClause
{
Location location;
std::string node_type;
std::vector<std::string> units;
};
struct UnitDefinition
{
Location location;
Location name_location;
std::string node_type;
std::string name;
std::optional<UsesClause> uses;
std::vector<ASTNode> interface_statements;
std::vector<ASTNode> implementation_statements;
std::vector<ASTNode> initialization_statements;
std::vector<ASTNode> finalization_statements;
};
// ==================== 错误处理 ====================
enum class ErrorSeverity
{
Warning,
Error,
Fatal
};
struct ParseError
{
Location location;
std::string node_type;
std::string message;
ErrorSeverity severity = ErrorSeverity::Error;
};
struct ParseResult
{
std::vector<ASTNode> nodes;
std::vector<ParseError> errors;
inline bool HasErrors() const { return !errors.empty(); }
inline bool IsSuccess() const { return errors.empty(); }
};
// ==================== Result<T> 模板 ====================
template<typename T>
class Result
{
public:
static Result<T> Ok(T&& val)
{
return { true, std::move(val), "", ErrorSeverity::Error };
}
static Result<T> Err(const std::string& msg, ErrorSeverity sev = ErrorSeverity::Error)
{
return { false, std::nullopt, msg, sev };
}
inline bool IsOk() const { return success; }
inline bool IsErr() const { return !success; }
T Unwrap() &&
{
if (!success)
throw std::runtime_error("Unwrap on error: " + error);
return std::move(value).value();
}
T UnwrapOr(T&& default_val) &&
{
return success ? std::move(*value) : std::move(default_val);
}
bool success = false;
std::optional<T> value;
std::string error;
ErrorSeverity severity = ErrorSeverity::Error;
};
}