From 7c9a23746f2e600102731e1f1fa71c8e449bc895 Mon Sep 17 00:00:00 2001 From: "ci[bot]" Date: Thu, 30 Jul 2026 01:23:00 +0000 Subject: [PATCH] :package: deps(tsl): sync tsl-playbook from aa8a3e73 Source-Commit: aa8a3e73a8f8500121032c44f5c7aca22f467b3e --- tools/tsl-codegen/STANDARD.md | 187 ++-- tools/tsl-codegen/scripts/convert_tsf.py | 1007 +++++++++++++++++-- tools/tsl-codegen/tests/test_convert_tsf.py | 711 ++++++++++--- 3 files changed, 1608 insertions(+), 297 deletions(-) diff --git a/tools/tsl-codegen/STANDARD.md b/tools/tsl-codegen/STANDARD.md index 2b7630d5..e7ebc242 100644 --- a/tools/tsl-codegen/STANDARD.md +++ b/tools/tsl-codegen/STANDARD.md @@ -24,22 +24,19 @@ - [function](#function-2) - [class](#class-2) - [unit](#unit-2) - - [录入格式（用户必看）](#录入格式用户必看) + - [录入文件格式](#录入文件格式) - [yaml](#yaml) - [json](#json) - [统一 API 索引](#统一-api-索引) ## 基本原则 -- markdown 是唯一存储源 -- yaml/json 用于生成 markdown -- tsf 源码文档可以转换为 json 或 yaml 录入稿，再由同一生成流程产出 markdown +- 发布的 API 文档使用本标准定义的 markdown 存储格式 +- tsf 源码文档、yaml/json 录入数据与 markdown 的字段映射以本标准为准 - 一个录入文件对应一个 markdown 叶子页；录入根固定为 `module`、`path`、`declarations` -- `declarations` 是非空有序列表，function、class、unit 可以按输入顺序混合 -- 生成器写入 markdown 前必须使用仓库锁定版本的 Prettier 统一格式 -- 转换器允许生成描述或类型不完整的草稿；生成器严格校验，不发布不完整录入稿 -- function 调用签名由维护者或转换器提供；工具不推断 API 语义 +- `declarations` 是非空有序列表，function、class、unit 可以混合；列表顺序即文档顺序 +- function 调用签名只记录显式名称和参数名，不包含类型、默认值或语义推断内容 - API 标题的反引号内只写名称或调用签名，不写 function、class、property 等声明术语 - 每个 API 用独立的 `声明：...` 行记录声明种类 - 描述紧跟声明行；tags 是描述的检索补充，位于完整描述之后 @@ -56,11 +53,10 @@ function、class、unit 都是同级的顶级声明，可以出现在同一页 种类写在自身正文的 `声明：function|class|unit` 行中。 目录承担领域或模块分组，叶子页承担 API 主题分组，H2 承担顶级声明分组。markdown -不要求一个 class 对应一个 md；多个独立 class TSF 的对外声明可以进入同一叶子页， -也可以在内容过长或主题不同时拆到同一目录下的多个叶子页。例如 +不要求一个 class 对应一个 md；多个独立 class 声明可以进入同一叶子页，也可以在 +内容过长或主题不同时拆到同一目录下的多个叶子页。例如 `OpenXmlAttribute` 和 `OpenXmlElement` 可以作为两个 H2 同处 -`officexml/openxml/elements.md`。每个 TSF 文件对应一个对外顶级声明，一个 markdown -叶子页可以收录多个对外顶级声明。 +`officexml/openxml/elements.md`。一个 markdown 叶子页可以收录多个对外顶级声明。 顶级声明和成员都按文档顺序存储，不按名称、种类或可见性重新排序。每个顶级声明 按以下共同顺序开始： @@ -161,7 +157,7 @@ function 重载和跨声明种类同名允许。 #### 示例代码 -- 每个示例以“范例NN：说明”开头，编号按出现顺序自动生成并至少保留两位 +- 每个示例以“范例NN：说明”开头，编号按出现顺序排列并至少保留两位 - 代码围栏使用 `tsl` - 一个代码围栏只放一个独立示例 - 字符串使用直引号 `'` 或 `"` @@ -247,7 +243,7 @@ return demoFn(src, 0); 3. class 描述，必填 4. tags，可选；位于完整描述之后 5. `父类：BaseClass`，可选；多父类按声明顺序列出 -6. class 成员，按源码顺序书写 +6. class 成员，按声明顺序书写 #### 成员标题与正文 @@ -411,7 +407,7 @@ unit function 的参数取值/示例使用 H4，interface class function 的参 2. `声明：unit` 3. unit 描述，必填 4. tags，可选；位于完整描述之后 -5. interface direct member，按源码顺序书写 +5. interface direct member，按声明顺序书写 #### 成员标题与正文 @@ -423,8 +419,8 @@ H3 direct member 标题只写名称或调用签名。标题后的声明行只允 - interface class：`声明：class` 每个 direct member 都按“标题、声明行、描述、tags、成员数据”的共同顺序开始。 -unit direct member 都来自 interface，因此正文不重复写 public；统一索引把其 -visibility 规范化为 `public`。unit function 复用顶级 function 格式并要求返回 +unit direct member 都来自 interface，visibility 固定为 `public`，正文不重复记录。 +unit function 复用顶级 function 格式并要求返回 类型，var 必须写类型，const 必须写值。interface class 的 H4 成员复用顶级 class 的成员格式，并显式写 `可见性：public|protected`。 @@ -490,7 +486,7 @@ class 的成员格式，并显式写 `可见性：public|protected`。 ### 综合示例 -同一叶子页可以按录入顺序混合三种顶级声明。以下 API 仅用于说明存储格式，不代表 +同一叶子页可以按文档顺序混合三种顶级声明。以下 API 仅用于说明存储格式，不代表 真实 TSL API： ````markdown @@ -571,30 +567,29 @@ return ParseOpenXml(''); ## tsf 源码文档格式 tsf 主要使用 `function`、`unit` 和 `type` 三种顶层组织方式。本标准按这三种方式 -分别定义源码文档格式。转换器可以一次接收任意混合的 TSF 输入；每个文件贡献一个 -对外顶级声明，并严格保持命令行输入顺序。 +分别定义源码文档格式。每个 tsf 源码文档对应一个对外顶级声明。 ### function -独立顶层 `function` 可以在源码中记录录入数据所需的函数级内容。签名、参数类型、 -默认参数和返回类型直接读取源码声明。 +独立顶层 `function` 可以在源码中记录函数级文档。签名、参数类型、默认参数和返回 +类型由源码声明提供，描述、标签、参数说明、枚举值和示例由文档块提供。 -- 一个 tsf 文件只记录第一个主函数；文件中的后续辅助函数不进入录入数据 -- 顶层 `procedure` 不按 `function` 格式处理 -- 多个 tsf 可以与 class/unit TSF 混合生成同一个 json/yaml 录入文件；页面级 - `module` 和 `path` 在转换时统一提供，不写进单个函数的注释 +- 一个 function tsf 文件只以第一个顶层 function 作为对外主函数；后续辅助函数不属于 + 该文件的对外文档 +- 顶层 `procedure` 不在本节规定的 function 文档格式范围内 +- `module` 和 `path` 是页面级录入字段，不属于单个函数的文档块 #### 文档块位置 文档块必须是主函数 `begin` 之后的第一段非空内容，并且位于任何可执行语句、 编译指令或其他注释之前 -文档块由连续的 `///` 行组成。允许按照函数体缩进；解析时忽略 `///` 之前的空白， -并移除 `///` 及其后的一个可选空格。遇到第一行非 `///` 内容时，文档块结束；函数体 -后续位置的注释是普通注释 +文档块由连续的 `///` 行组成，并允许按照函数体缩进。`///` 之前的空白以及其后的 +一个可选空格不属于文档内容。遇到第一行非 `///` 内容时，文档块结束；函数体后续 +位置的注释是普通注释 -TSL 解释器将 `///` 作为普通的 `//` 行注释；第三个 `/` 是 codegen 用来识别文档行的 -标记 +TSL 解释器将 `///` 作为普通的 `//` 行注释；本标准使用第三个 `/` 区分文档行与普通 +注释 #### 文档块结构 @@ -609,7 +604,7 @@ TSL 解释器将 `///` 作为普通的 `//` 行注释；第三个 `/` 是 codege 空的 `///` 行可以在多行函数描述或示例中保留空行。指令名固定为小写；未知指令、 重复的 `@tags:`/`@returns:`、同一参数重复的 `@param:`/`@values:`，以及不符合上述 -顺序的指令均视为错误。`@example:` 可以重复，`@output:` 在同一示例组内最多出现 +顺序的指令均不符合本标准。`@example:` 可以重复，`@output:` 在同一示例组内最多出现 一次。 | 写法 | 必填 | json 映射 | 规则 | @@ -623,36 +618,35 @@ TSL 解释器将 `///` 作为普通的 `//` 行注释；第三个 `/` 是 codege | 示例代码 | 示例组内 | `declarations[].examples[].code` | 内容行额外缩进两个空格；至少包含一个非空代码行 | | `@output:` | 否 | `declarations[].examples[].output` | 原始输出额外缩进两个空格；存在时不得为空 | -`@param:` 和 `@values:` 中的参数名与函数声明大小写无关地匹配，json 使用函数声明中的 -参数拼写。 +`@param:` 和 `@values:` 中的参数名与函数声明大小写无关地匹配；映射后的参数名保留 +函数声明中的拼写。 #### 字段来源与映射 以下字段由主函数声明以及必要的文档指令确定： -| tsf 声明内容 | json 字段 | 转换规则 | -| -------------- | ---------------------------------- | --------------------------------------------------- | -| 函数名 | `declarations[].name` | 保留声明名称 | -| 函数名和参数名 | `declarations[].signature` | 规范化为只含名称的调用形式，不复制类型或默认值 | -| 参数类型 | `declarations[].params[].type` | 保留声明中的类型 | -| 参数默认值 | `declarations[].params[].optional` | 存在默认值时写入 `true`，默认表达式不另建 json 字段 | -| 返回类型 | `declarations[].returns` | 读取声明和 `@returns:`，按下述规则合并 | +| tsf 声明内容 | json 字段 | 映射规则 | +| -------------- | ---------------------------------- | ------------------------------------------------- | +| 函数名 | `declarations[].name` | 保留声明名称 | +| 函数名和参数名 | `declarations[].signature` | 规范化为只含名称的调用形式，不复制类型或默认值 | +| 参数类型 | `declarations[].params[].type` | 保留声明中的类型 | +| 参数默认值 | `declarations[].params[].optional` | 存在默认值时为 `true`，默认表达式不另建 json 字段 | +| 返回类型 | `declarations[].returns` | 取自声明和 `@returns:`，按下述规则合并 | -要直接得到可生成 markdown 的完整录入数据，函数必须显式声明每个参数的类型，为每个 -参数提供非空的 `@param:`，并通过函数声明或 `@returns:` 提供返回类型。缺少这些内容 -时只能得到待手工完善的录入稿。 +完整录入数据必须包含每个参数的显式类型和非空说明，并通过函数声明或 `@returns:` +提供返回类型。 可选参数的说明仍应写清默认值含义；`optional: true` 只表达该参数可以省略。 返回类型按以下规则合并： - 只有函数声明时，使用声明中的拼写 - 只有 `@returns:` 时，去除首尾空白后使用指令中的拼写 -- 两处同时存在时，转换器使用与函数声明相同的词法规则拆分类型，忽略 token 之间的 - 空白，并按 TSL 标识符大小写无关的规则比较标识符 token；其他 token 必须一致。 - 校验通过后使用声明中的拼写，校验失败则定位 `@returns:` 行、报错并停止转换 -- 两处都不存在时，录入稿中的 `returns` 为空，不能直接生成 markdown +- 两处同时存在时，使用相同的词法规则拆分类型，忽略 token 之间的空白，并按 TSL + 标识符大小写无关的规则比较标识符 token；其他 token 必须一致。两处类型必须匹配， + 映射后使用声明中的拼写 +- 两处都不存在时，不满足顶级 function 的 `returns` 必填要求 -转换器只校验类型的词法结构，不判断类型别名等语义等价。 +本标准只定义类型的词法一致性，不定义类型别名等语义等价关系。 #### 枚举值表 @@ -673,10 +667,10 @@ TSL 解释器将 `///` 作为普通的 `//` 行注释；第三个 `/` 是 codege - 不接受数组、对象或 json `null` - 值和说明以值后的第一个分隔冒号分开；说明不得为空 - 保留枚举项的声明顺序 -- 同一参数最多有一个非空值表，重复值视为错误；不同 json 类型的值不视为重复， +- 同一参数最多有一个非空值表，不允许重复值；不同 json 类型的值不视为重复， 例如数字 `1` 与字符串 `"1"` 是两个值 - `@values:` 引用的参数必须存在 -- 转换器不推断枚举值是否与参数类型兼容，该语义由 tsf 作者负责 +- tsf 作者必须保证枚举值与参数类型在语义上兼容 #### 示例组 @@ -685,20 +679,20 @@ TSL 解释器将 `///` 作为普通的 `//` 行注释；第三个 `/` 是 codege `@example:` 开始新的示例组。第一个示例组出现后，不得再写参数、返回类型等其他函数 级指令。 -转换时移除代码和输出的两个结构缩进，保留其余空白和换行。示例说明、代码和输出 -分别映射为 `examples[].desc`、`examples[].code` 和 `examples[].output`。`desc` 与 +映射到录入数据时，移除代码和输出的两个结构缩进，保留其余空白和换行。示例说明、 +代码和输出分别映射为 `examples[].desc`、`examples[].code` 和 `examples[].output`。`desc` 与 `code` 必填且非空；`output` 可选，但出现时必须包含至少一个非空行。示例顺序保持 不变。 `code` 只记录示例源码，不得包含标准输出标记 `// 输出：`。`output` 只记录原始输出， -不写 `//` 注释标记。生成 markdown 时： +不写 `//` 注释标记。映射到 markdown 时： - 所有示例共用一个 `### 示例` 标题 -- 每项按顺序生成“范例01：说明”“范例02：说明” -- 每项生成一个独立的 `tsl` 代码块 +- 每项按顺序写为“范例01：说明”“范例02：说明” +- 每项使用一个独立的 `tsl` 代码块 - 没有 `output` 时，代码块只包含 `code` -- 单行 `output` 在代码末尾生成 `// 输出：<值>` -- 多行 `output` 先生成 `// 输出：`，再为每个输出行添加 `//` 和一个空格；空输出行生成 +- 单行 `output` 在代码末尾写为 `// 输出：<值>` +- 多行 `output` 先写 `// 输出：`，再为每个输出行添加 `//` 和一个空格；空输出行使用 单独的 `//` #### 完整 tsf 示例 @@ -950,33 +944,32 @@ end; ### unit -unit 只接受包含显式 `interface`、`implementation` 并以 `end.` 结束的完整形态。 -unit 名称必须与文件名大小写无关地一致；简写 unit 明确报错。 +本标准的 unit 源码文档采用包含显式 `interface`、`implementation` 并以 `end.` 结束的 +完整形态；简写 unit 不在本标准范围内。unit 名称必须与文件名大小写无关地一致。 unit 文档块位于 `unit Name;` 之后、`interface` 之前，只允许描述和可选 -`@tags:`。转换器按源码顺序收录 interface 中的 function、var、const 和 class； -`uses` 只表示依赖，不进入 API。interface class 完整复用独立 class 的成员规则， -但 interface 中的所有 class 都进入文档。受支持区域中无法绑定到 unit 或 interface -成员的 `///` 文档块按原始行号报错。 +`@tags:`。对外文档按声明顺序包含 interface 中的 function、var、const 和 class； +`uses` 只表示依赖，不属于 API。interface class 完整复用独立 class 的成员规则， +interface 中的所有 class 均属于对外文档。每个 `///` 文档块必须绑定到 unit 或 +interface 成员。 -进入 implementation 后停止收集 API；其中的函数、类、变量、常量和 `///` 文档块 -全部忽略。interface 中的 procedure 和非 class type 不支持，必须在声明行报错， -不能静默遗漏。interface function 复用 function 文档指令并要求最终返回类型； -var/const 只允许描述和可选 `@tags:`，且一项一条声明。 +implementation 中的函数、类、变量、常量和 `///` 文档块不属于对外文档。interface +中的 procedure 和非 class type 不在本标准范围内。interface function 复用 function +文档指令并要求返回类型；var/const 只允许描述和可选 `@tags:`，且一项一条声明。 ## 录入数据结构 一个录入文件对应一个 markdown 叶子页。录入根只允许 `module`、`path`、 `declarations`。`declarations` 必须是非空有序列表，function、class、unit 可以 -任意混合，生成器严格保持数组顺序。 +任意混合；数组顺序即 markdown 中的 H2 声明顺序。 顶层字段： -| 字段 | 必填 | 说明 | -| -------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `module` | 是 | markdown 一级标题内容；不是目录名或文件名。例如 `module: 示例 / 数组` 生成 `# 示例 / 数组` | -| `path` | 是 | 目标 markdown 在 scope 目录下的相对路径，包含子目录和文件名，使用 `/` 分隔且不含 `.md` 后缀。例如 `path: base/example` 在默认 `project` scope 下生成 `references/codegen/project/base/example.md` | -| `declarations` | 是 | 非空顶级声明列表；每项由 `kind` 判别并要求 `name`，按数组顺序生成 H2 | +| 字段 | 必填 | 说明 | +| -------------- | ---- | -------------------------------------------------------------------------------------------------------------------- | +| `module` | 是 | markdown 一级标题内容；不是目录名或文件名。例如 `module: 示例 / 数组` 对应 `# 示例 / 数组` | +| `path` | 是 | markdown 在所属 scope 下的相对路径，使用 `/` 分隔且不含 `.md` 后缀。例如 `path: base/example` 对应 `base/example.md` | +| `declarations` | 是 | 非空顶级声明列表；每项由 `kind` 判别并要求 `name`，数组顺序即 H2 顺序 | ### function @@ -986,22 +979,22 @@ var/const 只允许描述和可选 `@tags:`，且一项一条声明。 | ----------- | ------ | ------------------------------------------------------ | | `kind` | 是 | 固定为 `function` | | `name` | 是 | function 简单名称；必须与 `signature` 中的名称一致 | -| `signature` | 是 | 维护者提供的调用签名 | +| `signature` | 是 | 只包含名称和参数名的调用签名 | | `desc` | 是 | 函数描述，可包含多行 | | `tags` | 否 | 检索关键词列表；推荐填写，有助于更准确地识别和检索函数 | | `params` | 有参时 | 参数列表；无参函数省略 | | `returns` | 是 | 返回类型 | -| `examples` | 否 | 示例列表；按顺序生成独立的 TSL 代码块 | +| `examples` | 否 | 示例列表；按顺序映射为独立的 TSL 代码块 | 参数字段： -| 字段 | 必填 | 说明 | -| ---------- | ---- | ------------------------------- | -| `name` | 是 | 参数名，与签名一致 | -| `type` | 是 | 参数类型 | -| `desc` | 是 | 参数说明 | -| `optional` | 否 | `true` 时自动添加 `可选。` 前缀 | -| `values` | 否 | 枚举值列表，生成参数取值说明 | +| 字段 | 必填 | 说明 | +| ---------- | ---- | ------------------------------------------------------ | +| `name` | 是 | 参数名，与签名一致 | +| `type` | 是 | 参数类型 | +| `desc` | 是 | 参数说明 | +| `optional` | 否 | `true` 表示可选参数；markdown 参数说明以 `可选。` 开头 | +| `values` | 否 | 枚举值列表，对应参数取值说明 | `values` 每项包含： @@ -1012,11 +1005,11 @@ var/const 只允许描述和可选 `@tags:`，且一项一条声明。 `examples` 每项包含： -| 字段 | 必填 | 说明 | -| -------- | ---- | ---------------------------------------------------- | -| `desc` | 是 | 示例场景说明；生成“范例NN：说明” | -| `code` | 是 | 不含代码围栏和标准输出注释的 TSL 代码 | -| `output` | 否 | 原始输出；生成器按单行或多行规则转换为 `//` 输出注释 | +| 字段 | 必填 | 说明 | +| -------- | ---- | ---------------------------------------------- | +| `desc` | 是 | 示例场景说明；对应“范例NN：说明” | +| `code` | 是 | 不含代码围栏和标准输出注释的 TSL 代码 | +| `output` | 否 | 原始输出；按单行或多行规则映射为 `//` 输出注释 | ### class @@ -1045,8 +1038,8 @@ class 对象字段： `kind: method` 对象不使用 `static` 字段；class function 由 `binding: class` 唯一 表达。`value` 的存在性与真假值分开判断，因此数字 `0` 和布尔值 `false` 都是合法 常量值。 -这里的 `method` 是录入结构内部用于归一化实例 function 和 class function 的 kind， -不会写进 markdown 标题或 `声明：...` 行。 +这里的 `method` 是录入结构内部用于统一表达实例 function 和 class function 的 kind， +不对应 markdown 标题或 `声明：...` 行中的声明值。 ### unit @@ -1062,13 +1055,11 @@ class 对象字段： 录入数据不保存 unit 的 `uses`、implementation、initialization 或 finalization。 -## 录入格式（用户必看） +## 录入文件格式 ### yaml -yaml 适合包含多行示例的页面。解析 yaml 需要安装 `pyyaml` - -注意： +yaml 录入文件应符合以下规则： - `examples[].code` 和多行 `examples[].output` 使用 `|` 块标量 - 参数名 `...` 必须加引号 @@ -1078,9 +1069,7 @@ yaml 适合包含多行示例的页面。解析 yaml 需要安装 `pyyaml` ### json -json 使用 Python 标准库解析，无额外依赖 - -注意： +json 录入文件应符合以下规则： - json 不支持注释 - 多行 `examples[].code` 和 `examples[].output` 使用 `\n` @@ -1130,7 +1119,3 @@ name scope module signature page anchor tags summary kind binding visibility own 例如 class function `Widget.Create` 的 owner 是 `Widget`；unit interface class function `DemoUnit.Document.Save` 的 owner 是 `DemoUnit.Document`。重载共享 `qualified_name`，由 `signature` 和唯一的 `page#anchor` 区分。 - -lookup 的 `--name` 同时精确匹配简单名称与 `qualified_name`，比较大小写不敏感。 -简单成员名会返回所有 owner 下的同名 API；完全限定名称用于缩小到指定 class/unit。 -`--kw` 还会搜索 kind、binding、visibility、owner 和 qualified_name。 diff --git a/tools/tsl-codegen/scripts/convert_tsf.py b/tools/tsl-codegen/scripts/convert_tsf.py index 4df518f8..999ee48a 100644 --- a/tools/tsl-codegen/scripts/convert_tsf.py +++ b/tools/tsl-codegen/scripts/convert_tsf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Convert documented TSF function, class, and unit files to declarations.""" +"""Convert TSF function, class, and unit files to declaration drafts.""" import argparse import json @@ -12,6 +12,35 @@ from pathlib import Path STRUCTURE_KEYWORDS = {"function", "procedure", "type", "unit"} PARAM_MODIFIERS = {"const", "var"} +RECOVERY_DECLARATION_KEYWORDS = { + "begin", + "class", + "const", + "constructor", + "destructor", + "end", + "function", + "finalization", + "implementation", + "initialization", + "interface", + "private", + "procedure", + "property", + "protected", + "public", + "static", + "type", + "unit", + "uses", + "var", +} +RESERVED_IDENTIFIER_KEYWORDS = RECOVERY_DECLARATION_KEYWORDS | { + "overload", + "override", + "virtual", +} +SIGNATURE_BOUNDARY_KEYWORDS = RECOVERY_DECLARATION_KEYWORDS - {"const", "var"} DOCUMENT_LINE_RE = re.compile(r"^\s*///(.*)$") DIRECTIVE_RE = re.compile(r"^@([a-z]+):(.*)$") @@ -70,6 +99,13 @@ def fail(path, line, message): raise ConversionError(path, line, message) +def is_valid_identifier_token(token): + return ( + token.kind == "identifier" + and token.value.casefold() not in RESERVED_IDENTIFIER_KEYWORDS + ) + + def is_identifier_start(char): return char == "_" or char.isalpha() @@ -159,7 +195,7 @@ def tokenize(source): return tokens -def first_structure_token(tokens, path): +def first_structure_token(tokens): for token in tokens: if token.kind != "identifier": continue @@ -167,7 +203,7 @@ def first_structure_token(tokens, path): if keyword not in STRUCTURE_KEYWORDS: continue return token - fail(path, 1, "没有找到受支持的顶层声明") + return None def matching_parenthesis(tokens, open_index, path): @@ -196,7 +232,14 @@ def declaration_end(tokens, start_index, path): depth -= 1 elif value == ";" and depth == 0: return index - fail(path, tokens[start_index].line, "function 声明缺少分号") + line = ( + tokens[start_index].line + if start_index < len(tokens) + else tokens[-1].line + if tokens + else 1 + ) + fail(path, line, "function 声明缺少分号") def split_parameter_ranges(tokens, start, end): @@ -255,7 +298,7 @@ def parse_parameter(source, tokens, start, end, path, fallback_line): if len(name_tokens) != 1: fail(path, fallback_line, "参数声明必须包含一个参数名") name_token = name_tokens[0] - if name_token.kind != "identifier" and name_token.value != "...": + if not is_valid_identifier_token(name_token) and name_token.value != "...": fail(path, name_token.line, "参数名不是有效标识符") param_type = "" @@ -269,10 +312,17 @@ def parse_function_signature(source, tokens, function_index, path): if function_index + 1 >= len(tokens): fail(path, function_token.line, "function 声明缺少函数名") name_token = tokens[function_index + 1] - if name_token.kind != "identifier": + if not is_valid_identifier_token(name_token): fail(path, name_token.line, "function 声明缺少有效函数名") end_index = declaration_end(tokens, function_index + 2, path) + if any( + token.line > function_token.line + and token.kind == "identifier" + and token.value.casefold() in SIGNATURE_BOUNDARY_KEYWORDS + for token in tokens[function_index + 2 : end_index] + ): + fail(path, function_token.line, "function 声明缺少分号") end_token = tokens[end_index] between = tokens[function_index + 2 : end_index] open_index = next( @@ -325,6 +375,18 @@ def parse_function_signature(source, tokens, function_index, path): return name_token.value, parameters, return_type, end_index +def function_begin_before_next_structure(tokens, start_index): + for token in tokens[start_index:]: + if token.kind != "identifier": + continue + lowered = token.value.casefold() + if lowered == "begin": + return token + if lowered in STRUCTURE_KEYWORDS: + return None + return None + + def parse_declaration(source, tokens, function_token, path): function_index = tokens.index(function_token) name, parameters, return_type, end_index = parse_function_signature( @@ -332,27 +394,20 @@ def parse_declaration(source, tokens, function_token, path): ) end_token = tokens[end_index] - begin_token = next( - ( - token - for token in tokens[end_index + 1 :] - if token.kind == "identifier" and token.value.casefold() == "begin" - ), - None, - ) + begin_token = function_begin_before_next_structure(tokens, end_index + 1) if begin_token is None: fail(path, end_token.line, "function 缺少 begin") return name, parameters, return_type, begin_token -def extract_document_lines(source, begin_token, path): +def extract_document_lines(source, begin_token): physical_lines = source.splitlines() begin_index = begin_token.line - 1 line_end = source.find("\n", begin_token.end) if line_end == -1: line_end = len(source) if source[begin_token.end : line_end].strip(): - fail(path, begin_token.line, "begin 后第一段内容必须是 /// 文档块") + return [] first_content = None for index in range(begin_index + 1, len(physical_lines)): @@ -360,13 +415,9 @@ def extract_document_lines(source, begin_token, path): first_content = index break if first_content is None: - fail(path, begin_token.line, "function 缺少 /// 文档块") + return [] if not DOCUMENT_LINE_RE.match(physical_lines[first_content]): - fail( - path, - first_content + 1, - "begin 后第一段内容必须是 /// 文档块", - ) + return [] document = [] for index in range(first_content, len(physical_lines)): @@ -754,6 +805,19 @@ def converted_parameters(parameters, descriptions=None, values=None): return result +def best_effort_function_details(document, parameters, declared_return, path): + if document: + try: + return parse_document(document, parameters, declared_return, path) + except ConversionError: + pass + return { + "desc": "", + "params": converted_parameters(parameters), + "returns": declared_return, + } + + def skip_begin_block(tokens, begin_index, path): depth = 0 for index in range(begin_index, len(tokens)): @@ -825,7 +889,7 @@ def parse_property_member(source, tokens, property_index, path, visibility): if property_index + 1 >= len(tokens): fail(path, tokens[property_index].line, "property 声明缺少名称") name_token = tokens[property_index + 1] - if name_token.kind != "identifier": + if not is_valid_identifier_token(name_token): fail(path, name_token.line, "property 声明缺少有效名称") end_index = declaration_end(tokens, property_index + 2, path) end_token = tokens[end_index] @@ -923,7 +987,7 @@ def parse_field_member(source, tokens, start_index, path, visibility, is_static) end_index = declaration_end(tokens, start_index, path) cursor = start_index + (1 if is_static else 0) cursor = skip_attributes(tokens, cursor, end_index, path) - if cursor >= end_index or tokens[cursor].kind != "identifier": + if cursor >= end_index or not is_valid_identifier_token(tokens[cursor]): fail(path, tokens[start_index].line, "字段声明缺少有效名称") name_token = tokens[cursor] colon_index = next( @@ -975,7 +1039,10 @@ def parse_field_member(source, tokens, start_index, path, visibility, is_static) def parse_constant_member(source, tokens, start_index, path, visibility, is_static): const_index = start_index + (1 if is_static else 0) end_index = declaration_end(tokens, const_index + 1, path) - if const_index + 1 >= end_index or tokens[const_index + 1].kind != "identifier": + if ( + const_index + 1 >= end_index + or not is_valid_identifier_token(tokens[const_index + 1]) + ): fail(path, tokens[const_index].line, "const 声明缺少有效名称") name_token = tokens[const_index + 1] equals_index = next( @@ -1057,7 +1124,10 @@ def parse_class(source, tokens, type_index, path, *, require_filename_match=True if type_index + 3 >= len(tokens): fail(path, tokens[type_index].line, "class 声明不完整") name_token = tokens[type_index + 1] - if name_token.kind != "identifier" or tokens[type_index + 2].value != "=": + if ( + not is_valid_identifier_token(name_token) + or tokens[type_index + 2].value != "=" + ): fail(path, tokens[type_index].line, "type class 声明格式错误") class_index = type_index + 3 if not keyword_at(tokens, class_index, "class"): @@ -1229,7 +1299,7 @@ def parse_bare_unit_variable(source, tokens, start_index, path): def parse_bare_unit_constant(source, tokens, start_index, path): end_index = declaration_end(tokens, start_index + 1, path) name_token = tokens[start_index] - if name_token.kind != "identifier": + if not is_valid_identifier_token(name_token): fail(path, name_token.line, "const 声明缺少有效名称") equals_index = next( ( @@ -1274,7 +1344,10 @@ def parse_unit(source, tokens, unit_index, path): if unit_index + 2 >= len(tokens): fail(path, tokens[unit_index].line, "unit 声明不完整") name_token = tokens[unit_index + 1] - if name_token.kind != "identifier" or tokens[unit_index + 2].value != ";": + if ( + not is_valid_identifier_token(name_token) + or tokens[unit_index + 2].value != ";" + ): fail(path, tokens[unit_index].line, "unit 声明格式错误") if name_token.value.casefold() != Path(path).stem.casefold(): fail(path, name_token.line, "unit 名称必须与文件名一致") @@ -1418,17 +1491,840 @@ def parse_unit(source, tokens, unit_index, path): } +def mask_document_lines(source): + masked = [] + for line in source.splitlines(keepends=True): + content = line.rstrip("\r\n") + ending = line[len(content) :] + if DOCUMENT_LINE_RE.match(content): + masked.append(" " * len(content) + ending) + else: + masked.append(line) + return "".join(masked) + + +def next_semicolon(tokens, start_index, limit): + depth = 0 + for index in range(start_index, limit): + value = tokens[index].value + if value in {"(", "["}: + depth += 1 + elif value in { + ")", + "]", + }: + depth = max(0, depth - 1) + elif value == ";" and depth == 0: + return index + return None + + +def next_declaration_boundary(tokens, start_index, limit): + if start_index >= min(limit, len(tokens)): + return None + start_line = tokens[start_index].line + for index in range(start_index + 1, limit): + token = tokens[index] + if ( + token.line > start_line + and token.kind == "identifier" + and token.value.casefold() in RECOVERY_DECLARATION_KEYWORDS + ): + return index + return None + + +def declaration_scan_end(tokens, start_index, limit): + semicolon_index = next_semicolon(tokens, start_index, limit) + boundary_index = next_declaration_boundary(tokens, start_index, limit) + candidates = [ + index + for index in (semicolon_index, boundary_index) + if index is not None + ] + return min(candidates) if candidates else limit + + +def token_start_or_eof(tokens, index, source): + return tokens[index].start if index < len(tokens) else len(source) + + +def recover_after_declaration(tokens, start_index, limit): + boundary_index = next_declaration_boundary(tokens, start_index, limit) + semicolon_index = next_semicolon(tokens, start_index, limit) + if boundary_index is not None and ( + semicolon_index is None or boundary_index < semicolon_index + ): + if keyword_at(tokens, boundary_index, "begin"): + try: + return min( + skip_begin_block(tokens, boundary_index, Path("")), + limit, + ) + except ConversionError: + return limit + return boundary_index + if semicolon_index is None: + return limit + index = semicolon_index + 1 + while ( + index + 1 < limit + and tokens[index].value.casefold() in {"overload", "virtual", "override"} + and tokens[index + 1].value == ";" + ): + index += 2 + if keyword_at(tokens, index, "begin"): + try: + return min(skip_begin_block(tokens, index, Path("")), limit) + except ConversionError: + return limit + return index + + +def matching_parenthesis_in_range(tokens, open_index, limit): + depth = 0 + for index in range(open_index, limit): + value = tokens[index].value + if value == "(": + depth += 1 + elif value == ")": + depth -= 1 + if depth == 0: + return index + return None + + +def draft_parameter(source, tokens, start, end, fallback_line): + if not source[start:end].strip(): + return None + try: + return parse_parameter( + source, + tokens, + start, + end, + Path(""), + fallback_line, + ) + except ConversionError: + pass + + equals = top_level_separator(tokens, start, end, "=") + declaration_end_offset = equals.start if equals else end + colon = top_level_separator(tokens, start, declaration_end_offset, ":") + name_end = colon.start if colon else declaration_end_offset + name_tokens = tokens_in_range(tokens, start, name_end) + if name_tokens and name_tokens[0].value.casefold() in PARAM_MODIFIERS: + name_tokens = name_tokens[1:] + name_token = next( + ( + token + for token in name_tokens + if is_valid_identifier_token(token) or token.value == "..." + ), + None, + ) + if name_token is None: + return None + param_type = "" + if colon: + param_type = source[colon.end : declaration_end_offset].strip() + return Parameter( + name_token.value, + param_type, + equals is not None, + name_token.line, + ) + + +def recover_signature_parts( + source, + tokens, + name_index, + scan_start, + fallback_name, + limit, +): + name_token = tokens[name_index] if name_index is not None else None + name = name_token.value if name_token is not None else fallback_name + open_index = ( + scan_start + if scan_start < limit and tokens[scan_start].value == "(" + else None + ) + close_index = ( + matching_parenthesis_in_range(tokens, open_index, limit) + if open_index is not None + else None + ) + + parameters = [] + if open_index is not None: + if close_index is not None: + parameter_end = tokens[close_index].start + else: + boundary = next_declaration_boundary(tokens, open_index, limit) + begin_index = next( + ( + index + for index in range(open_index + 1, limit) + if keyword_at(tokens, index, "begin") + ), + None, + ) + candidates = [ + index + for index in (boundary, begin_index) + if index is not None + ] + parameter_end_index = min(candidates) if candidates else limit + parameter_end = token_start_or_eof( + tokens, parameter_end_index, source + ) + if source[tokens[open_index].end : parameter_end].strip(): + for start, end in split_parameter_ranges( + tokens, + tokens[open_index].end, + parameter_end, + ): + parameter = draft_parameter( + source, + tokens, + start, + end, + tokens[open_index].line, + ) + if parameter is not None: + parameters.append(parameter) + + return_type = "" + if open_index is None or close_index is not None: + return_start_index = ( + close_index + 1 if close_index is not None else scan_start + ) + return_end_index = limit + for index in range(return_start_index, limit): + if tokens[index].value == ";" or keyword_at(tokens, index, "begin"): + return_end_index = index + break + colon_index = next( + ( + index + for index in range(return_start_index, return_end_index) + if tokens[index].value == ":" + ), + None, + ) + if colon_index is not None: + return_type = source[ + tokens[colon_index].end : token_start_or_eof( + tokens, return_end_index, source + ) + ].strip() + return name, parameters, return_type + + +def draft_function_signature(source, tokens, function_index, path, limit=None): + limit = min(limit if limit is not None else len(tokens), len(tokens)) + scan_end = declaration_scan_end(tokens, function_index, limit) + if scan_end < limit and tokens[scan_end].value == ";": + try: + name, parameters, return_type, end_index = parse_function_signature( + source, + tokens, + function_index, + Path(""), + ) + if end_index == scan_end: + return name, parameters, return_type + except ConversionError: + pass + + candidate_index = function_index + 1 + name_index = None + if candidate_index < limit: + candidate = tokens[candidate_index] + if is_valid_identifier_token(candidate): + name_index = candidate_index + scan_start = (name_index + 1) if name_index is not None else candidate_index + return recover_signature_parts( + source, + tokens, + name_index, + scan_start, + Path(path).stem, + limit, + ) + + +def draft_unknown_function(source, tokens, path): + name_index = next( + ( + index + for index in range(len(tokens) - 1) + if is_valid_identifier_token(tokens[index]) + and tokens[index + 1].value == "(" + ), + None, + ) + if name_index is None: + name = Path(path).stem + parameters = [] + return_type = "" + else: + name, parameters, return_type = recover_signature_parts( + source, + tokens, + name_index, + name_index + 1, + Path(path).stem, + len(tokens), + ) + names = ", ".join(parameter.name for parameter in parameters) + return { + "kind": "function", + "name": name, + "signature": f"{name}({names})", + "desc": "", + "params": converted_parameters(parameters), + "returns": return_type, + } + + +def draft_method_member(source, tokens, function_index, visibility, binding, limit): + name_index = function_index + 1 + if ( + name_index >= limit + or not is_valid_identifier_token(tokens[name_index]) + ): + return None, recover_after_declaration(tokens, function_index, limit) + try: + member, next_index = parse_method_member( + source, + tokens, + function_index, + Path(""), + visibility, + binding, + ) + return member, min(next_index, limit) + except ConversionError: + pass + + name, parameters, declared_return = draft_function_signature( + source, + tokens, + function_index, + Path(""), + limit, + ) + next_index = recover_after_declaration(tokens, function_index, limit) + + names = ", ".join(parameter.name for parameter in parameters) + return { + "kind": "method", + "name": name, + "visibility": visibility, + "binding": binding, + "signature": f"{name}({names})", + "desc": "", + "params": converted_parameters(parameters), + "returns": declared_return, + }, next_index + + +def draft_property_member(source, tokens, property_index, visibility, limit): + try: + member, next_index = parse_property_member( + source, tokens, property_index, Path(""), visibility + ) + return member, min(next_index, limit) + except ConversionError: + pass + + name_token = tokens[property_index + 1] if property_index + 1 < limit else None + if name_token is None or not is_valid_identifier_token(name_token): + return None, recover_after_declaration(tokens, property_index, limit) + declaration_end_index = declaration_scan_end(tokens, property_index, limit) + has_read = any( + keyword_at(tokens, index, "read") + for index in range(property_index + 2, declaration_end_index) + ) + has_write = any( + keyword_at(tokens, index, "write") + for index in range(property_index + 2, declaration_end_index) + ) + access = "readwrite" if has_read and has_write else ( + "read" if has_read else "write" if has_write else "" + ) + return { + "kind": "property", + "name": name_token.value, + "visibility": visibility, + "desc": "", + "type": "", + "params": [], + "access": access, + }, recover_after_declaration(tokens, property_index, limit) + + +def draft_field_members(source, tokens, start_index, visibility, is_static, limit): + try: + member, next_index = parse_field_member( + source, + tokens, + start_index, + Path(""), + visibility, + is_static, + ) + return [member], min(next_index, limit) + except ConversionError: + pass + + cursor = start_index + (1 if is_static else 0) + if ( + cursor < limit + and tokens[cursor].kind == "identifier" + and not is_valid_identifier_token(tokens[cursor]) + ): + return [], cursor + declaration_end_index = declaration_scan_end(tokens, start_index, limit) + colon_index = next( + ( + index + for index in range(cursor, declaration_end_index) + if tokens[index].value == ":" + ), + None, + ) + equals_index = next( + ( + index + for index in range(cursor, declaration_end_index) + if tokens[index].value == "=" + ), + None, + ) + names_end = colon_index or equals_index or declaration_end_index + names = [ + token.value + for token in tokens[cursor:names_end] + if is_valid_identifier_token(token) + ] + field_type = "" + if colon_index is not None: + type_end = equals_index or declaration_end_index + field_type = source[ + tokens[colon_index].end : token_start_or_eof(tokens, type_end, source) + ].strip() + members = [] + for name in names: + member = { + "kind": "field", + "name": name, + "visibility": visibility, + "desc": "", + "type": field_type, + } + if is_static: + member["static"] = True + members.append(member) + return members, recover_after_declaration(tokens, start_index, limit) + + +def draft_constant_members( + source, + tokens, + start_index, + visibility, + is_static, + limit, + *, + bare=False, +): + try: + if bare: + member, next_index = parse_bare_unit_constant( + source, tokens, start_index, Path("") + ) + member["visibility"] = visibility + else: + member, next_index = parse_constant_member( + source, + tokens, + start_index, + Path(""), + visibility, + is_static, + ) + return [member], min(next_index, limit) + except ConversionError: + pass + + const_index = start_index + (1 if is_static else 0) + name_start_index = const_index if bare else const_index + 1 + if name_start_index >= min(limit, len(tokens)): + return [], limit + if not is_valid_identifier_token(tokens[name_start_index]): + return [], name_start_index + declaration_end_index = declaration_scan_end( + tokens, name_start_index, limit + ) + equals_index = next( + ( + index + for index in range(name_start_index, declaration_end_index) + if tokens[index].value == "=" + ), + None, + ) + names_end = equals_index or declaration_end_index + colon_index = next( + ( + index + for index in range(name_start_index, names_end) + if tokens[index].value == ":" + ), + None, + ) + names_end = colon_index or names_end + names = [ + token.value + for token in tokens[name_start_index:names_end] + if is_valid_identifier_token(token) + ] + value = "" + if equals_index is not None: + value = source[ + tokens[equals_index].end : token_start_or_eof( + tokens, declaration_end_index, source + ) + ].strip() + constant_type = "" + if colon_index is not None and equals_index is not None: + constant_type = source[ + tokens[colon_index].end : tokens[equals_index].start + ].strip() + members = [] + for name in names: + member = { + "kind": "constant", + "name": name, + "visibility": visibility, + "desc": "", + "type": constant_type, + "value": value, + } + if is_static: + member["static"] = True + members.append(member) + return members, recover_after_declaration(tokens, name_start_index, limit) + + +def draft_class(source, tokens, type_index, path, limit=None): + limit = min(limit if limit is not None else len(tokens), len(tokens)) + name_token = tokens[type_index + 1] if type_index + 1 < limit else None + has_declared_name = ( + name_token is not None and is_valid_identifier_token(name_token) + ) + name = name_token.value if has_declared_name else Path(path).stem + class_search_start = type_index + (2 if has_declared_name else 1) + class_index = next( + ( + index + for index in range(class_search_start, min(type_index + 10, limit)) + if keyword_at(tokens, index, "class") + ), + None, + ) + if class_index is None: + return {"name": name, "desc": "", "members": []}, limit + + index = class_index + 1 + bases = [] + if index < len(tokens) and tokens[index].value == "(": + try: + bases, index, _ = parse_bases(source, tokens, class_index, Path("")) + if index > limit: + bases = [] + index = ( + next_declaration_boundary(tokens, class_index, limit) + or class_index + 1 + ) + except ConversionError: + index = ( + next_declaration_boundary(tokens, class_index, limit) + or class_index + 1 + ) + + masked_source = mask_document_lines(source) + visibility = "public" + members = [] + while index < limit: + token = tokens[index] + lowered = token.value.casefold() if token.kind == "identifier" else "" + if lowered == "end": + has_semicolon = ( + index + 1 < limit and tokens[index + 1].value == ";" + ) + next_index = index + 2 if has_semicolon else index + 1 + result = {"name": name, "desc": "", "members": members} + if bases: + result["bases"] = bases + return result, next_index + if lowered in {"public", "protected", "private"}: + visibility = lowered + index += 1 + continue + if lowered == "uses": + index = recover_after_declaration(tokens, index, limit) + continue + + member = None + parsed_members = None + if lowered == "class" and keyword_at(tokens, index + 1, "function"): + member, index = draft_method_member( + masked_source, tokens, index + 1, visibility, "class", limit + ) + elif lowered in {"function", "procedure"}: + member, index = draft_method_member( + masked_source, tokens, index, visibility, "instance", limit + ) + elif lowered == "property": + member, index = draft_property_member( + masked_source, tokens, index, visibility, limit + ) + elif lowered == "static": + if keyword_at(tokens, index + 1, "function") or keyword_at( + tokens, index + 1, "procedure" + ): + member, index = draft_method_member( + masked_source, tokens, index + 1, visibility, "class", limit + ) + elif keyword_at(tokens, index + 1, "const"): + parsed_members, index = draft_constant_members( + masked_source, tokens, index, visibility, True, limit + ) + elif keyword_at(tokens, index + 1, "property"): + member, index = draft_property_member( + masked_source, tokens, index + 1, visibility, limit + ) + else: + parsed_members, index = draft_field_members( + masked_source, tokens, index, visibility, True, limit + ) + elif lowered == "const": + parsed_members, index = draft_constant_members( + masked_source, tokens, index, visibility, False, limit + ) + elif lowered in { + "constructor", + "destructor", + }: + member, index = draft_method_member( + masked_source, tokens, index, visibility, "instance", limit + ) + elif lowered in { + "class", + "finalization", + "implementation", + "initialization", + "interface", + "type", + "unit", + "var", + }: + index = recover_after_declaration(tokens, index, limit) + elif token.kind == "identifier": + parsed_members, index = draft_field_members( + masked_source, tokens, index, visibility, False, limit + ) + else: + index += 1 + + if visibility != "private": + if member is not None: + members.append(member) + if parsed_members: + members.extend(parsed_members) + + result = {"name": name, "desc": "", "members": members} + if bases: + result["bases"] = bases + return result, limit + + +def as_unit_function(member): + if member is None: + return None + converted = dict(member) + converted["kind"] = "function" + converted.pop("visibility", None) + converted.pop("binding", None) + converted.pop("modifiers", None) + return converted + + +def draft_unit(source, tokens, unit_index, path): + name_token = tokens[unit_index + 1] if unit_index + 1 < len(tokens) else None + has_declared_name = ( + name_token is not None and is_valid_identifier_token(name_token) + ) + name = name_token.value if has_declared_name else Path(path).stem + interface_index = next( + ( + index + for index in range(unit_index + 1, len(tokens)) + if keyword_at(tokens, index, "interface") + ), + None, + ) + implementation_index = next( + ( + index + for index in range( + interface_index + 1 + if interface_index is not None + else unit_index + 1, + len(tokens), + ) + if keyword_at(tokens, index, "implementation") + ), + None, + ) + terminal_end_index = next( + ( + index + for index in range(unit_index + 1, len(tokens)) + if keyword_at(tokens, index, "end") + and index + 1 < len(tokens) + and tokens[index + 1].value == "." + ), + None, + ) + header_end_index = unit_index + (2 if has_declared_name else 1) + if header_end_index < len(tokens) and tokens[header_end_index].value == ";": + header_end_index += 1 + index = interface_index + 1 if interface_index is not None else header_end_index + limit = implementation_index or terminal_end_index or len(tokens) + masked_source = mask_document_lines(source) + members = [] + declaration_section = None + + while index < limit: + token = tokens[index] + lowered = token.value.casefold() if token.kind == "identifier" else "" + if lowered == "uses": + declaration_section = None + index = recover_after_declaration(tokens, index, limit) + continue + if lowered in {"function", "procedure"}: + declaration_section = None + member, index = draft_method_member( + masked_source, tokens, index, "public", "instance", limit + ) + member = as_unit_function(member) + if member is not None: + members.append(member) + continue + if lowered == "type": + declaration_section = None + if index + 3 < limit and keyword_at(tokens, index + 3, "class"): + class_data, index = draft_class( + masked_source, + tokens, + index, + path, + limit, + ) + members.append({"kind": "class", **class_data}) + else: + index = recover_after_declaration(tokens, index, limit) + continue + if lowered == "const": + declaration_section = "const" + constants, index = draft_constant_members( + masked_source, tokens, index, "public", False, limit + ) + for constant in constants: + constant.pop("visibility", None) + members.extend(constants) + continue + if lowered == "var": + declaration_section = "var" + if index + 1 >= limit: + index += 1 + continue + variables, index = draft_field_members( + masked_source, tokens, index + 1, "public", False, limit + ) + for variable in variables: + variable["kind"] = "variable" + variable.pop("visibility", None) + members.extend(variables) + continue + if token.kind == "identifier" and declaration_section == "const": + constants, index = draft_constant_members( + masked_source, + tokens, + index, + "public", + False, + limit, + bare=True, + ) + for constant in constants: + constant.pop("visibility", None) + members.extend(constants) + continue + if token.kind == "identifier" and declaration_section == "var": + variables, index = draft_field_members( + masked_source, tokens, index, "public", False, limit + ) + for variable in variables: + variable["kind"] = "variable" + variable.pop("visibility", None) + members.extend(variables) + continue + index = recover_after_declaration(tokens, index, limit) + + return {"name": name, "desc": "", "members": members} + + def convert_source(source, path): tokens = tokenize(source) - structure_token = first_structure_token(tokens, path) + structure_token = first_structure_token(tokens) + if structure_token is None: + return draft_unknown_function(source, tokens, path) structure_index = tokens.index(structure_token) kind = structure_token.value.casefold() - if kind == "function": - name, parameters, declared_return, begin_token = parse_declaration( - source, tokens, structure_token, path + if kind in {"function", "procedure"}: + try: + name, parameters, declared_return, begin_token = parse_declaration( + source, tokens, structure_token, path + ) + except ConversionError: + name, parameters, declared_return = draft_function_signature( + source, + tokens, + structure_index, + path, + ) + begin_token = function_begin_before_next_structure( + tokens, + structure_index + 1, + ) + document = ( + extract_document_lines(source, begin_token) + if begin_token is not None + else [] ) - document = extract_document_lines(source, begin_token, path) - function = parse_document(document, parameters, declared_return, path) + function = best_effort_function_details( + document, parameters, declared_return, path + ) + if kind == "procedure": + function["returns"] = "" names = ", ".join(parameter.name for parameter in parameters) return { "kind": "function", @@ -1437,14 +2333,18 @@ def convert_source(source, path): **function, } if kind == "type": - class_data, _ = parse_class(source, tokens, structure_index, path) + try: + class_data, _ = parse_class(source, tokens, structure_index, path) + except ConversionError: + class_data, _ = draft_class(source, tokens, structure_index, path) return {"kind": "class", **class_data} if kind == "unit": - return { - "kind": "unit", - **parse_unit(source, tokens, structure_index, path), - } - fail(path, structure_token.line, "目前支持独立顶层 function、class 和完整 unit") + try: + unit_data = parse_unit(source, tokens, structure_index, path) + except ConversionError: + unit_data = draft_unit(source, tokens, structure_index, path) + return {"kind": "unit", **unit_data} + return draft_unknown_function(source, tokens, path) def read_tsf(path): @@ -1509,30 +2409,6 @@ def serialize(data, output_format, output_path): ) -def validate_declaration_uniqueness(declarations, path): - function_signatures = set() - class_names = set() - unit_names = set() - for declaration in declarations: - kind = declaration["kind"] - if kind == "function": - key = declaration["signature"].casefold() - if key in function_signatures: - fail( - path, - None, - "重复 function signature：" - f"{declaration['signature']}", - ) - function_signatures.add(key) - continue - names = class_names if kind == "class" else unit_names - key = declaration["name"].casefold() - if key in names: - fail(path, None, f"重复 {kind}：{declaration['name']}") - names.add(key) - - def atomic_write(path, text): path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( @@ -1555,7 +2431,7 @@ def atomic_write(path, text): def build_parser(): parser = argparse.ArgumentParser( description=( - "把带标准文档块的 tsf function、class 和 unit " + "把 tsf function、class 和 unit " "转换为 json 或 yaml declarations 录入文件" ), formatter_class=ChineseHelpFormatter, @@ -1620,7 +2496,6 @@ def main(argv=None): complete_draft_fields(convert_source(read_tsf(input_path), input_path)) for input_path in input_paths ] - validate_declaration_uniqueness(declarations, output_path) data = { "module": args.module, "path": args.path, diff --git a/tools/tsl-codegen/tests/test_convert_tsf.py b/tools/tsl-codegen/tests/test_convert_tsf.py index 265a70ac..c5df88dc 100644 --- a/tools/tsl-codegen/tests/test_convert_tsf.py +++ b/tools/tsl-codegen/tests/test_convert_tsf.py @@ -237,7 +237,7 @@ class ConvertTsfCliTest(unittest.TestCase): self.read_declaration()["params"], ) - def test_parameter_type_mismatch_reports_param_line_and_preserves_output(self): + def test_parameter_type_mismatch_creates_structural_draft(self): source = self.write_tsf( "ParamTypeMismatch", """ @@ -249,32 +249,24 @@ class ConvertTsfCliTest(unittest.TestCase): end; """, ) - self.output.write_text("原内容\n", encoding="utf-8") - result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:4:", result.stderr) - self.assertIn("参数 value 的类型与函数声明不一致", result.stderr) - self.assertIn("注释为 string，声明为 integer", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual( + [{"name": "value", "type": "integer", "desc": ""}], + function["params"], + ) + self.assertEqual("integer", function["returns"]) - def test_parameter_type_braces_must_be_complete_and_nonempty(self): + def test_malformed_parameter_docs_create_structural_draft(self): cases = { - "EmptyParamType": ( - "/// @param: value {} 输入值", - "@param: 参数类型不能为空", - ), - "UnclosedParamType": ( - "/// @param: value {integer 输入值", - "@param: 参数类型缺少右花括号", - ), - "MissingParamDescription": ( - "/// @param: value {integer}", - "@param: 参数说明不能为空", - ), + "EmptyParamType": "/// @param: value {} 输入值", + "UnclosedParamType": "/// @param: value {integer 输入值", + "MissingParamDescription": "/// @param: value {integer}", } - for name, (param_line, expected) in cases.items(): + for name, param_line in cases.items(): with self.subTest(name=name): source = self.write_tsf( name, @@ -290,10 +282,14 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:4:", result.stderr) - self.assertIn(expected, result.stderr) - self.assertFalse(self.output.exists()) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual( + [{"name": "value", "type": "", "desc": ""}], + function["params"], + ) + self.assertEqual("integer", function["returns"]) def test_multiple_tsf_files_are_merged_in_input_order(self): first = self.write_tsf( @@ -514,6 +510,45 @@ class ConvertTsfCliTest(unittest.TestCase): ) self.assertEqual("", function["returns"]) + def test_legacy_document_block_creates_structural_draft(self): + source = self.write_tsf( + "LegacyDraft", + """ + function LegacyDraft(required: integer; optional = 1): array; + begin + {** + @param(required)(integer) 必填值 + @param(optional)(integer) 可选值 + @return(array) 结果 + **} + return array(); + end; + """, + ) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + { + "kind": "function", + "name": "LegacyDraft", + "signature": "LegacyDraft(required, optional)", + "desc": "", + "params": [ + {"name": "required", "type": "integer", "desc": ""}, + { + "name": "optional", + "type": "", + "optional": True, + "desc": "", + }, + ], + "returns": "array", + }, + self.read_declaration(), + ) + def test_json_draft_completes_fixed_fields_for_every_declaration_kind(self): function_source = self.write_tsf( "DraftFunction", @@ -649,7 +684,7 @@ class ConvertTsfCliTest(unittest.TestCase): self.assertNotIn("positional arguments:", result.stdout) self.assertNotIn("options:", result.stdout) - def test_return_type_mismatch_reports_directive_line_and_preserves_output(self): + def test_return_type_mismatch_creates_structural_draft(self): source = self.write_tsf( "Mismatch", """ @@ -661,17 +696,14 @@ class ConvertTsfCliTest(unittest.TestCase): end; """, ) - self.output.write_text("原内容\n", encoding="utf-8") - result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:4:", result.stderr) - self.assertIn("返回类型与函数声明不一致", result.stderr) - self.assertIn("注释为 string，声明为 integer", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual("integer", function["returns"]) - def test_document_block_must_be_first_content_after_begin(self): + def test_late_document_block_creates_structural_draft(self): source = self.write_tsf( "LateDoc", """ @@ -686,12 +718,30 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:3:", result.stderr) - self.assertIn("begin 后第一段内容必须是 /// 文档块", result.stderr) - self.assertFalse(self.output.exists()) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual("integer", function["returns"]) - def test_duplicate_enum_value_is_rejected(self): + def test_missing_body_does_not_borrow_later_function_document(self): + source = self.write_tsf( + "MissingOwnBody", + "function MissingOwnBody(): integer;\n" + "function Helper(): integer;\n" + "begin\n" + " /// 辅助函数文档。\n" + " return 1;\n" + "end;\n", + ) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("MissingOwnBody", function["name"]) + self.assertEqual("", function["desc"]) + + def test_duplicate_enum_value_creates_structural_draft(self): source = self.write_tsf( "DuplicateValue", """ @@ -709,11 +759,16 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:7:", result.stderr) - self.assertIn("枚举值重复", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual( + [{"name": "mode", "type": "integer", "desc": ""}], + function["params"], + ) + self.assertEqual("integer", function["returns"]) - def test_directives_after_examples_are_rejected(self): + def test_directives_after_examples_create_structural_draft(self): source = self.write_tsf( "BadOrder", """ @@ -730,9 +785,14 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:6:", result.stderr) - self.assertIn("示例组之后不能再写函数级指令", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + function = self.read_declaration() + self.assertEqual("", function["desc"]) + self.assertEqual( + [{"name": "value", "type": "integer", "desc": ""}], + function["params"], + ) + self.assertEqual("integer", function["returns"]) def test_class_converts_only_first_matching_public_class(self): source = self.write_tsf( @@ -982,7 +1042,7 @@ class ConvertTsfCliTest(unittest.TestCase): members = self.read_declaration()["members"] self.assertEqual(["Value"], [member["name"] for member in members]) - def test_class_rejects_unsupported_public_declaration_at_exact_line(self): + def test_class_skips_unsupported_public_declaration(self): source = self.write_tsf( "UnsupportedClass", """ @@ -992,14 +1052,10 @@ class ConvertTsfCliTest(unittest.TestCase): end; """, ) - self.output.write_text("原内容\n", encoding="utf-8") - result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:3:", result.stderr) - self.assertIn("class 不支持成员声明：type", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([], self.read_declaration()["members"]) def test_missing_class_and_unit_member_types_remain_explicit_drafts(self): cls_source = self.write_tsf( @@ -1126,7 +1182,33 @@ class ConvertTsfCliTest(unittest.TestCase): self.assertNotIn("HiddenValue", serialized) self.assertNotIn("生命周期文档", serialized) - def test_unit_filename_must_match_name_at_declaration_line(self): + def test_incomplete_interface_class_does_not_scan_implementation(self): + source = self.write_tsf( + "IncompleteInterfaceClass", + "unit IncompleteInterfaceClass;\n" + "interface\n" + "type PublicType = class\n" + "public\n" + "function Keep(): integer;\n" + "implementation\n" + "function Hidden(): integer;\n" + "begin\n" + " return 1;\n" + "end;\n" + "end.\n", + ) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + unit = self.read_declaration() + self.assertEqual(["PublicType"], [member["name"] for member in unit["members"]]) + self.assertEqual( + ["Keep"], + [member["name"] for member in unit["members"][0]["members"]], + ) + + def test_unit_filename_mismatch_keeps_declared_name(self): source = self.write_tsf( "Wrong", "unit Right;\ninterface\nimplementation\nend.\n", @@ -1134,9 +1216,8 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:1:", result.stderr) - self.assertIn("unit 名称必须与文件名一致", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("Right", self.read_declaration()["name"]) def test_unit_preserves_multiple_interface_classes_in_source_order(self): source = self.write_tsf( @@ -1162,7 +1243,7 @@ class ConvertTsfCliTest(unittest.TestCase): members = self.read_declaration()["members"] self.assertEqual(["First", "Second"], [member["name"] for member in members]) - def test_unit_unbound_document_reports_exact_line(self): + def test_unit_unbound_document_is_ignored(self): source = self.write_tsf( "UnboundUnit", """ @@ -1178,11 +1259,12 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:3:", result.stderr) - self.assertIn("unit interface 文档块无法绑定", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + current = self.read_declaration()["members"][0] + self.assertEqual("Current", current["name"]) + self.assertEqual("", current["desc"]) - def test_unit_unbound_header_document_reports_exact_line(self): + def test_unit_unbound_header_document_is_ignored(self): source = self.write_tsf( "UnboundHeaderUnit", """ @@ -1197,11 +1279,10 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:2:", result.stderr) - self.assertIn("unit interface 文档块无法绑定", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("", self.read_declaration()["desc"]) - def test_unit_end_dot_must_be_the_terminal_tokens(self): + def test_unit_ignores_tokens_after_terminal_end_dot(self): source = self.write_tsf( "TrailingUnit", """ @@ -1215,9 +1296,8 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:5:", result.stderr) - self.assertIn("end. 必须结束整个文件", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("TrailingUnit", self.read_declaration()["name"]) def test_class_missing_docs_default_visibility_and_private_filter_create_draft(self): source = self.write_tsf( @@ -1269,35 +1349,37 @@ class ConvertTsfCliTest(unittest.TestCase): members = self.read_declaration()["members"] self.assertEqual(["value_"], [member["name"] for member in members]) - def test_class_structure_errors_preserve_existing_output(self): + def test_class_structure_errors_create_best_effort_drafts(self): cases = { "WrongName": ( "type Actual = class\nend;\n", - 1, - "对外 class 名称必须与文件名一致", + "Actual", + [], ), "StaticMethod": ( "type StaticMethod = class\npublic\nstatic function Bad();\nend;\n", - 3, - "不存在 static function", + "StaticMethod", + ["Bad"], ), "ManyFields": ( "type ManyFields = class\npublic\nleft_, right_: integer;\nend;\n", - 3, - "一项一条声明", + "ManyFields", + ["left_", "right_"], ), } - for name, (text, line, expected) in cases.items(): + for name, (text, expected_name, expected_members) in cases.items(): with self.subTest(name=name): source = self.write_tsf(name, text) - self.output.write_text("原内容\n", encoding="utf-8") result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:{line}:", result.stderr) - self.assertIn(expected, result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(expected_name, declaration["name"]) + self.assertEqual( + expected_members, + [member["name"] for member in declaration["members"]], + ) - def test_class_and_unit_constants_reject_multiple_names_at_declaration_line(self): + def test_class_and_unit_constants_split_multiple_names_into_drafts(self): cases = { "ManyConstants": ( "type ManyConstants = class\npublic\nconst Left, Right = 1;\nend;\n", @@ -1312,19 +1394,19 @@ class ConvertTsfCliTest(unittest.TestCase): 3, ), } - for name, (text, line) in cases.items(): + for name, (text, _line) in cases.items(): with self.subTest(name=name): source = self.write_tsf(name, text) - self.output.write_text("原内容\n", encoding="utf-8") result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:{line}:", result.stderr) - self.assertIn("对外常量必须一项一条声明", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["Left", "Right"], + [member["name"] for member in self.read_declaration()["members"]], + ) - def test_class_rejects_procedure_at_declaration_line(self): + def test_class_converts_procedure_to_method_draft(self): source = self.write_tsf( "ProcedureClass", """ @@ -1334,16 +1416,15 @@ class ConvertTsfCliTest(unittest.TestCase): end; """, ) - self.output.write_text("原内容\n", encoding="utf-8") - result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:3:", result.stderr) - self.assertIn("class 暂不支持 procedure", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + method = self.read_declaration()["members"][0] + self.assertEqual("Open", method["name"]) + self.assertEqual("method", method["kind"]) + self.assertEqual("", method["returns"]) - def test_class_unbound_document_reports_its_line(self): + def test_class_unbound_document_is_ignored(self): source = self.write_tsf( "UnboundDoc", """ @@ -1359,35 +1440,323 @@ class ConvertTsfCliTest(unittest.TestCase): result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:4:", result.stderr) - self.assertIn("文档块无法绑定", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + field = self.read_declaration()["members"][0] + self.assertEqual("value_", field["name"]) + self.assertEqual("", field["desc"]) - def test_unit_rejects_shorthand_procedure_and_non_class_type(self): + def test_unit_converts_shorthand_and_skips_unsupported_types(self): cases = { "ShortUnit": ( "unit ShortUnit;\nfunction Open(): integer;\nbegin return 1; end;\nend.\n", - 1, - "显式 interface", + ["Open"], ), "ProcedureUnit": ( "unit ProcedureUnit;\ninterface\nprocedure Open();\nimplementation\nend.\n", - 3, - "暂不支持 procedure", + ["Open"], ), "AliasUnit": ( "unit AliasUnit;\ninterface\ntype Size = integer;\nimplementation\nend.\n", - 3, - "只支持 class type", + [], ), } - for name, (text, line, expected) in cases.items(): + for name, (text, expected_members) in cases.items(): with self.subTest(name=name): source = self.write_tsf(name, text) result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn(f"{source}:{line}:", result.stderr) - self.assertIn(expected, result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + expected_members, + [member["name"] for member in self.read_declaration()["members"]], + ) + + def test_malformed_class_and_unit_documents_create_structural_drafts(self): + cls = self.write_tsf( + "BadClassDocs", + """ + type BadClassDocs = class + public + /// 打开值 + /// @param: value {string} 输入值 + function Open(value: integer): integer; + end; + """, + ) + unit = self.write_tsf( + "BadUnitDocs", + """ + unit BadUnitDocs; + interface + /// 打开值 + /// @returns: string + function Open(): integer; + implementation + end. + """, + ) + + result = self.run_cli(cls, unit) + + self.assertEqual(0, result.returncode, result.stderr) + class_method = self.read_declarations()[0]["members"][0] + unit_function = self.read_declarations()[1]["members"][0] + self.assertEqual("", class_method["desc"]) + self.assertEqual("integer", class_method["params"][0]["type"]) + self.assertEqual("", unit_function["desc"]) + self.assertEqual("integer", unit_function["returns"]) + + def test_incomplete_class_and_unit_keep_recognizable_members(self): + cases = { + "IncompleteClass": ( + "type IncompleteClass = class\n" + "public\n" + "function Keep(value: integer): integer;\n", + "Keep", + ), + "IncompleteUnit": ( + "unit IncompleteUnit;\n" + "function Keep(value: integer): integer;\n", + "Keep", + ), + } + for name, (text, expected_member) in cases.items(): + with self.subTest(name=name): + source = self.write_tsf(name, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + [expected_member], + [member["name"] for member in self.read_declaration()["members"]], + ) + + def test_broken_member_does_not_hide_following_function(self): + cases = { + "RecoverClass": ( + "type RecoverClass = class\n" + "public\n" + "property Broken\n" + "function Keep(): integer;\n" + "end;\n", + "class", + ), + "RecoverUnit": ( + "unit RecoverUnit;\n" + "interface\n" + "const Broken\n" + "function Keep(): integer;\n" + "implementation\n" + "end.\n", + "unit", + ), + } + for name, (text, expected_kind) in cases.items(): + with self.subTest(name=name): + source = self.write_tsf(name, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(expected_kind, declaration["kind"]) + self.assertIn( + "Keep", + [member["name"] for member in declaration["members"]], + ) + + def test_unnamed_class_and_unit_functions_are_skipped(self): + cases = { + "UnnamedClassFunction": ( + "type UnnamedClassFunction = class\n" + "public\n" + "function\n" + "function Keep(): integer;\n" + "end;\n", + "class", + ), + "UnnamedUnitFunction": ( + "unit UnnamedUnitFunction;\n" + "interface\n" + "function\n" + "function Keep(): integer;\n" + "implementation\n" + "end.\n", + "unit", + ), + } + for name, (text, expected_kind) in cases.items(): + with self.subTest(name=name): + source = self.write_tsf(name, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(expected_kind, declaration["kind"]) + self.assertEqual( + ["Keep"], + [member["name"] for member in declaration["members"]], + ) + + def test_unnamed_members_do_not_hide_following_functions(self): + cases = { + "UnnamedProperty": ( + "type UnnamedProperty = class\n" + "public\n" + "property\n" + "function Keep(): integer;\n" + "end;\n", + "class", + ), + "UnnamedConstant": ( + "unit UnnamedConstant;\n" + "interface\n" + "const\n" + "function Keep(): integer;\n" + "implementation\n" + "end.\n", + "unit", + ), + } + for name, (text, expected_kind) in cases.items(): + with self.subTest(name=name): + source = self.write_tsf(name, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(expected_kind, declaration["kind"]) + self.assertEqual( + ["Keep"], + [member["name"] for member in declaration["members"]], + ) + + def test_method_missing_signature_semicolon_does_not_hide_next_member(self): + source = self.write_tsf( + "BrokenInlineMethod", + "type BrokenInlineMethod = class\n" + "public\n" + "function Broken()\n" + "begin\n" + " return 1;\n" + "end;\n" + "function Keep(): integer;\n" + "end;\n", + ) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["Broken", "Keep"], + [member["name"] for member in self.read_declaration()["members"]], + ) + + def test_broken_headers_do_not_hide_first_function(self): + cases = { + "BrokenBase": ( + "type BrokenBase = class(Base\n" + "function Keep(): integer;\n", + "class", + ), + "MissingClassName": ( + "type\n" + "class\n" + "function Keep(): integer;\n", + "class", + ), + "BrokenUnitHeader": ( + "unit BrokenUnitHeader\n" + "function Keep(): integer;\n", + "unit", + ), + "MissingUnitName": ( + "unit\n" + "function Keep(): integer;\n", + "unit", + ), + } + for name, (text, expected_kind) in cases.items(): + with self.subTest(name=name): + source = self.write_tsf(name, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(expected_kind, declaration["kind"]) + self.assertEqual( + ["Keep"], + [member["name"] for member in declaration["members"]], + ) + + def test_declaration_truncation_after_name_always_creates_a_draft(self): + module = load_script() + samples = { + "function": ( + "TruncateFunction.tsf", + "function TruncateFunction(required: integer; optional = 1): array;\n" + "begin\n" + " return array();\n" + "end;\n", + ), + "class": ( + "TruncateClass.tsf", + "type TruncateClass = class(Base)\n" + "public\n" + "function Open(value: integer): string;\n" + "property Value: integer read value_ write value_;\n" + "const Limit = 2;\n" + "value_: integer;\n" + "end;\n", + ), + "unit": ( + "TruncateUnit.tsf", + "unit TruncateUnit;\n" + "interface\n" + "function Open(value: integer): string;\n" + "const Limit = 2;\n" + "var Current: integer;\n" + "implementation\n" + "end.\n", + ), + } + for expected_kind, (filename, source) in samples.items(): + name_end = module.tokenize(source)[1].end + for cut in range(name_end, len(source) + 1): + with self.subTest(kind=expected_kind, cut=cut): + declaration = module.complete_draft_fields( + module.convert_source(source[:cut], Path(filename)) + ) + self.assertEqual(expected_kind, declaration["kind"]) + self.assertTrue(declaration["name"]) + + def test_unit_fallback_keeps_bare_const_and_var_sections(self): + source = self.write_tsf( + "RecoverSections", + """ + unit RecoverSections; + interface + /// 无法绑定 + // 切断文档绑定 + const First = 1; + Second = 2; + var Current: integer; + Previous: integer; + implementation + end. + """, + ) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["First", "Second", "Current", "Previous"], + [member["name"] for member in self.read_declaration()["members"]], + ) def test_unit_var_and_const_sections_keep_each_declaration(self): source = self.write_tsf( @@ -1458,7 +1827,7 @@ class ConvertTsfCliTest(unittest.TestCase): [item["signature"] for item in self.read_declarations()], ) - def test_duplicate_function_signature_preserves_existing_output(self): + def test_duplicate_function_signature_is_preserved_in_input_order(self): first = self.write_nested_tsf( "first", "Parse", @@ -1483,15 +1852,15 @@ class ConvertTsfCliTest(unittest.TestCase): end; """, ) - self.output.write_text("原内容\n", encoding="utf-8") - result = self.run_cli(first, second) - self.assertEqual(1, result.returncode) - self.assertIn("重复 function signature", result.stderr) - self.assertEqual("原内容\n", self.output.read_text(encoding="utf-8")) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["Parse(value)", "parse(value)"], + [item["signature"] for item in self.read_declarations()], + ) - def test_duplicate_class_and_unit_names_are_rejected_case_insensitively(self): + def test_duplicate_class_and_unit_names_are_preserved_in_input_order(self): cases = ( ( self.write_nested_tsf( @@ -1516,16 +1885,14 @@ class ConvertTsfCliTest(unittest.TestCase): "重复 unit", ), ) - for first, second, expected in cases: - with self.subTest(expected=expected): - self.output.write_text("原内容\n", encoding="utf-8") - + for first, second, kind in cases: + with self.subTest(kind=kind): result = self.run_cli(first, second) - self.assertEqual(1, result.returncode) - self.assertIn(expected, result.stderr) + self.assertEqual(0, result.returncode, result.stderr) self.assertEqual( - "原内容\n", self.output.read_text(encoding="utf-8") + 2, + len(self.read_declarations()), ) def test_same_name_across_kinds_is_allowed(self): @@ -1574,16 +1941,100 @@ class ConvertTsfCliTest(unittest.TestCase): [item["name"] for item in self.read_declarations()], ) - def test_top_level_procedure_remains_unsupported(self): + def test_top_level_procedure_creates_function_draft(self): source = self.write_tsf( "ProcedureDemo", - "procedure ProcedureDemo();\nbegin\nend;\n", + "procedure ProcedureDemo(value: integer);\nbegin\nend;\n", ) result = self.run_cli(source) - self.assertEqual(1, result.returncode) - self.assertIn("目前支持独立顶层 function、class 和完整 unit", result.stderr) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + { + "kind": "function", + "name": "ProcedureDemo", + "signature": "ProcedureDemo(value)", + "desc": "", + "params": [ + {"name": "value", "type": "integer", "desc": ""} + ], + "returns": "", + }, + self.read_declaration(), + ) + + def test_malformed_top_level_function_keeps_recognizable_signature(self): + cases = { + "MissingBody": ( + "function MissingBody(value: integer): array;\n", + "MissingBody", + [{"name": "value", "type": "integer", "desc": ""}], + "array", + ), + "BrokenParams": ( + "function BrokenParams(required: integer; optional = 1\n", + "BrokenParams", + [ + {"name": "required", "type": "integer", "desc": ""}, + { + "name": "optional", + "type": "", + "optional": True, + "desc": "", + }, + ], + "", + ), + "MissingName": ( + "function\n", + "MissingName", + [], + "", + ), + } + for filename, (text, name, params, returns) in cases.items(): + with self.subTest(filename=filename): + source = self.write_tsf(filename, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual(name, declaration["name"]) + parameter_names = ", ".join( + parameter["name"] for parameter in params + ) + self.assertEqual( + f"{name}({parameter_names})", + declaration["signature"], + ) + self.assertEqual(params, declaration["params"]) + self.assertEqual(returns, declaration["returns"]) + + def test_unrecognized_tsf_header_still_creates_function_draft(self): + cases = { + "LooseCall": ( + "LegacyName(first, second)\n", + "LegacyName", + ["first", "second"], + ), + "EmptySource": ("", "EmptySource", []), + } + for filename, (text, name, params) in cases.items(): + with self.subTest(filename=filename): + source = self.write_tsf(filename, text) + + result = self.run_cli(source) + + self.assertEqual(0, result.returncode, result.stderr) + declaration = self.read_declaration() + self.assertEqual("function", declaration["kind"]) + self.assertEqual(name, declaration["name"]) + self.assertEqual( + params, + [parameter["name"] for parameter in declaration["params"]], + ) if __name__ == "__main__":