📦 deps(thirdparty): update snapshots

2026-06-23 16:03:00 +00:00
parent c4c6a41c21
commit 59e15f8999
557 changed files with 10501 additions and 3168 deletions
@@ -0,0 +1,66 @@
+"""Path guards for local Monte Carlo template manifests."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+
+def _allow_external_paths() -> bool:
+    return os.getenv("MCD_ALLOW_EXTERNAL_PATHS", "").lower() in {"1", "true", "yes"}
+
+
+def _is_relative_to(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _resolve_local_path(raw_path: str, *, expect_file: bool = False, create_parent: bool = False) -> Path:
+    value = str(raw_path).strip()
+    if not value or "\0" in value:
+        raise ValueError("Path must be a non-empty filesystem path")
+    base = Path.cwd().resolve()
+    candidate = Path(value).expanduser()
+    resolved = (candidate if candidate.is_absolute() else base / candidate).resolve()
+    if not _allow_external_paths() and not _is_relative_to(resolved, base):
+        raise ValueError(f"Path must stay under the current working directory: {raw_path!r}")
+    if expect_file and not resolved.is_file():
+        raise FileNotFoundError(f"Input file not found: {resolved}")
+    if create_parent:
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+    return resolved
+
+
+def safe_input_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, expect_file=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Input manifest must be a .json file: {path}")
+    return path
+
+
+def safe_output_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, create_parent=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Output manifest must be a .json file: {path}")
+    return path
+
+
+def safe_existing_directory(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path)
+    if not path.is_dir():
+        raise NotADirectoryError(f"Directory not found: {path}")
+    return path
+
+
+def read_json_file(raw_path: str):
+    with safe_input_json_path(raw_path).open() as fh:
+        return json.load(fh)
+
+
+def write_json_file(raw_path: str, payload, *, indent: int = 2, default=None) -> None:
+    with safe_output_json_path(raw_path).open("w") as fh:
+        json.dump(payload, fh, indent=indent, default=default)
@@ -34,6 +34,7 @@ import os

 from collect_lineage import collect
 from push_lineage import DEFAULT_BATCH_SIZE, DEFAULT_TIMEOUT_SECONDS, push
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file


 def main() -> None:
@@ -109,8 +110,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Lineage manifest written to {args.output_file}")
    print("Done.")

@@ -30,8 +30,9 @@ import argparse
 import json
 import os

-from collect_metadata import collect
+from collect_metadata import _bounded_int, collect
 from push_metadata import DEFAULT_BATCH_SIZE, DEFAULT_TIMEOUT_SECONDS, push
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file


 def main() -> None:
@@ -95,6 +96,8 @@ def main() -> None:
    if not args.resource_uuid:
        parser.error("--resource-uuid is required (or set MCD_RESOURCE_UUID)")

+    args.hive_port = _bounded_int(args.hive_port, "hive_port", minimum=1, maximum=65535)
+
    manifest = collect(
        hive_host=args.hive_host,
        hive_port=args.hive_port,
@@ -109,8 +112,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Manifest written to {args.output_file}")
    print("Done.")

@@ -35,6 +35,7 @@ import os

 from collect_query_logs import collect
 from push_query_logs import DEFAULT_BATCH_SIZE, DEFAULT_TIMEOUT_SECONDS, push
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file


 def main() -> None:
@@ -107,8 +108,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Query log manifest written to {args.output_file}")
    print("Done.")

@@ -31,6 +31,7 @@ import json
 import re
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "data-lake"
@@ -255,8 +256,7 @@ def main() -> None:
        print("No lineage edges detected — no CTAS or INSERT INTO ... SELECT patterns found.")
        return

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Lineage manifest written to {args.output_file}")
    print("Done.")

@@ -31,6 +31,7 @@ import re
 from datetime import datetime, timezone

 from pyhive import hive
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file


 def _check_available_memory(min_gb: float = 2.0) -> None:
@@ -82,6 +83,47 @@ _HIVE_TYPE_MAP: dict[str, str] = {

 # ← SUBSTITUTE: add any internal table name prefixes you want to skip
 _INTERNAL_TABLE_PREFIXES = ("tmp_", "__", "hive_")
+_SAFE_HIVE_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def _safe_hive_identifier(identifier: str) -> str:
+    value = str(identifier).strip()
+    if not value:
+        raise ValueError("Hive identifier must not be empty")
+    match = _SAFE_HIVE_IDENTIFIER_RE.fullmatch(value)
+    if not match:
+        raise ValueError("Hive identifier contains characters outside the safe default set")
+    return match.group(0)
+
+
+def _safe_hive_identifier_from_row(row: tuple, index: int = 0) -> str:
+    value = str(row[index]).strip()
+    match = _SAFE_HIVE_IDENTIFIER_RE.fullmatch(value)
+    if not match:
+        raise ValueError("Hive identifier contains characters outside the safe default set")
+    return match.group(0)
+
+
+def _quote_hive_identifier(identifier: str) -> str:
+    value = str(identifier).strip()
+    if not value:
+        raise ValueError("Hive identifier must not be empty")
+    allow_extended = os.getenv("HIVE_ALLOW_EXTENDED_IDENTIFIERS", "").lower() in {"1", "true", "yes"}
+    if not allow_extended:
+        value = _safe_hive_identifier(value)
+    elif not _SAFE_HIVE_IDENTIFIER_RE.fullmatch(value):
+        raise ValueError(
+            "Hive identifier contains characters outside the safe default set; "
+            "set HIVE_ALLOW_EXTENDED_IDENTIFIERS=1 to use escaped extended identifiers"
+        )
+    return "`" + value.replace("`", "``") + "`"
+
+
+def _bounded_int(value: int, field: str, *, minimum: int, maximum: int) -> int:
+    value = int(value)
+    if value < minimum or value > maximum:
+        raise ValueError(f"{field} must be between {minimum} and {maximum}")
+    return value


 def _normalize_hive_type(hive_type: str) -> str:
@@ -101,9 +143,8 @@ def _connect(host: str, port: int) -> hive.Connection:
    return hive.connect(host=host, port=port, username="hadoop", auth="NONE")


-def _fetch_rows(cursor, query: str) -> list[tuple]:
-    """Execute a query and fetch results in memory-safe chunks."""
-    cursor.execute(query)
+def _fetch_rows(cursor) -> list[tuple]:
+    """Fetch query results in memory-safe chunks."""
    rows: list[tuple] = []
    while True:
        chunk = cursor.fetchmany(1000)
@@ -207,13 +248,15 @@ def collect(
        Manifest dict with keys: resource_type, collected_at, assets.
    """
    _check_available_memory()
+    hive_port = _bounded_int(hive_port, "hive_port", minimum=1, maximum=65535)
    print(f"Connecting to HiveServer2 at {hive_host}:{hive_port} ...")
    conn = _connect(hive_host, hive_port)
    cursor = conn.cursor()
    assets: list[dict] = []

    print("Collecting table metadata ...")
-    databases = [row[0] for row in _fetch_rows(cursor, "SHOW DATABASES")]
+    cursor.execute("SHOW DATABASES")
+    databases = [_safe_hive_identifier_from_row(row) for row in _fetch_rows(cursor)]
    print(f"  Found databases: {databases}")

    for db in databases:
@@ -221,8 +264,13 @@ def collect(
        if db in ("information_schema",):
            continue

-        tables = _fetch_rows(cursor, f"SHOW TABLES IN {db}")
-        table_names = [row[0] for row in tables]
+        db_match = _SAFE_HIVE_IDENTIFIER_RE.fullmatch(db)
+        if not db_match:
+            raise ValueError("Hive database identifier contains characters outside the safe default set")
+        quoted_db = f"`{db_match.group(0)}`"
+        cursor.execute(f"SHOW TABLES IN {quoted_db}")
+        tables = _fetch_rows(cursor)
+        table_names = [_safe_hive_identifier_from_row(row) for row in tables]
        print(f"  {db}: {len(table_names)} table(s)")

        for table in table_names:
@@ -230,7 +278,12 @@ def collect(
                continue

            try:
-                desc_rows = _fetch_rows(cursor, f"DESCRIBE FORMATTED {db}.{table}")
+                table_match = _SAFE_HIVE_IDENTIFIER_RE.fullmatch(table)
+                if not table_match:
+                    raise ValueError("Hive table identifier contains characters outside the safe default set")
+                quoted_table = f"`{table_match.group(0)}`"
+                cursor.execute(f"DESCRIBE FORMATTED {quoted_db}.{quoted_table}")
+                desc_rows = _fetch_rows(cursor)
            except Exception as exc:
                print(f"    WARNING: could not describe {db}.{table}: {exc}")
                continue
@@ -303,8 +356,7 @@ def main() -> None:
        hive_port=args.hive_port,
    )

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Asset manifest written to {args.output_file}")
    print("Done.")

@@ -133,7 +133,7 @@ def _load_returned_rows(op_logs_dir: str) -> dict[str, int]:
    each file, which reflects the final number of rows delivered to the client.
    """
    rows_by_id: dict[str, int] = {}
-    for log_file in Path(op_logs_dir).glob("*.log"):
+    for log_file in safe_existing_directory(op_logs_dir).glob("*.log"):
        query_id = log_file.stem
        last_count: int | None = None
        try:
@@ -193,6 +193,7 @@ def collect(
        op_logs_dir: Optional directory containing per-query operation logs
                     (<queryId>.log). When provided, returned_rows is populated
                     from SelectOperator RECORDS_OUT counts.
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

    Returns:
        Manifest dict with keys: log_type, collected_at, entry_count,
@@ -274,8 +275,7 @@ def main() -> None:

    manifest = collect(log_file=args.log_file, op_logs_dir=args.op_logs_dir)

-    with open(args.output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.output_file, manifest)
    print(f"Query log manifest written to {args.output_file}")
    print("Done.")

@@ -43,6 +43,7 @@ from pycarlo.features.ingestion.models import (
    LineageAssetRef,
    LineageEvent,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "data-lake"
@@ -286,8 +287,7 @@ def main() -> None:
    if not args.resource_uuid:
        parser.error("--resource-uuid is required (or set MCD_RESOURCE_UUID)")

-    with open(args.input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(args.input_file)

    push(
        manifest=manifest,
@@ -299,8 +299,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.input_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.input_file, manifest)
    print(f"Manifest updated in-place: {args.input_file}")
    print("Done.")

@@ -43,6 +43,7 @@ from pycarlo.features.ingestion.models import (
    AssetVolume,
    RelationalAsset,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: default batch size for metadata push (assets per request)
 DEFAULT_BATCH_SIZE = 500
@@ -223,8 +224,7 @@ def main() -> None:
    if not args.resource_uuid:
        parser.error("--resource-uuid is required (or set MCD_RESOURCE_UUID)")

-    with open(args.input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(args.input_file)

    push(
        manifest=manifest,
@@ -235,8 +235,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.input_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.input_file, manifest)
    print(f"Manifest updated in-place: {args.input_file}")
    print("Done.")

@@ -39,6 +39,7 @@ from dateutil.parser import isoparse
 from pycarlo.core import Client, Session
 from pycarlo.features.ingestion import IngestionService
 from pycarlo.features.ingestion.models import QueryLogEntry
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: default batch size for query log push (events per request)
 # Query logs include full SQL text — keep batches small to stay under the 1 MB
@@ -233,8 +234,7 @@ def main() -> None:
    if not args.key_id or not args.key_token:
        parser.error("--key-id and --key-token are required (or set MCD_INGEST_ID / MCD_INGEST_TOKEN)")

-    with open(args.input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(args.input_file)

    push(
        manifest=manifest,
@@ -245,8 +245,7 @@ def main() -> None:
        timeout_seconds=args.timeout,
    )

-    with open(args.input_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(args.input_file, manifest)
    print(f"Manifest updated in-place: {args.input_file}")
    print("Done.")