📦 deps(thirdparty): update snapshots

2026-06-23 16:03:00 +00:00
parent c4c6a41c21
commit 59e15f8999
557 changed files with 10501 additions and 3168 deletions
@@ -0,0 +1,66 @@
+"""Path guards for local Monte Carlo template manifests."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+
+def _allow_external_paths() -> bool:
+    return os.getenv("MCD_ALLOW_EXTERNAL_PATHS", "").lower() in {"1", "true", "yes"}
+
+
+def _is_relative_to(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _resolve_local_path(raw_path: str, *, expect_file: bool = False, create_parent: bool = False) -> Path:
+    value = str(raw_path).strip()
+    if not value or "\0" in value:
+        raise ValueError("Path must be a non-empty filesystem path")
+    base = Path.cwd().resolve()
+    candidate = Path(value).expanduser()
+    resolved = (candidate if candidate.is_absolute() else base / candidate).resolve()
+    if not _allow_external_paths() and not _is_relative_to(resolved, base):
+        raise ValueError(f"Path must stay under the current working directory: {raw_path!r}")
+    if expect_file and not resolved.is_file():
+        raise FileNotFoundError(f"Input file not found: {resolved}")
+    if create_parent:
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+    return resolved
+
+
+def safe_input_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, expect_file=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Input manifest must be a .json file: {path}")
+    return path
+
+
+def safe_output_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, create_parent=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Output manifest must be a .json file: {path}")
+    return path
+
+
+def safe_existing_directory(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path)
+    if not path.is_dir():
+        raise NotADirectoryError(f"Directory not found: {path}")
+    return path
+
+
+def read_json_file(raw_path: str):
+    with safe_input_json_path(raw_path).open() as fh:
+        return json.load(fh)
+
+
+def write_json_file(raw_path: str, payload, *, indent: int = 2, default=None) -> None:
+    with safe_output_json_path(raw_path).open("w") as fh:
+        json.dump(payload, fh, indent=indent, default=default)
@@ -40,6 +40,7 @@ import os

 from collect_lineage import collect, _LOOKBACK_HOURS
 from push_lineage import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -126,6 +127,9 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments: {', '.join(missing)}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
    # Step 1: Collect
    collect(
        account=args.account,
@@ -134,17 +138,17 @@ def main() -> None:
        warehouse=args.warehouse,
        lookback_hours=args.lookback_hours,
        column_lineage=args.column_lineage,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )

    print("Done.")
@@ -34,8 +34,9 @@ Usage
 import argparse
 import os

-from collect_metadata import collect
+from collect_metadata import _quote_identifier, collect
 from push_metadata import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -111,23 +112,28 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments: {', '.join(missing)}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
+    _quote_identifier(args.warehouse)
+
    # Step 1: Collect
    collect(
        account=args.account,
        user=args.user,
        password=args.password,
        warehouse=args.warehouse,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )

    print("Done.")
@@ -36,6 +36,7 @@ import os

 from collect_query_logs import collect
 from push_query_logs import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -111,23 +112,26 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments: {', '.join(missing)}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
    # Step 1: Collect
    collect(
        account=args.account,
        user=args.user,
        password=args.password,
        warehouse=args.warehouse,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )

    print("Done.")
@@ -43,6 +43,7 @@ from dataclasses import dataclass, field
 from datetime import datetime, timezone

 import snowflake.connector
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "snowflake"
@@ -70,6 +71,13 @@ def _check_available_memory(min_gb: float = 2.0) -> None:
 # ← SUBSTITUTE: adjust the lookback window to match your collection cadence
 _LOOKBACK_HOURS = 24

+
+def _bounded_int(value: int, field: str, *, minimum: int, maximum: int) -> int:
+    value = int(value)
+    if value < minimum or value > maximum:
+        raise ValueError(f"{field} must be between {minimum} and {maximum}")
+    return value
+
 # Regex for CTAS: CREATE [OR REPLACE] [TRANSIENT] TABLE [IF NOT EXISTS] [db.][schema.]table AS SELECT
 _CTAS_RE = re.compile(
    r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:TRANSIENT\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?"
@@ -181,17 +189,19 @@ def _parse_edges(rows: list[dict]) -> list[_LineageEdge]:


 def _fetch_query_history(conn, lookback_hours: int) -> list[dict]:
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
    cursor = conn.cursor()
    cursor.execute(
-        f"""
+        """
        SELECT QUERY_ID, QUERY_TEXT, START_TIME, END_TIME, USER_NAME, DATABASE_NAME, EXECUTION_STATUS
        FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
-        WHERE START_TIME >= DATEADD(hour, -{lookback_hours}, CURRENT_TIMESTAMP())
+        WHERE START_TIME >= DATEADD(hour, -%s, CURRENT_TIMESTAMP())
          AND EXECUTION_STATUS = 'SUCCESS'
          AND QUERY_TYPE IN ('CREATE_TABLE_AS_SELECT', 'INSERT', 'MERGE', 'CREATE_VIEW')
        ORDER BY START_TIME
        LIMIT 50000
-        """
+        """,
+        (lookback_hours,),
        # ← SUBSTITUTE: adjust QUERY_TYPE list, LIMIT, or add a WHERE clause to scope to specific databases
    )
    columns = [col[0] for col in cursor.description]
@@ -220,6 +230,7 @@ def collect(
    Returns the manifest dict.
    """
    _check_available_memory()
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
    print(f"Connecting to Snowflake account: {account} ...")
    conn = snowflake.connector.connect(
        account=account,
@@ -241,8 +252,7 @@ def collect(
            "column_lineage": column_lineage,
            "edges": [],
        }
-        with open(output_file, "w") as fh:
-            json.dump(manifest, fh, indent=2)
+        write_json_file(output_file, manifest)
        return manifest

    edges = _parse_edges(rows)
@@ -271,8 +281,7 @@ def collect(
            for e in edges
        ],
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(output_file, manifest)
    print(f"Lineage manifest written to {output_file}")

    return manifest
@@ -35,6 +35,7 @@ import os
 from datetime import datetime, timezone

 import snowflake.connector
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "snowflake"
@@ -78,6 +79,13 @@ _TABLE_TYPE_MAP = {
 }


+def _quote_identifier(identifier: str) -> str:
+    value = str(identifier).strip()
+    if not value:
+        raise ValueError("Identifier must not be empty")
+    return '"' + value.replace('"', '""') + '"'
+
+
 def _normalize_table_type(raw_type: str | None) -> str:
    """Map Snowflake's TABLE_TYPE value to MC-accepted 'TABLE' or 'VIEW'."""
    if not raw_type:
@@ -115,7 +123,7 @@ def _collect_assets(conn) -> list[dict]:
    for db in databases:
        # --- Discover schemas in each database ---
        try:
-            cursor.execute(f'SHOW SCHEMAS IN DATABASE "{db}"')
+            cursor.execute("SHOW SCHEMAS IN DATABASE IDENTIFIER(%s)", (db,))
        except Exception as exc:
            print(f"  WARNING: could not list schemas in {db}: {exc}")
            continue
@@ -142,10 +150,11 @@ def _collect_assets(conn) -> list[dict]:
                    BYTES,
                    LAST_ALTERED,
                    COMMENT
-                FROM "{db}".INFORMATION_SCHEMA.TABLES
+                FROM IDENTIFIER(%s)
                WHERE TABLE_SCHEMA != 'INFORMATION_SCHEMA'
                ORDER BY TABLE_SCHEMA, TABLE_NAME
-                """
+                """,
+                (f"{db}.INFORMATION_SCHEMA.TABLES",),
            )
        except Exception as exc:
            print(f"  WARNING: could not query INFORMATION_SCHEMA.TABLES in {db}: {exc}")
@@ -172,11 +181,11 @@ def _collect_assets(conn) -> list[dict]:
                cursor.execute(
                    f"""
                    SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COMMENT
-                    FROM "{db}".INFORMATION_SCHEMA.COLUMNS
+                    FROM IDENTIFIER(%s)
                    WHERE TABLE_SCHEMA = %s
                    ORDER BY TABLE_NAME, ORDINAL_POSITION
                    """,
-                    (schema,),
+                    (f"{db}.INFORMATION_SCHEMA.COLUMNS", schema),
                )
            except Exception as exc:
                print(f"  WARNING: could not fetch columns for {db}.{schema}: {exc}")
@@ -264,8 +273,7 @@ def collect(
        "collected_at": datetime.now(tz=timezone.utc).isoformat(),
        "assets": assets,
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(output_file, manifest)
    print(f"Asset manifest written to {output_file}")

    return manifest
@@ -35,6 +35,7 @@ import os
 from datetime import datetime, timezone

 import snowflake.connector
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 # ← SUBSTITUTE: set LOG_TYPE to match your warehouse type (query logs use log_type, not resource_type)
 LOG_TYPE = "snowflake"
@@ -162,8 +163,7 @@ def collect(
            "window_end": None,
            "queries": [],
        }
-        with open(output_file, "w") as fh:
-            json.dump(manifest, fh, indent=2, default=str)
+        write_json_file(output_file, manifest, default=str)
        return manifest

    start_times = [r["START_TIME"] for r in rows if r.get("START_TIME") is not None]
@@ -189,8 +189,7 @@ def collect(
            for r in rows
        ],
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2, default=str)
+    write_json_file(output_file, manifest, default=str)
    print(f"Query log manifest written to {output_file}")

    return manifest
@@ -43,6 +43,7 @@ from pycarlo.features.ingestion.models import (
    LineageAssetRef,
    LineageEvent,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "snowflake"
@@ -155,8 +156,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    edges = manifest.get("edges", [])
    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
@@ -182,8 +182,7 @@ def push(
            "batch_count": 0,
            "batch_size": batch_size,
        }
-        with open(output_file, "w") as fh:
-            json.dump(push_result, fh, indent=2)
+        write_json_file(output_file, push_result)
        return push_result

    # Split into batches
@@ -236,8 +235,7 @@ def push(
        "batch_size": batch_size,
        "edges": edges,  # preserve for downstream validation
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    print(f"Push result written to {output_file}")

    return push_result
@@ -42,6 +42,7 @@ from pycarlo.features.ingestion.models import (
    AssetVolume,
    RelationalAsset,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: set RESOURCE_TYPE to match your Monte Carlo connection type
 RESOURCE_TYPE = "snowflake"
@@ -102,8 +103,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    asset_dicts = manifest.get("assets", [])
    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
@@ -157,8 +157,7 @@ def push(
        "batch_count": total_batches,
        "batch_size": batch_size,
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    print(f"Push result written to {output_file}")

    return push_result
@@ -37,6 +37,7 @@ from dateutil.parser import isoparse
 from pycarlo.core import Client, Session
 from pycarlo.features.ingestion import IngestionService
 from pycarlo.features.ingestion.models import QueryLogEntry
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 # ← SUBSTITUTE: set LOG_TYPE to match your warehouse type (query logs use log_type, not resource_type)
 LOG_TYPE = "snowflake"
@@ -107,8 +108,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    queries = manifest.get("queries", [])
    log_type = manifest.get("log_type", LOG_TYPE)
@@ -126,8 +126,7 @@ def push(
            "batch_count": 0,
            "batch_size": batch_size,
        }
-        with open(output_file, "w") as fh:
-            json.dump(push_result, fh, indent=2)
+        write_json_file(output_file, push_result)
        return push_result

    # Split into batches
@@ -177,8 +176,7 @@ def push(
        "batch_count": total_batches,
        "batch_size": batch_size,
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    print(f"Push result written to {output_file}")

    return push_result