📦 deps(thirdparty): update snapshots

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/_safe_paths.py

@@ -0,0 +1,66 @@
+"""Path guards for local Monte Carlo template manifests."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+
+def _allow_external_paths() -> bool:
+    return os.getenv("MCD_ALLOW_EXTERNAL_PATHS", "").lower() in {"1", "true", "yes"}
+
+
+def _is_relative_to(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _resolve_local_path(raw_path: str, *, expect_file: bool = False, create_parent: bool = False) -> Path:
+    value = str(raw_path).strip()
+    if not value or "\0" in value:
+        raise ValueError("Path must be a non-empty filesystem path")
+    base = Path.cwd().resolve()
+    candidate = Path(value).expanduser()
+    resolved = (candidate if candidate.is_absolute() else base / candidate).resolve()
+    if not _allow_external_paths() and not _is_relative_to(resolved, base):
+        raise ValueError(f"Path must stay under the current working directory: {raw_path!r}")
+    if expect_file and not resolved.is_file():
+        raise FileNotFoundError(f"Input file not found: {resolved}")
+    if create_parent:
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+    return resolved
+
+
+def safe_input_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, expect_file=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Input manifest must be a .json file: {path}")
+    return path
+
+
+def safe_output_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, create_parent=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Output manifest must be a .json file: {path}")
+    return path
+
+
+def safe_existing_directory(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path)
+    if not path.is_dir():
+        raise NotADirectoryError(f"Directory not found: {path}")
+    return path
+
+
+def read_json_file(raw_path: str):
+    with safe_input_json_path(raw_path).open() as fh:
+        return json.load(fh)
+
+
+def write_json_file(raw_path: str, payload, *, indent: int = 2, default=None) -> None:
+    with safe_output_json_path(raw_path).open("w") as fh:
+        json.dump(payload, fh, indent=indent, default=default)

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_and_push_lineage.py

@@ -30,6 +30,7 @@ import os
 
 from collect_lineage import LOOKBACK_DAYS, collect
 from push_lineage import DEFAULT_BATCH_SIZE, push
+from _safe_paths import safe_output_json_path
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -57,19 +58,21 @@ def main() -> None:
     if missing:
         parser.error(f"Missing required arguments/env vars: {missing}")
 
+    manifest_path = str(safe_output_json_path(args.manifest))
+
     log.info("Step 1: Collecting lineage …")
     collect(
         host=args.host,
         http_path=args.http_path,
         token=args.token,
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
         include_column_lineage=args.column_lineage,
         lookback_days=args.lookback_days,
     )
 
     log.info("Step 2: Pushing lineage to Monte Carlo …")
     push(
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
         resource_uuid=args.resource_uuid,
         key_id=args.key_id,
         key_token=args.key_token,

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_and_push_metadata.py

@@ -27,8 +27,9 @@ import argparse
 import logging
 import os
 
-from collect_metadata import collect
+from collect_metadata import _quote_identifier, collect
 from push_metadata import DEFAULT_BATCH_SIZE, push
+from _safe_paths import safe_output_json_path
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -52,18 +53,22 @@ def main() -> None:
     if missing:
         parser.error(f"Missing required arguments/env vars: {missing}")
 
+    manifest_path = str(safe_output_json_path(args.manifest))
+
+    _quote_identifier(args.catalog)
+
     log.info("Step 1: Collecting metadata …")
     collect(
         host=args.host,
         http_path=args.http_path,
         token=args.token,
         catalog=args.catalog,
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
     )
 
     log.info("Step 2: Pushing metadata to Monte Carlo …")
     push(
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
         resource_uuid=args.resource_uuid,
         key_id=args.key_id,
         key_token=args.key_token,

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_and_push_query_logs.py

@@ -31,6 +31,7 @@ import os
 
 from collect_query_logs import LOOKBACK_HOURS, LOOKBACK_LAG_HOURS, MAX_ROWS, collect
 from push_query_logs import DEFAULT_BATCH_SIZE, push
+from _safe_paths import safe_output_json_path
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -56,12 +57,14 @@ def main() -> None:
     if missing:
         parser.error(f"Missing required arguments/env vars: {missing}")
 
+    manifest_path = str(safe_output_json_path(args.manifest))
+
     log.info("Step 1: Collecting query logs …")
     collect(
         host=args.host,
         http_path=args.http_path,
         token=args.token,
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
         lookback_hours=args.lookback_hours,
         lookback_lag_hours=args.lookback_lag_hours,
         max_rows=args.max_rows,
@@ -69,7 +72,7 @@ def main() -> None:
 
     log.info("Step 2: Pushing query logs to Monte Carlo …")
     push(
-        manifest_path=args.manifest,
+        manifest_path=manifest_path,
         resource_uuid=args.resource_uuid,
         key_id=args.key_id,
         key_token=args.key_token,

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_lineage.py

@@ -29,6 +29,7 @@ from datetime import datetime, timezone
 from typing import Any
 
 from databricks import sql
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -37,6 +38,13 @@ RESOURCE_TYPE = "databricks"
 LOOKBACK_DAYS: int = int(os.getenv("LOOKBACK_DAYS", "30"))  # ← SUBSTITUTE
 
 
+def _bounded_int(value: int, field: str, *, minimum: int, maximum: int) -> int:
+    value = int(value)
+    if value < minimum or value > maximum:
+        raise ValueError(f"{field} must be between {minimum} and {maximum}")
+    return value
+
+
 def _check_available_memory(min_gb: float = 2.0) -> None:
     """Warn if available memory is below the threshold."""
     try:
@@ -80,6 +88,7 @@ def _parse_full_name(full_name: str) -> tuple[str, str, str]:
 
 
 def collect_table_lineage(cursor: Any, lookback_days: int) -> list[dict[str, Any]]:
+    lookback_days = _bounded_int(lookback_days, "lookback_days", minimum=1, maximum=366)
     rows = _query(
         cursor,
         f"""
@@ -114,6 +123,7 @@ def collect_table_lineage(cursor: Any, lookback_days: int) -> list[dict[str, Any
 
 
 def collect_column_lineage(cursor: Any, lookback_days: int) -> list[dict[str, Any]]:
+    lookback_days = _bounded_int(lookback_days, "lookback_days", minimum=1, maximum=366)
     rows = _query(
         cursor,
         f"""
@@ -176,6 +186,7 @@ def collect(
 ) -> list[dict[str, Any]]:
     """Connect to Databricks, collect lineage, write a JSON manifest, and return events."""
     _check_available_memory(min_gb=2.0)
+    lookback_days = _bounded_int(lookback_days, "lookback_days", minimum=1, maximum=366)
     collected_at = datetime.now(timezone.utc).isoformat()
 
     with sql.connect(
@@ -201,8 +212,7 @@ def collect(
         "column_lineage_events": len(col_events),
         "events": all_events,
     }
-    with open(manifest_path, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(manifest_path, manifest)
     log.info("Manifest written to %s (%d events)", manifest_path, len(all_events))
 
     return all_events

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_metadata.py

@@ -22,15 +22,18 @@ import argparse
 import json
 import logging
 import os
+import re
 from datetime import datetime, timezone
 from typing import Any
 
 from databricks import sql
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
 
 RESOURCE_TYPE = "databricks"
+_SAFE_DATABRICKS_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
 # Schemas to skip across all catalogs
 SCHEMA_EXCLUSIONS: set[str] = {  # ← SUBSTITUTE: add any internal schemas to skip
@@ -39,6 +42,21 @@ SCHEMA_EXCLUSIONS: set[str] = {  # ← SUBSTITUTE: add any internal schemas to s
 }
 
 
+def _quote_identifier(identifier: str) -> str:
+    value = str(identifier).strip()
+    if not value:
+        raise ValueError("Identifier must not be empty")
+    if not _SAFE_DATABRICKS_IDENTIFIER_RE.fullmatch(value):
+        raise ValueError(
+            "Databricks identifier contains characters outside the safe default set"
+        )
+    return "`" + value.replace("`", "``") + "`"
+
+
+def _sql_literal(value: str) -> str:
+    return "'" + str(value).replace("'", "''") + "'"
+
+
 def _check_available_memory(min_gb: float = 2.0) -> None:
     """Warn if available memory is below the threshold."""
     try:
@@ -59,8 +77,7 @@ def _check_available_memory(min_gb: float = 2.0) -> None:
         )
 
 
-def _query(cursor: Any, sql_text: str, params: tuple | None = None) -> list[dict[str, Any]]:
-    cursor.execute(sql_text, params)
+def _fetch_dict_rows(cursor: Any) -> list[dict[str, Any]]:
     cols = [d[0] for d in cursor.description]
     rows = []
     while True:
@@ -72,32 +89,40 @@ def _query(cursor: Any, sql_text: str, params: tuple | None = None) -> list[dict
 
 
 def collect_tables(cursor: Any, catalog: str) -> list[dict[str, Any]]:
-    return _query(
-        cursor,
+    exclusions = sorted(SCHEMA_EXCLUSIONS)
+    placeholders = ", ".join(["%s"] * len(exclusions))
+    cursor.execute(
         f"""
         SELECT table_catalog, table_schema, table_name, table_type, comment
-        FROM {catalog}.information_schema.tables
-        WHERE table_schema NOT IN ({", ".join(f"'{s}'" for s in SCHEMA_EXCLUSIONS)})
+        FROM system.information_schema.tables
+        WHERE table_catalog = %s AND table_schema NOT IN ({placeholders})
         ORDER BY table_schema, table_name
         """,  # ← SUBSTITUTE: add additional WHERE filters if needed
+        (catalog, *exclusions),
     )
+    return _fetch_dict_rows(cursor)
 
 
 def collect_columns(cursor: Any, catalog: str, schema: str, table: str) -> list[dict[str, Any]]:
-    return _query(
-        cursor,
-        f"""
+    cursor.execute(
+        """
         SELECT column_name, data_type, comment
-        FROM {catalog}.information_schema.columns
-        WHERE table_schema = '{schema}' AND table_name = '{table}'
+        FROM system.information_schema.columns
+        WHERE table_catalog = %s AND table_schema = %s AND table_name = %s
         ORDER BY ordinal_position
         """,
+        (catalog, schema, table),
     )
+    return _fetch_dict_rows(cursor)
 
 
 def collect_detail(cursor: Any, catalog: str, schema: str, table: str) -> dict[str, Any] | None:
     try:
-        rows = _query(cursor, f"DESCRIBE DETAIL `{catalog}`.`{schema}`.`{table}`")
+        cursor.execute(
+            "DESCRIBE DETAIL "
+            f"{_quote_identifier(catalog)}.{_quote_identifier(schema)}.{_quote_identifier(table)}",
+        )
+        rows = _fetch_dict_rows(cursor)
         return rows[0] if rows else None
     except Exception:
         log.debug("DESCRIBE DETAIL failed for %s.%s.%s", catalog, schema, table, exc_info=True)
@@ -178,8 +203,7 @@ def collect(
         "asset_count": len(assets),
         "assets": assets,
     }
-    with open(manifest_path, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(manifest_path, manifest)
     log.info("Manifest written to %s (%d assets)", manifest_path, len(assets))
 
     return assets

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/collect_query_logs.py

@@ -27,6 +27,7 @@ from datetime import datetime, timezone
 from typing import Any
 
 from databricks import sql
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -57,6 +58,13 @@ LIMIT {max_rows}
 """  # ← SUBSTITUTE: adjust status filter or add warehouse_id filter as needed
 
 
+def _bounded_int(value: int, field: str, *, minimum: int, maximum: int) -> int:
+    value = int(value)
+    if value < minimum or value > maximum:
+        raise ValueError(f"{field} must be between {minimum} and {maximum}")
+    return value
+
+
 def _check_available_memory(min_gb: float = 2.0) -> None:
     """Warn if available memory is below the threshold."""
     try:
@@ -105,6 +113,9 @@ def collect_query_logs(
     lag_hours: int,
     max_rows: int,
 ) -> list[dict[str, Any]]:
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
+    lag_hours = _bounded_int(lag_hours, "lag_hours", minimum=0, maximum=24 * 7)
+    max_rows = _bounded_int(max_rows, "max_rows", minimum=1, maximum=100000)
     rendered_sql = _QUERY_LOG_SQL.format(
         lookback_hours=lookback_hours + lag_hours,  # offset from NOW() to cover the window
         lag_hours=lag_hours,
@@ -146,6 +157,9 @@ def collect(
 ) -> list[dict[str, Any]]:
     """Connect to Databricks, collect query logs, write a JSON manifest, and return entries."""
     _check_available_memory(min_gb=2.0)
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
+    lookback_lag_hours = _bounded_int(lookback_lag_hours, "lookback_lag_hours", minimum=0, maximum=24 * 7)
+    max_rows = _bounded_int(max_rows, "max_rows", minimum=1, maximum=100000)
     collected_at = datetime.now(timezone.utc).isoformat()
 
     with sql.connect(
@@ -166,8 +180,7 @@ def collect(
         "query_log_count": len(entries),
         "entries": entries,
     }
-    with open(manifest_path, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(manifest_path, manifest)
     log.info("Manifest written to %s (%d entries)", manifest_path, len(entries))
 
     return entries

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/push_lineage.py

@@ -32,6 +32,7 @@ from pycarlo.features.ingestion.models import (
     LineageAssetRef,
     LineageEvent,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -96,8 +97,7 @@ def push(
 
     Returns a summary dict with invocation IDs and counts.
     """
-    with open(manifest_path) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(manifest_path)
 
     event_dicts: list[dict[str, Any]] = manifest["events"]
     events = [_event_from_dict(d) for d in event_dicts]
@@ -158,8 +158,7 @@ def push(
     }
 
     push_manifest_path = manifest_path.replace(".json", "_push_result.json")
-    with open(push_manifest_path, "w") as fh:
-        json.dump(summary, fh, indent=2)
+    write_json_file(push_manifest_path, summary)
     log.info("Push result written to %s", push_manifest_path)
 
     return summary

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/push_metadata.py

@@ -33,6 +33,7 @@ from pycarlo.features.ingestion.models import (
     AssetVolume,
     RelationalAsset,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -85,8 +86,7 @@ def push(
 
     Returns a summary dict with invocation IDs and counts.
     """
-    with open(manifest_path) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(manifest_path)
 
     asset_dicts: list[dict[str, Any]] = manifest["assets"]
     assets = [_asset_from_dict(d) for d in asset_dicts]
@@ -144,8 +144,7 @@ def push(
 
     # Write push result alongside the collect manifest
     push_manifest_path = manifest_path.replace(".json", "_push_result.json")
-    with open(push_manifest_path, "w") as fh:
-        json.dump(summary, fh, indent=2)
+    write_json_file(push_manifest_path, summary)
     log.info("Push result written to %s", push_manifest_path)
 
     return summary

antigravity-awesome-skills/skills/monte-carlo-push-ingestion/scripts/templates/databricks/push_query_logs.py

@@ -28,6 +28,7 @@ from dateutil.parser import isoparse
 from pycarlo.core import Client, Session
 from pycarlo.features.ingestion import IngestionService
 from pycarlo.features.ingestion.models import QueryLogEntry
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -91,8 +92,7 @@ def push(
 
     Returns a summary dict with invocation IDs and counts.
     """
-    with open(manifest_path) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(manifest_path)
 
     entry_dicts: list[dict[str, Any]] = manifest["entries"]
     entries = _build_query_log_entries(entry_dicts)
@@ -110,8 +110,7 @@ def push(
             "batch_size": batch_size,
         }
         push_manifest_path = manifest_path.replace(".json", "_push_result.json")
-        with open(push_manifest_path, "w") as fh:
-            json.dump(summary, fh, indent=2)
+        write_json_file(push_manifest_path, summary)
         return summary
 
     # Split into batches
@@ -166,8 +165,7 @@ def push(
     }
 
     push_manifest_path = manifest_path.replace(".json", "_push_result.json")
-    with open(push_manifest_path, "w") as fh:
-        json.dump(summary, fh, indent=2)
+    write_json_file(push_manifest_path, summary)
     log.info("Push result written to %s", push_manifest_path)
 
     return summary