📦 deps(thirdparty): update snapshots

2026-06-23 16:03:00 +00:00
parent c4c6a41c21
commit 59e15f8999
557 changed files with 10501 additions and 3168 deletions
@@ -0,0 +1,66 @@
+"""Path guards for local Monte Carlo template manifests."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+
+def _allow_external_paths() -> bool:
+    return os.getenv("MCD_ALLOW_EXTERNAL_PATHS", "").lower() in {"1", "true", "yes"}
+
+
+def _is_relative_to(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _resolve_local_path(raw_path: str, *, expect_file: bool = False, create_parent: bool = False) -> Path:
+    value = str(raw_path).strip()
+    if not value or "\0" in value:
+        raise ValueError("Path must be a non-empty filesystem path")
+    base = Path.cwd().resolve()
+    candidate = Path(value).expanduser()
+    resolved = (candidate if candidate.is_absolute() else base / candidate).resolve()
+    if not _allow_external_paths() and not _is_relative_to(resolved, base):
+        raise ValueError(f"Path must stay under the current working directory: {raw_path!r}")
+    if expect_file and not resolved.is_file():
+        raise FileNotFoundError(f"Input file not found: {resolved}")
+    if create_parent:
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+    return resolved
+
+
+def safe_input_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, expect_file=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Input manifest must be a .json file: {path}")
+    return path
+
+
+def safe_output_json_path(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path, create_parent=True)
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Output manifest must be a .json file: {path}")
+    return path
+
+
+def safe_existing_directory(raw_path: str) -> Path:
+    path = _resolve_local_path(raw_path)
+    if not path.is_dir():
+        raise NotADirectoryError(f"Directory not found: {path}")
+    return path
+
+
+def read_json_file(raw_path: str):
+    with safe_input_json_path(raw_path).open() as fh:
+        return json.load(fh)
+
+
+def write_json_file(raw_path: str, payload, *, indent: int = 2, default=None) -> None:
+    with safe_output_json_path(raw_path).open("w") as fh:
+        json.dump(payload, fh, indent=indent, default=default)
@@ -20,8 +20,9 @@ from __future__ import annotations
 import argparse
 import os

-from collect_lineage import collect, LOOKBACK_HOURS
+from collect_lineage import LOOKBACK_HOURS, _bounded_int, _require_bq_identifier, collect
 from push_lineage import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -47,22 +48,29 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments/env vars: {missing}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
+    args.project_id = _require_bq_identifier(args.project_id, "project_id")
+    args.region = _require_bq_identifier(args.region, "region")
+    args.lookback_hours = _bounded_int(args.lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
+
    # Step 1: Collect
    collect(
        project_id=args.project_id,
        region=args.region,
        lookback_hours=args.lookback_hours,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )


@@ -22,6 +22,7 @@ import os

 from collect_metadata import collect
 from push_metadata import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -44,20 +45,23 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments/env vars: {missing}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
    # Step 1: Collect
    collect(
        project_id=args.project_id,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )


@@ -22,6 +22,7 @@ import os

 from collect_query_logs import collect, LOOKBACK_HOURS, LOOKBACK_LAG_HOURS
 from push_query_logs import push, _BATCH_SIZE
+from _safe_paths import safe_output_json_path


 def main() -> None:
@@ -47,22 +48,25 @@ def main() -> None:
    if missing:
        parser.error(f"Missing required arguments/env vars: {missing}")

+    output_path = str(safe_output_json_path(args.output_file))
+    push_result_path = str(safe_output_json_path(args.push_result_file))
+
    # Step 1: Collect
    collect(
        project_id=args.project_id,
        lookback_hours=args.lookback_hours,
        lookback_lag_hours=args.lookback_lag_hours,
-        output_file=args.output_file,
+        output_file=output_path,
    )

    # Step 2: Push
    push(
-        input_file=args.output_file,
+        input_file=output_path,
        resource_uuid=args.resource_uuid,
        key_id=args.key_id,
        key_token=args.key_token,
        batch_size=args.batch_size,
-        output_file=args.push_result_file,
+        output_file=push_result_path,
    )


@@ -29,12 +29,28 @@ import re
 from datetime import datetime, timedelta, timezone

 from google.cloud import bigquery
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)

 RESOURCE_TYPE = "bigquery"
 LOOKBACK_HOURS = int(os.getenv("LOOKBACK_HOURS", "24"))  # ← SUBSTITUTE: adjust lookback window
+_BQ_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+def _require_bq_identifier(value: str, field: str) -> str:
+    value = str(value).strip()
+    if not value or not _BQ_IDENTIFIER_RE.fullmatch(value):
+        raise ValueError(f"Invalid BigQuery {field}: {value!r}")
+    return value
+
+
+def _bounded_int(value: int, field: str, *, minimum: int, maximum: int) -> int:
+    value = int(value)
+    if value < minimum or value > maximum:
+        raise ValueError(f"{field} must be between {minimum} and {maximum}")
+    return value

 # Regex patterns to detect CTAS and INSERT INTO SELECT in BigQuery SQL
 _CTAS_PATTERN = re.compile(
@@ -65,6 +81,8 @@ def _collect_schema_link_lineage(
    region: str,
 ) -> list[dict]:
    """Collect cross-project lineage from INFORMATION_SCHEMA.SCHEMATA_LINKS."""
+    project_id = _require_bq_identifier(project_id, "project_id")
+    region = _require_bq_identifier(region, "region")
    query = f"""
        SELECT
            CATALOG_NAME            AS source_project,
@@ -103,6 +121,8 @@ def _collect_query_lineage(
    lookback_hours: int,
 ) -> list[dict]:
    """Derive lineage by parsing CTAS/INSERT patterns in job query history."""
+    project_id = _require_bq_identifier(project_id, "project_id")
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
    end_dt = datetime.now(timezone.utc)
    start_dt = end_dt - timedelta(hours=lookback_hours)

@@ -161,6 +181,9 @@ def collect(

    Returns the manifest dict.
    """
+    project_id = _require_bq_identifier(project_id, "project_id")
+    region = _require_bq_identifier(region, "region")
+    lookback_hours = _bounded_int(lookback_hours, "lookback_hours", minimum=1, maximum=24 * 31)
    bq_client = bigquery.Client(project=project_id)

    log.info("Collecting lineage from project %s ...", project_id)
@@ -180,8 +203,7 @@ def collect(
        "query_derived_edges": len(query_edges),
        "edges": all_edges,
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(output_file, manifest)
    log.info("Lineage manifest written to %s", output_file)

    return manifest
@@ -24,6 +24,7 @@ import os
 from datetime import datetime, timezone

 from google.cloud import bigquery
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -131,8 +132,7 @@ def collect(
        "collected_at": datetime.now(timezone.utc).isoformat(),
        "assets": assets,
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(output_file, manifest)
    log.info("Asset manifest written to %s", output_file)

    return manifest
@@ -26,6 +26,7 @@ import os
 from datetime import datetime, timedelta, timezone

 from google.cloud import bigquery
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -130,8 +131,7 @@ def collect(
        "query_log_count": len(entries),
        "queries": entries,
    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
+    write_json_file(output_file, manifest)
    log.info("Query log manifest written to %s", output_file)

    return manifest
@@ -30,6 +30,7 @@ from pycarlo.features.ingestion.models import (
    LineageAssetRef,
    LineageEvent,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -83,8 +84,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    edges = manifest.get("edges", [])
    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
@@ -102,8 +102,7 @@ def push(
            "batch_count": 0,
            "batch_size": batch_size,
        }
-        with open(output_file, "w") as fh:
-            json.dump(push_result, fh, indent=2)
+        write_json_file(output_file, push_result)
        return push_result

    # Split into batches
@@ -155,8 +154,7 @@ def push(
        "batch_count": total_batches,
        "batch_size": batch_size,
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    log.info("Push result written to %s", output_file)

    return push_result
@@ -33,6 +33,7 @@ from pycarlo.features.ingestion.models import (
    AssetVolume,
    RelationalAsset,
 )
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -95,8 +96,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    asset_dicts = manifest.get("assets", [])
    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
@@ -150,8 +150,7 @@ def push(
        "batch_count": total_batches,
        "batch_size": batch_size,
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    log.info("Push result written to %s", output_file)

    return push_result
@@ -28,6 +28,7 @@ from dateutil.parser import isoparse
 from pycarlo.core import Client, Session
 from pycarlo.features.ingestion import IngestionService
 from pycarlo.features.ingestion.models import QueryLogEntry
+from _safe_paths import safe_existing_directory, safe_input_json_path, safe_output_json_path, read_json_file, write_json_file

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -94,8 +95,7 @@ def push(

    Returns a result dict with invocation IDs for each batch.
    """
-    with open(input_file) as fh:
-        manifest = json.load(fh)
+    manifest = read_json_file(input_file)

    queries = manifest.get("queries", [])
    log_type = manifest.get("log_type", LOG_TYPE)
@@ -113,8 +113,7 @@ def push(
            "batch_count": 0,
            "batch_size": batch_size,
        }
-        with open(output_file, "w") as fh:
-            json.dump(push_result, fh, indent=2)
+        write_json_file(output_file, push_result)
        return push_result

    # Split into batches
@@ -164,8 +163,7 @@ def push(
        "batch_count": total_batches,
        "batch_size": batch_size,
    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
+    write_json_file(output_file, push_result)
    log.info("Push result written to %s", output_file)

    return push_result