Merge branch 'main' into feat/garak-scan-integration

Author: kevinmessiaen

.github/workflows/integration-tests.yml

@@ -83,6 +83,14 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
           persist-credentials: false
+          # Deliberate opt-in: this job installs and executes fork-authored
+          # code (make install, pytest) with live provider API keys in scope.
+          # Compensating controls: the `ci` environment requires a reviewer to
+          # approve each run before secrets are injected (server-enforced,
+          # per-run), the authorize job requires a fresh `labeled` event for
+          # external contributors, and the immutable head.sha pin keeps the
+          # reviewed commit identical to the executed one.
+          allow-unsafe-pr-checkout: true
       - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
         with:
           enable-cache: true
@@ -135,6 +143,9 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
           persist-credentials: false
+          # Deliberate opt-in — see test-agents-functional for the full
+          # rationale and compensating controls.
+          allow-unsafe-pr-checkout: true
       - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
         with:
           enable-cache: true
@@ -173,6 +184,9 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
           persist-credentials: false
+          # Deliberate opt-in — see test-agents-functional for the full
+          # rationale and compensating controls.
+          allow-unsafe-pr-checkout: true
       - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
         with:
           enable-cache: true

.gitignore

@@ -182,3 +182,7 @@ mlruns
 local/
 *.local
 .claude/settings.local.json
+
+# Agent tooling state (generated, machine-local)
+.deepeval/
+.superpowers/

libs/giskard-agents/src/giskard/agents/workflow.py

@@ -166,7 +166,7 @@ async def execute(
 
     async def _run_tools(self, chat: Chat[Any]) -> AsyncGenerator[ToolMessage, None]:
         if (
-            not chat.last
+            not chat.messages
             or not isinstance(chat.last, AssistantMessage)
             or not chat.last.tool_calls
         ):

libs/giskard-agents/tests/test_workflow_steps.py

@@ -120,3 +120,29 @@ async def test_run_returns_successful_chat():
 
     assert not chat.failed
     assert chat.last.content == "World!"
+
+
+async def test_run_with_no_messages_does_not_raise_indexerror():
+    """A ChatWorkflow with no messages added must not crash with IndexError.
+
+    ``_run_tools``'s guard checks ``not chat.last`` before checking whether
+    there is anything to process, but ``Chat.last`` indexes ``messages[-1]``,
+    which raises ``IndexError`` on an empty list instead of behaving falsy.
+    """
+    gen = MagicMock(spec=BaseGenerator)
+    gen.complete = AsyncMock(
+        return_value=CompletionResponse(
+            choices=[
+                Choice(
+                    message=AssistantMessage(content="Hello!"),
+                    finish_reason="stop",
+                    index=0,
+                )
+            ]
+        )
+    )
+
+    chat = await ChatWorkflow(generator=gen).run()
+
+    assert not chat.failed
+    assert chat.last.content == "Hello!"

libs/giskard-checks/src/giskard/checks/__init__.py

@@ -44,6 +44,7 @@
     SuiteResult,
     Target,
     TestCase,
+    TestCaseError,
     TestCaseResult,
     Trace,
     resolve,
@@ -97,6 +98,7 @@
     "SuiteResult",
     "Target",
     "TestCase",
+    "TestCaseError",
     "TestCaseResult",
     "Trace",
     "resolve",

libs/giskard-checks/src/giskard/checks/core/__init__.py

@@ -10,6 +10,7 @@
     Metric,
     ScenarioResult,
     SuiteResult,
+    TestCaseError,
     TestCaseResult,
 )
 from .scenario import Scenario, Step
@@ -32,6 +33,7 @@
     "Metric",
     "ScenarioResult",
     "SuiteResult",
+    "TestCaseError",
     "TestCaseResult",
     "TestCase",
     "InputGenerationException",

libs/giskard-checks/src/giskard/checks/core/extraction.py

@@ -4,6 +4,7 @@
     Child,
     DatumInContext,
     Descendants,
+    Fields,
     Intersect,
     JSONPath,
     Slice,
@@ -75,6 +76,9 @@ def _is_list_expression(expression: JSONPath) -> bool:
     if isinstance(expression, Slice | Union | Intersect):
         return True
 
+    if isinstance(expression, Fields):
+        return len(expression.fields) > 1 or "*" in expression.fields
+
     return False
 
 

libs/giskard-checks/src/giskard/checks/core/result.py

@@ -355,6 +355,8 @@ def __rich_console__(
         yield Rule(status["title"], style=f"{status['color']} bold")
 
         for step in self.steps:
+            if step.error is not None:
+                yield step.error.rich_line(status["color"])
             for result in step.results:
                 yield from result.__rich_console__(console, options)
 
@@ -382,6 +384,28 @@ class TestCaseStatus(str, Enum):
     SKIP = "skip"
 
 
+class TestCaseError(BaseModel, frozen=True):
+    """Captures why a test case failed to execute."""
+
+    message: str
+    exception_type: str
+    traceback: str | None = None
+    phase: str | None = None
+
+    def summary(self) -> str:
+        """One-line description: ``<ExceptionType>[ during <phase>]: <message>``."""
+        phase = f" during {self.phase}" if self.phase else ""
+        return f"{self.exception_type}{phase}: {self.message}"
+
+    def rich_line(self, color: str) -> str:
+        """Rich-markup row rendering this error under a step, in the given color."""
+        return (
+            f"[{color} bold]Test case[/{color} bold]\t"
+            f"[{color}]ERROR[/{color}]\t"
+            f"{self.summary()}"
+        )
+
+
 class TestCaseResult(BaseResult, frozen=True):
     """Immutable summary of a test case execution with full run history.
 
@@ -396,6 +420,9 @@ class TestCaseResult(BaseResult, frozen=True):
         this step added before its checks ran. ``None`` when the step added no
         interactions (e.g. skipped). Consumers (such as the Giskard Hub upload
         flow) use this to attribute check results to a specific interaction.
+    error : TestCaseError | None
+        Execution error that prevented the test case from running normally,
+        such as an input-generation failure.
     status : TestCaseStatus
         Aggregated outcome of the test case derived from its results.
     passed : bool
@@ -417,11 +444,19 @@ class TestCaseResult(BaseResult, frozen=True):
             "interacts before checks ran; None when no interactions were added."
         ),
     )
+    error: TestCaseError | None = Field(
+        default=None,
+        description=(
+            "Execution error that prevented this test case from running normally."
+        ),
+    )
 
     @computed_field
     @property
     def status(self) -> TestCaseStatus:
         """The status of the test case."""
+        if self.error is not None:
+            return TestCaseStatus.ERROR
         if not self.results:
             return TestCaseStatus.PASS
 
@@ -470,6 +505,8 @@ def format_failures(self) -> list[str]:
             the check name/kind and the failure reason.
         """
         failure_messages: list[str] = []
+        if self.error is not None:
+            failure_messages.append(f"Test case ERRORED: {self.error.summary()}")
         for result in self.results:
             if result.failed or result.errored:
                 check_name: str = result.details.get(
@@ -508,11 +545,15 @@ def __rich_console__(
         status = STATUS_MAPPING[self.status]
         yield Rule(status["title"], style=f"{status['color']} bold")
 
+        if self.error is not None:
+            yield self.error.rich_line(status["color"])
+
         for result in self.results:
             yield from result.__rich_console__(console, options)
 
         status_counts = {
-            "error": sum(1 for r in self.results if r.errored),
+            "error": sum(1 for r in self.results if r.errored)
+            + (1 if self.error is not None else 0),
             "fail": sum(1 for r in self.results if r.failed),
             "skip": sum(1 for r in self.results if r.skipped),
             "pass": sum(1 for r in self.results if r.passed),

libs/giskard-checks/src/giskard/checks/scenarios/runner.py

@@ -6,6 +6,7 @@
 """
 
 import time
+import traceback
 from typing import Any, cast
 
 from giskard.core import (
@@ -18,7 +19,7 @@
 from .._telemetry_props import scenario_shape_properties
 from ..core import Trace
 from ..core.interaction import Interact
-from ..core.result import CheckResult, ScenarioResult, TestCaseResult
+from ..core.result import CheckResult, ScenarioResult, TestCaseError, TestCaseResult
 from ..core.scenario import Scenario, Step
 from ..core.testcase import TestCase
 from ..core.types import Target
@@ -74,6 +75,22 @@ def _resolve_trace_type[InputType, OutputType, TraceType: Trace[Any, Any]](
     return cast(type[TraceType], inferred if inferred is not None else Trace)
 
 
+def _skipped_check_results_for_step[InputType, OutputType, TraceType: Trace[Any, Any]](
+    step: Step[InputType, OutputType, TraceType], message: str
+) -> list[CheckResult]:
+    return [
+        CheckResult.skip(
+            message=message,
+            details={
+                "check_kind": check.kind,
+                "check_name": check.name,
+                "check_description": check.description,
+            },
+        )
+        for check in step.checks
+    ]
+
+
 class ScenarioRunner:
     """Execute scenarios by running their steps sequentially.
 
@@ -133,7 +150,32 @@ async def _run_once[InputType, OutputType, TraceType: Trace[Any, Any]](
         )
 
         for step in steps:
-            trace = await trace.with_interactions(*step.interacts)
+            try:
+                for interaction in step.interacts:
+                    trace = await trace.with_interaction(interaction)
+            except Exception as e:
+                if not return_exception:
+                    raise
+
+                step_result = TestCaseResult(
+                    results=_skipped_check_results_for_step(
+                        step,
+                        "Checks were skipped due to input generation failure",
+                    ),
+                    duration_ms=int((time.perf_counter() - start_time) * 1000),
+                    last_interaction_index=(
+                        len(trace.interactions) - 1 if trace.interactions else None
+                    ),
+                    error=TestCaseError(
+                        message=str(e),
+                        exception_type=type(e).__name__,
+                        traceback=traceback.format_exc(),
+                        phase="input_generation",
+                    ),
+                )
+                steps_results.append(step_result)
+                break
+
             last_interaction_index = (
                 len(trace.interactions) - 1 if trace.interactions else None
             )
@@ -160,17 +202,10 @@ async def _run_once[InputType, OutputType, TraceType: Trace[Any, Any]](
             )
             for i in range(len(steps_results), len(steps)):
                 step_result = TestCaseResult(
-                    results=[
-                        CheckResult.skip(
-                            message=f"Step {i + 1} was skipped due to previous failure",
-                            details={
-                                "check_kind": check.kind,
-                                "check_name": check.name,
-                                "check_description": check.description,
-                            },
-                        )
-                        for check in steps[i].checks
-                    ],
+                    results=_skipped_check_results_for_step(
+                        steps[i],
+                        f"Step {i + 1} was skipped due to previous failure",
+                    ),
                     duration_ms=0,
                     last_interaction_index=skipped_last_interaction_index,
                 )

libs/giskard-checks/tests/core/test_extraction.py

@@ -1,7 +1,12 @@
 """Unit tests for JSONPath validation in extraction.py."""
 
 import pytest
-from giskard.checks.core.extraction import JSONPathStr, _validate_jsonpath_syntax
+from giskard.checks.core.extraction import (
+    JSONPathStr,
+    _validate_jsonpath_syntax,
+    resolve,
+)
+from giskard.checks.core.interaction import Interaction, Trace
 from pydantic import BaseModel, ValidationError
 
 
@@ -75,6 +80,44 @@ def test_valid_descendants_operator(self):
         assert _validate_jsonpath_syntax("trace..outputs") == "trace..outputs"
 
 
+class TestResolveDotWildcard:
+    """Tests for resolve()'s handling of dot-wildcard/multi-field JSONPath expressions.
+
+    A dot-wildcard (``trace.last.metadata.*``) or multi-field selector must always
+    resolve to a list, regardless of how many keys it actually matches -- otherwise
+    ComparisonCheck's match=any/all/none (which requires list/set/tuple) breaks
+    whenever the matched dict happens to have exactly one key.
+    """
+
+    def test_wildcard_with_single_match_returns_a_list_not_a_bare_scalar(self):
+        trace = Trace[str, str](
+            interactions=[
+                Interaction(inputs="hi", outputs="hello", metadata={"only_key": "v1"})
+            ]
+        )
+        assert resolve(trace, "trace.last.metadata.*") == ["v1"]
+
+    def test_wildcard_with_multiple_matches_returns_a_list(self):
+        trace = Trace[str, str](
+            interactions=[
+                Interaction(
+                    inputs="hi",
+                    outputs="hello",
+                    metadata={"k1": "v1", "k2": "v2"},
+                )
+            ]
+        )
+        result = resolve(trace, "trace.last.metadata.*")
+        assert isinstance(result, list)
+        assert sorted(result) == ["v1", "v2"]
+
+    def test_non_wildcard_single_field_still_returns_a_bare_scalar(self):
+        trace = Trace[str, str](
+            interactions=[Interaction(inputs="hi", outputs="hello")]
+        )
+        assert resolve(trace, "trace.last.outputs") == "hello"
+
+
 class TestJSONPathStrAnnotatedType:
     """Tests for JSONPathStr as a Pydantic Annotated field type."""