fix(checks): join list[str] context in Groundedness/Contradiction judges instead of str()-ing it 🤖🤖🤖🤖

libs/giskard-checks/src/giskard/checks/judges/base.py

@@ -19,6 +19,24 @@ class LLMCheckResult(BaseModel):
     passed: bool = Field(..., description="Whether the check passed or failed")
 
 
+def format_prompt_text(value: Any) -> str:
+    """Render a resolved trace/input value as plain text for an LLM prompt.
+
+    Several judge inputs (e.g. ``context``) accept either a single string or a
+    list of strings (multiple context documents). A bare ``str(value)`` on a
+    list renders Python's ``repr`` -- brackets, comma separators, and quoted
+    (sometimes escaped) items -- which leaks implementation syntax into the
+    prompt instead of the actual text. Join list values into a single
+    newline-separated block instead, matching the ``"\\n".join(...)``
+    convention used elsewhere in this codebase for combining text fragments.
+    Any other value (a plain string, ``NoMatch``, or arbitrary trace payload)
+    is stringified as before.
+    """
+    if isinstance(value, list):
+        return "\n".join(str(item) for item in value)
+    return str(value)
+
+
 class BaseLLMCheck[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
     Check[InputType, OutputType, TraceType], WithGeneratorMixin
 ):

libs/giskard-checks/src/giskard/checks/judges/contradiction.py

@@ -7,7 +7,7 @@
 from ..core import Trace
 from ..core.check import Check
 from ..core.extraction import JSONPathStr, provided_or_resolve
-from .base import BaseLLMCheck
+from .base import BaseLLMCheck, format_prompt_text
 
 
 @Check.register("contradiction")
@@ -49,7 +49,7 @@ async def get_inputs(self, trace: Trace[InputType, OutputType]) -> dict[str, str
             "answer": str(
                 provided_or_resolve(trace, key=self.answer_key, value=self.answer)
             ),
-            "context": str(
+            "context": format_prompt_text(
                 provided_or_resolve(trace, key=self.context_key, value=self.context)
             ),
         }

libs/giskard-checks/src/giskard/checks/judges/groundedness.py

@@ -7,7 +7,7 @@
 from ..core import Trace
 from ..core.check import Check
 from ..core.extraction import JSONPathStr, provided_or_resolve
-from .base import BaseLLMCheck
+from .base import BaseLLMCheck, format_prompt_text
 
 
 @Check.register("groundedness")
@@ -91,7 +91,7 @@ async def get_inputs(self, trace: Trace[InputType, OutputType]) -> dict[str, str
                     value=self.answer,
                 )
             ),
-            "context": str(
+            "context": format_prompt_text(
                 provided_or_resolve(
                     trace,
                     key=self.context_key,

libs/giskard-checks/tests/builtin/test_contradiction.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 from giskard.checks import CheckStatus, Contradiction, Interaction, Trace
 
 from ..testing_utils import MockJudgeGenerator as MockGenerator
@@ -58,7 +60,7 @@ async def test_direct_answer_and_context_are_passed_to_judge() -> None:
 
     assert result.status == CheckStatus.PASS
     assert result.details["inputs"]["answer"] == "Direct answer"
-    assert result.details["inputs"]["context"] == "['Context 1', 'Context 2']"
+    assert result.details["inputs"]["context"] == "Context 1\nContext 2"
     assert len(generator.calls) == 1
 
 
@@ -129,7 +131,7 @@ async def test_custom_answer_and_context_keys() -> None:
 
     assert result.status == CheckStatus.PASS
     assert result.details["inputs"]["answer"] == "The Eiffel Tower is in Paris."
-    assert result.details["inputs"]["context"] == "['The Eiffel Tower is in Paris.']"
+    assert result.details["inputs"]["context"] == "The Eiffel Tower is in Paris."
 
 
 async def test_direct_values_take_priority_over_trace() -> None:
@@ -149,7 +151,36 @@ async def test_direct_values_take_priority_over_trace() -> None:
 
     assert result.status == CheckStatus.PASS
     assert result.details["inputs"]["answer"] == "Direct answer"
-    assert result.details["inputs"]["context"] == "['Direct context']"
+    assert result.details["inputs"]["context"] == "Direct context"
+
+
+async def test_list_context_is_joined_without_python_repr_artifacts() -> None:
+    """A list[str] context must reach the judge prompt as readable text.
+
+    Regression test for the bug where get_inputs() rendered a list context via
+    bare str(), producing the Python repr (e.g. "['doc 1', 'doc 2']") instead of
+    the documents themselves. Documents containing apostrophes are especially
+    revealing: str(repr) mixes quote styles and escapes the apostrophe, which
+    should never appear in the actual prompt text.
+    """
+    generator = MockGenerator(passed=True, reason=None)
+    contradiction = Contradiction(
+        generator=generator,
+        answer="The Eiffel Tower is in Paris, and it's a landmark.",
+        context=[
+            "Paris is the capital of France.",
+            "It's located in Europe.",
+        ],
+    )
+
+    result = await contradiction.run(Trace())
+
+    context_str = cast(str, result.details["inputs"]["context"])
+    assert context_str == "Paris is the capital of France.\nIt's located in Europe."
+    assert "[" not in context_str
+    assert "]" not in context_str
+    assert '\\"' not in context_str
+    assert "\\'" not in context_str
 
 
 async def test_missing_trace_values_are_passed_to_judge_as_no_match() -> None:

libs/giskard-checks/tests/builtin/test_groundedness.py

@@ -167,7 +167,7 @@ async def test_context_priority_over_trace() -> None:
     result = await groundedness.run(Trace(interactions=[interaction]))
 
     assert result.status == CheckStatus.PASS
-    assert result.details["inputs"]["context"] == "['Direct context']"
+    assert result.details["inputs"]["context"] == "Direct context"
 
 
 async def test_empty_string_context_is_preserved() -> None:
@@ -200,7 +200,35 @@ async def test_empty_context() -> None:
     result = await groundedness.run(Trace())
 
     assert result.status == CheckStatus.FAIL
-    assert result.details["inputs"]["context"] == "[]"
+    assert result.details["inputs"]["context"] == ""
+
+
+async def test_list_context_is_joined_without_python_repr_artifacts() -> None:
+    """A list[str] context must reach the judge prompt as readable text.
+
+    Regression test for the bug where get_inputs() rendered a list context via
+    bare str(), producing the Python repr (e.g. "['doc 1', 'doc 2']") instead of
+    the documents themselves. Documents containing apostrophes are especially
+    revealing: str(repr) mixes quote styles and escapes the apostrophe, which
+    should never appear in the actual prompt text.
+    """
+    generator = MockGenerator(passed=True, reason=None)
+    groundedness = Groundedness(
+        generator=generator,
+        answer="The Eiffel Tower is in Paris, and it's a landmark.",
+        context=[
+            "Paris is the capital of France.",
+            "It's located in Europe.",
+        ],
+    )
+    result = await groundedness.run(Trace())
+
+    context_str = cast(str, result.details["inputs"]["context"])
+    assert context_str == "Paris is the capital of France.\nIt's located in Europe."
+    assert "[" not in context_str
+    assert "]" not in context_str
+    assert '\\"' not in context_str
+    assert "\\'" not in context_str
 
 
 async def test_missing_answer_in_trace() -> None: