Merge pull request #1748 from Giskard-AI/GSK-2617

GSK-2617 Missing kwargs in a test make the whole suite fail

Author: kevinmessiaen

giskard/core/suite.py

@@ -433,12 +433,9 @@ def run(self, verbose: bool = True, **suite_run_args):
         """
         run_args = self.default_params.copy()
         run_args.update(suite_run_args)
+        self.verify_required_params(run_args)
 
         results: List[SuiteResult] = list()
-        required_params = self.find_required_params()
-        undefined_params = {k: v for k, v in required_params.items() if k not in run_args}
-        if len(undefined_params):
-            raise ValueError(f"Missing {len(undefined_params)} required parameters: {undefined_params}")
 
         for test_partial in self.tests:
             test_params = self.create_test_params(test_partial, run_args)
@@ -497,10 +494,7 @@ def to_unittest(self, **suite_gen_args) -> List[TestPartial]:
         run_args.update(suite_gen_args)
 
         unittests: List[TestPartial] = list()
-        required_params = self.find_required_params()
-        undefined_params = {k: v for k, v in required_params.items() if k not in run_args}
-        if len(undefined_params):
-            raise ValueError(f"Missing {len(undefined_params)} required parameters: {undefined_params}")
+        self.verify_required_params(run_args)
 
         for test_partial in self.tests:
             test_params = self.create_test_params(test_partial, run_args)
@@ -516,6 +510,13 @@ def to_unittest(self, **suite_gen_args) -> List[TestPartial]:
 
         return unittests
 
+    def verify_required_params(self, run_args: Dict[str, Any]):
+        required_params = self.find_required_params()
+        undefined_params = {k: v for k, v in required_params.items() if k not in run_args}
+
+        if undefined_params:
+            warning(f"Missing {len(undefined_params)} required parameters: {undefined_params}")
+
     @staticmethod
     def create_test_params(test_partial, kwargs) -> TestParams:
         if isinstance(test_partial.giskard_test, GiskardTestMethod):

tests/test_programmatic.py

@@ -46,18 +46,38 @@ def test_a_greater_b_pass():
 
 
 def test_missing_arg():
-    with pytest.raises(Exception, match="Missing 1 required parameters: {'b': <class 'int'>}"):
-        Suite().add_test(_test_a_greater_b(a=2)).run()
+    with pytest.warns(match="Missing 1 required parameters: {'b': <class 'int'>}"):
+        result = Suite().add_test(_test_a_greater_b(a=2)).run()
+
+        assert not result.passed
+
+        assert len(result.results) == 1
+        _, test_result, _ = result.results[0]
+        assert "1 validation error" in test_result.messages[0].text
 
 
 def test_missing_args():
-    with pytest.raises(Exception, match="Missing 2 required parameters: {'a': <class 'int'>, 'b': <class 'int'>}"):
-        Suite().add_test(_test_a_greater_b()).run()
+    with pytest.warns(match="Missing 2 required parameters: {'a': <class 'int'>, 'b': <class 'int'>}"):
+        result = Suite().add_test(_test_a_greater_b()).run()
+
+        assert not result.passed
+
+        assert len(result.results) == 1
+        _, test_result, _ = result.results[0]
+        assert test_result.is_error
+        assert "2 validation errors" in test_result.messages[0].text
 
 
 def test_missing_arg_one_global():
-    with pytest.raises(Exception, match="Missing 1 required parameters: {'b': <class 'int'>}"):
-        Suite().add_test(_test_a_greater_b()).run(a=2)
+    with pytest.warns(match="Missing 1 required parameters: {'b': <class 'int'>}"):
+        result = Suite().add_test(_test_a_greater_b()).run(a=2)
+
+        assert not result.passed
+
+        assert len(result.results) == 1
+        _, test_result, _ = result.results[0]
+        assert test_result.is_error
+        assert "1 validation error" in test_result.messages[0].text, test_result.messages[0].text
 
 
 def test_all_global():

tests/test_suite.py

@@ -27,15 +27,25 @@ def test_default_parameters_are_used_at_runtime(german_credit_data, german_credi
     # This will miss dataset
     suite = Suite(default_params=dict(model=german_credit_model))
     suite.add_test(my_test)
-    with pytest.raises(ValueError):
-        suite.run()
+    result = suite.run()
+    assert not result.passed
+    _, test_result, _ = result.results[0]
+    assert test_result.is_error
 
     # But we can pass dataset at runtime
-    suite.run(dataset=german_credit_data)
+    result = suite.run(dataset=german_credit_data)
+    assert result.passed
+    _, test_result, _ = result.results[0]
+    assert not test_result.is_error
+    assert test_result.passed
 
     # Or we can provide it in the suite defaults
     suite.default_params["dataset"] = german_credit_data
-    suite.run()
+    result = suite.run()
+    assert result.passed
+    _, test_result, _ = result.results[0]
+    assert not test_result.is_error
+    assert test_result.passed
 
 
 def test_runtime_parameters_override_default_parameters(german_credit_data, german_credit_model):