Add interface for passing custom validators

This adds the necessary pieces for passing a custom validator class to
check-jsonschema. `--fill-defaults` help text contains a hint that it
may conflict (depending on the validator), to help warn of the
potential for surprising interplay between a custom validator class
and optional check-jsonschema behaviors.

This commit only adds the interface pieces for passing the validator
class to the CLI in a well-specified format, and handles the
import/load of that value. No implementation in terms of threading the
resulting value down to the validator(s) is included yet.

Author: sirosen

src/check_jsonschema/cli/main_command.py

@@ -4,6 +4,7 @@
 import textwrap
 
 import click
+import jsonschema
 
 from ..catalog import CUSTOM_SCHEMA_NAMES, SCHEMA_CATALOG
 from ..checker import SchemaChecker
@@ -18,7 +19,7 @@
     SchemaLoaderBase,
 )
 from ..transforms import TRANSFORM_LIBRARY
-from .param_types import CommaDelimitedList
+from .param_types import CommaDelimitedList, ValidatorClassName
 from .parse_result import ParseResult, SchemaLoadingMode
 
 BUILTIN_SCHEMA_NAMES = [f"vendor.{k}" for k in SCHEMA_CATALOG.keys()] + [
@@ -169,13 +170,27 @@ def pretty_helptext_list(values: list[str] | tuple[str, ...]) -> str:
 )
 @click.option(
     "--fill-defaults",
-    help="Autofill 'default' values prior to validation.",
+    help=(
+        "Autofill 'default' values prior to validation. "
+        "This may conflict with certain third-party validators used with "
+        "'--validator-class'"
+    ),
     is_flag=True,
 )
+@click.option(
+    "--validator-class",
+    help=(
+        "The fully qualified name of a python validator to use in place of "
+        "the 'jsonschema' library validators, in the form of '<package>:<class>'. "
+        "The validator must be importable in the same environment where "
+        "'check-jsonschema' is run."
+    ),
+    type=ValidatorClassName(),
+)
 @click.option(
     "-o",
     "--output-format",
-    help="Which output format to use",
+    help="Which output format to use.",
     type=click.Choice(tuple(REPORTER_BY_NAME.keys()), case_sensitive=False),
     default="text",
 )
@@ -217,6 +232,7 @@ def main(
     traceback_mode: str,
     data_transform: str | None,
     fill_defaults: bool,
+    validator_class: type[jsonschema.protocols.Validator] | None,
     output_format: str,
     verbose: int,
     quiet: int,
@@ -225,6 +241,8 @@ def main(
     args = ParseResult()
 
     args.set_schema(schemafile, builtin_schema, check_metaschema)
+    args.set_validator(validator_class)
+
     args.base_uri = base_uri
     args.instancefiles = instancefiles
 

src/check_jsonschema/cli/param_types.py

@@ -1,11 +1,17 @@
 from __future__ import annotations
 
+import importlib
+import re
 import typing as t
+import warnings
 
 import click
+import jsonschema
 
 
 class CommaDelimitedList(click.ParamType):
+    name = "comma_delimited"
+
     def __init__(
         self,
         *,
@@ -43,3 +49,69 @@ def convert(
                 )
 
         return resolved
+
+
+class ValidatorClassName(click.ParamType):
+    name = "validator"
+
+    def convert(
+        self, value: str, param: click.Parameter | None, ctx: click.Context | None
+    ) -> type[jsonschema.protocols.Validator]:
+        """
+        Use a colon-based parse to split this up and do the import with importlib.
+        This method is inspired by pkgutil.resolve_name and uses the newer syntax
+        documented there.
+
+        pkgutil supports both
+            W(.W)*
+        and
+            W(.W)*:(W(.W)*)?
+        as patterns, but notes that the first one is for backwards compatibility only.
+        The second form is preferred because it clarifies the division between the
+        importable name and any namespaced path to an object or class.
+
+        As a result, only one import is needed, rather than iterative imports over the
+        list of names.
+        """
+        value = super().convert(value, param, ctx)
+        pattern = re.compile(
+            r"^(?P<pkg>(?!\d)(\w+)(\.(?!\d)(\w+))*):"
+            r"(?P<cls>(?!\d)(\w+)(\.(?!\d)(\w+))*)$"
+        )
+        m = pattern.match(value)
+        if m is None:
+            self.fail(
+                f"'{value}' is not a valid specifier in '<package>:<class>' form",
+                param,
+                ctx,
+            )
+        pkg = m.group("pkg")
+        classname = m.group("cls")
+        try:
+            result: t.Any = importlib.import_module(pkg)
+        except ImportError as e:
+            self.fail(f"'{pkg}' was not an importable module. {str(e)}", param, ctx)
+        try:
+            for part in classname.split("."):
+                result = getattr(result, part)
+        except AttributeError as e:
+            self.fail(
+                f"'{classname}' was not resolvable to a class in '{pkg}'. {str(e)}",
+                param,
+                ctx,
+            )
+
+        if not callable(result):
+            self.fail(
+                f"'{classname}' in '{pkg}' is not a class or callable", param, ctx
+            )
+
+        if not isinstance(result, type):
+            warnings.warn(
+                f"'{classname}' in '{pkg}' is not a class. If it is a function "
+                f"returning a Validator, it still might work, but this usage "
+                "is not recommended.",
+                stacklevel=1,
+            )
+
+        return t.cast(t.Type[jsonschema.protocols.Validator], result)

src/check_jsonschema/cli/parse_result.py

@@ -3,6 +3,7 @@
 import enum
 
 import click
+import jsonschema
 
 from ..formats import FormatOptions, RegexVariantName
 from ..transforms import Transform
@@ -28,7 +29,8 @@ def __init__(self) -> None:
         self.default_filetype: str = "json"
         # data-transform (for Azure Pipelines and potentially future transforms)
         self.data_transform: Transform | None = None
-        # fill default values on instances during validation
+        # validation behavioral controls
+        self.validator_class: type[jsonschema.protocols.Validator] | None = None
         self.fill_defaults: bool = False
         # regex format options
         self.disable_all_formats: bool = False
@@ -65,6 +67,17 @@ def set_schema(
         else:
             self.schema_mode = SchemaLoadingMode.metaschema
 
+    def set_validator(
+        self, validator_class: type[jsonschema.protocols.Validator] | None
+    ) -> None:
+        if validator_class is None:
+            return
+        if self.schema_mode != SchemaLoadingMode.filepath:
+            raise click.UsageError(
+                "--validator-class can only be used with --schemafile for schema loading"
+            )
+        self.validator_class = validator_class
+
     @property
     def format_opts(self) -> FormatOptions:
         return FormatOptions(

tests/unit/test_cli_parse.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pathlib
+import sys
 from unittest import mock
 
 import click
@@ -27,6 +29,33 @@ def mock_parse_result():
         yield args
 
 
+@pytest.fixture
+def mock_module(tmp_path, monkeypatch):
+    monkeypatch.syspath_prepend(tmp_path)
+    all_names_to_clear = []
+
+    def func(path, text):
+        path = pathlib.Path(path)
+        mod_dir = tmp_path / (path.parent)
+        mod_dir.mkdir(parents=True, exist_ok=True)
+        for part in path.parts[:-1]:
+            (tmp_path / part / "__init__.py").touch()
+
+        (tmp_path / path).write_text(text)
+
+        for i in range(len(path.parts)):
+            modname = ".".join(path.parts[: i + 1])
+            if modname.endswith(".py"):
+                modname = modname[:-3]
+            all_names_to_clear.append(modname)
+
+    yield func
+
+    for name in all_names_to_clear:
+        if name in sys.modules:
+            del sys.modules[name]
+
+
 @pytest.fixture(autouse=True)
 def mock_cli_exec(boxed_context):
     def get_ctx(*args):
@@ -258,3 +287,93 @@ def test_disable_all_formats(runner, mock_parse_result, addargs):
         + addargs,
     )
     assert mock_parse_result.disable_all_formats is True
+
+
+def test_can_specify_custom_validator_class(runner, mock_parse_result, mock_module):
+    mock_module("foo.py", "class MyValidator: pass")
+    import foo
+
+    result = runner.invoke(
+        cli_main,
+        [
+            "--schemafile",
+            "schema.json",
+            "foo.json",
+            "--validator-class",
+            "foo:MyValidator",
+        ],
+    )
+    assert result.exit_code == 0
+    assert mock_parse_result.validator_class == foo.MyValidator
+
+
+def test_warns_on_validator_function(runner, mock_parse_result, mock_module):
+    mock_module(
+        "foo/bar.py",
+        """\
+class MyValidator: pass
+
+def validator(*args, **kwargs):
+    return MyValidator(*args, **kwargs)
+""",
+    )
+    import foo.bar
+
+    with pytest.warns(UserWarning, match="'validator' in 'foo.bar' is not a class"):
+        result = runner.invoke(
+            cli_main,
+            [
+                "--schemafile",
+                "schema.json",
+                "foo.json",
+                "--validator-class",
+                "foo.bar:validator",
+            ],
+        )
+        assert result.exit_code == 0
+    assert mock_parse_result.validator_class == foo.bar.validator
+
+
+@pytest.mark.parametrize("failmode", ("syntax", "import", "attr", "callability"))
+def test_can_custom_validator_class_fails(
+    runner, mock_parse_result, mock_module, failmode
+):
+    mock_module(
+        "foo.py",
+        """\
+class MyValidator: pass
+
+def validator(*args, **kwargs):
+    return MyValidator(*args, **kwargs)
+
+other_thing = 100
+""",
+    )
+
+    if failmode == "syntax":
+        arg = "foo.MyValidator"
+    elif failmode == "import":
+        arg = "foo.bar:MyValidator"
+    elif failmode == "attr":
+        arg = "foo:no_such_attr"
+    elif failmode == "callability":
+        arg = "foo:other_thing"
+    else:
+        raise NotImplementedError
+
+    result = runner.invoke(
+        cli_main,
+        ["--schemafile", "schema.json", "foo.json", "--validator-class", arg],
+    )
+    assert result.exit_code == 2
+
+    if failmode == "syntax":
+        assert "is not a valid specifier" in result.stderr
+    elif failmode == "import":
+        assert "was not an importable module" in result.stderr
+    elif failmode == "attr":
+        assert "was not resolvable to a class" in result.stderr
+    elif failmode == "callability":
+        assert "is not a class or callable" in result.stderr
+    else:
+        raise NotImplementedError