diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc index 520da97eef40..904f2330c65d 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc @@ -87,6 +87,12 @@ JUnit repository on GitHub. a test-scoped `ExtensionContext` in `Extension` methods called during test class instantiation. This behavior will become the default in future versions of JUnit. * `@TempDir` is now supported on test class constructors. +* Parameterized tests now support argument count validation. + If the `junit.jupiter.params.argumentCountValidation=strict` configuration parameter + or the `@ParameterizedTest(argumentCountValidation = STRICT)` attribute is set, any + mismatch between the declared number of arguments and the number of arguments provided + by the arguments source will result in an error. By default, it's still only an error if + there are fewer arguments provided than declared. [[release-notes-5.12.0-M1-junit-vintage]] diff --git a/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc b/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc index 8df05f70c3ba..3f408ec55cc2 100644 --- a/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc +++ b/documentation/src/docs/asciidoc/user-guide/writing-tests.adoc @@ -2020,6 +2020,29 @@ The following annotations are repeatable: * `@CsvFileSource` * `@ArgumentsSource` +[[writing-tests-parameterized-tests-argument-count-validation]] +==== Argument Count Validation + +WARNING: Argument count validation is currently an _experimental_ feature. You're invited to +give it a try and provide feedback to the JUnit team so they can improve and eventually +<> this feature. + +By default, when an arguments source provides more arguments than the test method needs, +those additional arguments are ignored and the test executes as usual. +This can lead to bugs where arguments are never passed to the parameterized test method. + +To prevent this, you can set argument count validation to 'strict'. +Then, any additional arguments will cause an error instead. + +To change this behavior for all tests, set the `junit.jupiter.params.argumentCountValidation` +<> to `strict`. +To change this behavior for a single test, +use the `argumentCountValidation` attribute of the `@ParameterizedTest` annotation: + +[source,java,indent=0] +---- +include::{testDir}/example/ParameterizedTestDemo.java[tags=argument_count_validation] +---- [[writing-tests-parameterized-tests-argument-conversion]] ==== Argument Conversion diff --git a/documentation/src/test/java/example/ParameterizedTestDemo.java b/documentation/src/test/java/example/ParameterizedTestDemo.java index 9027b86d67e4..894b7617761d 100644 --- a/documentation/src/test/java/example/ParameterizedTestDemo.java +++ b/documentation/src/test/java/example/ParameterizedTestDemo.java @@ -51,6 +51,7 @@ import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.params.ArgumentCountValidationMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.aggregator.AggregateWith; import org.junit.jupiter.params.aggregator.ArgumentsAccessor; @@ -607,4 +608,13 @@ static Stream otherProvider() { return Stream.of("bar"); } // end::repeatable_annotations[] + + @extensions.ExpectToFail + // tag::argument_count_validation[] + @ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.STRICT) + @CsvSource({ "42, -666" }) + void testWithArgumentCountValidation(int number) { + assertTrue(number > 0); + } + // end::argument_count_validation[] } diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ArgumentCountValidationMode.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ArgumentCountValidationMode.java new file mode 100644 index 000000000000..be65c2278002 --- /dev/null +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ArgumentCountValidationMode.java @@ -0,0 +1,52 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.jupiter.params; + +import org.apiguardian.api.API; +import org.junit.jupiter.params.provider.ArgumentsSource; + +/** + * Enumeration of argument count validation modes for {@link ParameterizedTest @ParameterizedTest}. + * + *

When an {@link ArgumentsSource} provides more arguments than declared by the test method, + * there might be a bug in the test method or the {@link ArgumentsSource}. + * By default, the additional arguments are ignored. + * {@link ArgumentCountValidationMode} allows you to control how additional arguments are handled. + * + * @since 5.12 + * @see ParameterizedTest + */ +@API(status = API.Status.EXPERIMENTAL, since = "5.12") +public enum ArgumentCountValidationMode { + /** + * Use the default validation mode. + * + *

The default validation mode may be changed via the + * {@value ParameterizedTestExtension#ARGUMENT_COUNT_VALIDATION_KEY} configuration parameter + * (see the User Guide for details on configuration parameters). + */ + DEFAULT, + + /** + * Use the "none" argument count validation mode. + * + *

When there are more arguments provided than declared by the test method, + * these additional arguments are ignored. + */ + NONE, + + /** + * Use the strict argument count validation mode. + * + *

When there are more arguments provided than declared by the test method, this raises an error. + */ + STRICT, +} diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java index 67296b5a4158..dfbd29296410 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java @@ -22,6 +22,7 @@ import org.apiguardian.api.API; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.provider.ArgumentsSource; /** * {@code @ParameterizedTest} is used to signal that the annotated method is a @@ -305,4 +306,21 @@ @API(status = EXPERIMENTAL, since = "5.12") boolean requireArguments() default true; + /** + * Configure how the number of arguments provided by an {@link ArgumentsSource} are validated. + * + *

Defaults to {@link ArgumentCountValidationMode#DEFAULT}. + * + *

When an {@link ArgumentsSource} provides more arguments than declared by the test method, + * there might be a bug in the test method or the {@link ArgumentsSource}. + * By default, the additional arguments are ignored. + * {@code argumentCountValidation} allows you to control how additional arguments are handled. + * The default can be configured via the {@value ParameterizedTestExtension#ARGUMENT_COUNT_VALIDATION_KEY} + * configuration parameter (see the User Guide for details on configuration parameters). + * + * @since 5.12 + * @see ArgumentCountValidationMode + */ + @API(status = EXPERIMENTAL, since = "5.12") + ArgumentCountValidationMode argumentCountValidation() default ArgumentCountValidationMode.DEFAULT; } diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTestExtension.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTestExtension.java index 74ff653e15cc..86b576999010 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTestExtension.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTestExtension.java @@ -14,10 +14,13 @@ import static org.junit.platform.commons.support.AnnotationSupport.findRepeatableAnnotations; import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.junit.jupiter.api.extension.TestTemplateInvocationContext; @@ -26,6 +29,8 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.junit.jupiter.params.support.AnnotationConsumerInitializer; +import org.junit.platform.commons.logging.Logger; +import org.junit.platform.commons.logging.LoggerFactory; import org.junit.platform.commons.util.ExceptionUtils; import org.junit.platform.commons.util.Preconditions; @@ -34,10 +39,13 @@ */ class ParameterizedTestExtension implements TestTemplateInvocationContextProvider { + private static final Logger logger = LoggerFactory.getLogger(ParameterizedTestExtension.class); + static final String METHOD_CONTEXT_KEY = "context"; static final String ARGUMENT_MAX_LENGTH_KEY = "junit.jupiter.params.displayname.argument.maxlength"; static final String DEFAULT_DISPLAY_NAME = "{default_display_name}"; static final String DISPLAY_NAME_PATTERN_KEY = "junit.jupiter.params.displayname.default"; + static final String ARGUMENT_COUNT_VALIDATION_KEY = "junit.jupiter.params.argumentCountValidation"; @Override public boolean supportsTestTemplate(ExtensionContext context) { @@ -61,7 +69,7 @@ public boolean supportsTestTemplate(ExtensionContext context) { + "and before any arguments resolved by another ParameterResolver.", templateMethod.toGenericString())); - getStore(context).put(METHOD_CONTEXT_KEY, methodContext); + getStoreInMethodNamespace(context).put(METHOD_CONTEXT_KEY, methodContext); return true; } @@ -82,6 +90,7 @@ public Stream provideTestTemplateInvocationContex .map(provider -> AnnotationConsumerInitializer.initialize(methodContext.method, provider)) .flatMap(provider -> arguments(provider, extensionContext)) .map(arguments -> { + validateArgumentCount(extensionContext, arguments); invocationCount.incrementAndGet(); return createInvocationContext(formatter, methodContext, arguments, invocationCount.intValue()); }) @@ -98,14 +107,80 @@ public boolean mayReturnZeroTestTemplateInvocationContexts(ExtensionContext exte } private ParameterizedTestMethodContext getMethodContext(ExtensionContext extensionContext) { - return getStore(extensionContext)// + return getStoreInMethodNamespace(extensionContext)// .get(METHOD_CONTEXT_KEY, ParameterizedTestMethodContext.class); } - private ExtensionContext.Store getStore(ExtensionContext context) { + private ExtensionContext.Store getStoreInMethodNamespace(ExtensionContext context) { return context.getStore(Namespace.create(ParameterizedTestExtension.class, context.getRequiredTestMethod())); } + private ExtensionContext.Store getStoreInExtensionNamespace(ExtensionContext context) { + return context.getRoot().getStore(Namespace.create(ParameterizedTestExtension.class)); + } + + private void validateArgumentCount(ExtensionContext extensionContext, Arguments arguments) { + ArgumentCountValidationMode argumentCountValidationMode = getArgumentCountValidationMode(extensionContext); + switch (argumentCountValidationMode) { + case DEFAULT: + case NONE: + return; + case STRICT: + int testParamCount = extensionContext.getRequiredTestMethod().getParameterCount(); + int argumentsCount = arguments.get().length; + Preconditions.condition(testParamCount == argumentsCount, () -> String.format( + "Configuration error: the @ParameterizedTest has %s argument(s) but there were %s argument(s) provided.%nNote: the provided arguments are %s", + testParamCount, argumentsCount, Arrays.toString(arguments.get()))); + break; + default: + throw new ExtensionConfigurationException( + "Unsupported argument count validation mode: " + argumentCountValidationMode); + } + } + + private ArgumentCountValidationMode getArgumentCountValidationMode(ExtensionContext extensionContext) { + ParameterizedTest parameterizedTest = findAnnotation(// + extensionContext.getRequiredTestMethod(), ParameterizedTest.class// + ).orElseThrow(NoSuchElementException::new); + if (parameterizedTest.argumentCountValidation() != ArgumentCountValidationMode.DEFAULT) { + return parameterizedTest.argumentCountValidation(); + } + else { + return getArgumentCountValidationModeConfiguration(extensionContext); + } + } + + private ArgumentCountValidationMode getArgumentCountValidationModeConfiguration(ExtensionContext extensionContext) { + String key = ARGUMENT_COUNT_VALIDATION_KEY; + ArgumentCountValidationMode fallback = ArgumentCountValidationMode.NONE; + ExtensionContext.Store store = getStoreInExtensionNamespace(extensionContext); + return store.getOrComputeIfAbsent(key, __ -> { + Optional optionalConfigValue = extensionContext.getConfigurationParameter(key); + if (optionalConfigValue.isPresent()) { + String configValue = optionalConfigValue.get(); + Optional enumValue = Arrays.stream( + ArgumentCountValidationMode.values()).filter( + mode -> mode.name().equalsIgnoreCase(configValue)).findFirst(); + if (enumValue.isPresent()) { + logger.config(() -> String.format( + "Using ArgumentCountValidationMode '%s' set via the '%s' configuration parameter.", + enumValue.get().name(), key)); + return enumValue.get(); + } + else { + logger.warn(() -> String.format( + "Invalid ArgumentCountValidationMode '%s' set via the '%s' configuration parameter. " + + "Falling back to the %s default value.", + configValue, key, fallback.name())); + return fallback; + } + } + else { + return fallback; + } + }, ArgumentCountValidationMode.class); + } + private TestTemplateInvocationContext createInvocationContext(ParameterizedTestNameFormatter formatter, ParameterizedTestMethodContext methodContext, Arguments arguments, int invocationIndex) { diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java index c80708edfae5..abb0273f2136 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java @@ -116,6 +116,7 @@ import org.junit.platform.testkit.engine.EngineExecutionResults; import org.junit.platform.testkit.engine.EngineTestKit; import org.junit.platform.testkit.engine.Event; +import org.junit.platform.testkit.engine.EventConditions; import org.opentest4j.TestAbortedException; /** @@ -1112,6 +1113,63 @@ private EngineExecutionResults execute(String methodName, Class... methodPara } + @Nested + class UnusedArgumentsWithStrictArgumentsCountIntegrationTests { + @Test + void failsWithArgumentsSourceProvidingUnusedArguments() { + var results = execute(ArgumentCountValidationMode.STRICT, UnusedArgumentsTestCase.class, + "testWithTwoUnusedStringArgumentsProvider", String.class); + results.allEvents().assertThatEvents() // + .haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format( + "Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]"))))); + } + + @Test + void failsWithMethodSourceProvidingUnusedArguments() { + var results = execute(ArgumentCountValidationMode.STRICT, UnusedArgumentsTestCase.class, + "testWithMethodSourceProvidingUnusedArguments", String.class); + results.allEvents().assertThatEvents() // + .haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format( + "Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]"))))); + } + + @Test + void failsWithCsvSourceUnusedArgumentsAndStrictArgumentCountValidationAnnotationAttribute() { + var results = execute(ArgumentCountValidationMode.NONE, UnusedArgumentsTestCase.class, + "testWithStrictArgumentCountValidation", String.class); + results.allEvents().assertThatEvents() // + .haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format( + "Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]"))))); + } + + @Test + void executesWithCsvSourceUnusedArgumentsAndArgumentCountValidationAnnotationAttribute() { + var results = execute(ArgumentCountValidationMode.NONE, UnusedArgumentsTestCase.class, + "testWithNoneArgumentCountValidation", String.class); + results.allEvents().assertThatEvents() // + .haveExactly(1, + event(test(), displayName("[1] argument=foo"), finishedWithFailure(message("foo")))); + } + + @Test + void executesWithMethodSourceProvidingUnusedArguments() { + var results = execute(ArgumentCountValidationMode.STRICT, RepeatableSourcesTestCase.class, + "testWithRepeatableCsvSource", String.class); + results.allEvents().assertThatEvents() // + .haveExactly(1, event(test(), displayName("[1] argument=a"), finishedWithFailure(message("a")))) // + .haveExactly(1, event(test(), displayName("[2] argument=b"), finishedWithFailure(message("b")))); + } + + private EngineExecutionResults execute(ArgumentCountValidationMode configurationValue, Class javaClass, + String methodName, Class... methodParameterTypes) { + return EngineTestKit.engine(new JupiterTestEngine()) // + .selectors(selectMethod(javaClass, methodName, methodParameterTypes)) // + .configurationParameter(ParameterizedTestExtension.ARGUMENT_COUNT_VALIDATION_KEY, + configurationValue.name().toLowerCase()) // + .execute(); + } + } + @Nested class RepeatableSourcesIntegrationTests { @@ -2028,6 +2086,17 @@ void testWithFieldSourceProvidingUnusedArguments(String argument) { static Supplier> unusedArgumentsProviderField = // () -> Stream.of(arguments("foo", "unused1"), arguments("bar", "unused2")); + @ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.STRICT) + @CsvSource({ "foo, unused1" }) + void testWithStrictArgumentCountValidation(String argument) { + fail(argument); + } + + @ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.NONE) + @CsvSource({ "foo, unused1" }) + void testWithNoneArgumentCountValidation(String argument) { + fail(argument); + } } static class LifecycleTestCase {