Skip to content

Commit

Permalink
Merge branch 'main' into origin/issues/3107-method-filters
Browse files Browse the repository at this point in the history
  • Loading branch information
yhkuo41 authored Apr 24, 2023
2 parents 2f26377 + 38c2158 commit 22210ca
Show file tree
Hide file tree
Showing 120 changed files with 1,396 additions and 710 deletions.
2 changes: 1 addition & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -61,5 +61,5 @@ nexusPublishing {
}

nohttp {
source.exclude("**/.gradle/**", "gradle/plugins/**/build/**")
source.exclude("**/.gradle/**", "gradle/plugins/**/build/**", "buildSrc/build/**")
}
3 changes: 0 additions & 3 deletions documentation/documentation.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ dependencies {
testRuntimeOnly(libs.apiguardian) {
because("it's required to generate API tables")
}
testRuntimeOnly(libs.openTestReporting.events) {
because("it's required to run tests via IntelliJ which does not consumed the shadowed jar of junit-platform-reporting")
}

testImplementation(libs.classgraph) {
because("ApiReportGenerator needs it")
Expand Down
2 changes: 1 addition & 1 deletion documentation/src/docs/asciidoc/link-attributes.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ endif::[]
:junit-jupiter-engine: {javadoc-root}/org.junit.jupiter.engine/org/junit/jupiter/engine/package-summary.html[junit-jupiter-engine]
// Jupiter Extension Implementations
:DisabledCondition: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/DisabledCondition.java[DisabledCondition]
:RepetitionInfoParameterResolver: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/RepetitionInfoParameterResolver.java[RepetitionInfoParameterResolver]
:RepetitionExtension: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/RepetitionExtension.java[RepetitionExtension]
:TempDirectory: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TempDirectory.java[TempDirectory]
:TestInfoParameterResolver: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TestInfoParameterResolver.java[TestInfoParameterResolver]
:TestReporterParameterResolver: {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TestReporterParameterResolver.java[TestReporterParameterResolver]
Expand Down
2 changes: 2 additions & 0 deletions documentation/src/docs/asciidoc/release-notes/index.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ include::{includedir}/link-attributes.adoc[]

include::{basedir}/release-notes-5.10.0-M1.adoc[]

include::{basedir}/release-notes-5.9.3.adoc[]

include::{basedir}/release-notes-5.9.2.adoc[]

include::{basedir}/release-notes-5.9.1.adoc[]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ repository on GitHub.

==== New Features and Improvements

* Promote various "experimental" APIs that have matured to "stable" including
`ModuleSelector`, `EngineDiscoveryListener`, `EngineDiscoveryRequestResolver`,
`LauncherSession`, `LauncherSessionListener`, parallel execution support classes,
`@Suite` and related annotations, and others.
* All utility methods in `ReflectionSupport` that return a `List` now have counterparts
which return a `Stream`.
* For consistency with JUnit Jupiter lifecycle callbacks, listener method pairs for
Expand All @@ -38,6 +42,7 @@ repository on GitHub.
methods of the former. Please refer to the
<<../user-guide/index.adoc#launcher-api-launcher-interceptors-custom, User Guide>> for
details.
* Support for limiting the `max-pool-size-factor` for parallel execution via a configuration parameter.
* The new `testfeed` details mode for `ConsoleLauncher` prints test execution events as
they occur in a concise format.
* New `--exclude-methodname` and `--include-methodname` options added to the
Expand All @@ -51,34 +56,36 @@ will exclude all methods called `methodName` under package `org.example`.

==== Bug Fixes

* Parameter types for _local_ `@MethodSource` factory method names are now validated. For
example, `@MethodSource("myFactory(example.NonexistentType)")` will now result in an
exception stating that `example.NonexistentType` cannot be resolved to a valid type.
* The syntax for parameter types in _local_ `@MethodSource` factory method names now
supports canonical array names -- for example, you may now specify `int[]` as in
`@MethodSource("myFactory(int[])"` instead of the _binary_ name `[I` as in
`@MethodSource("myFactory([I)"` (which was already supported) and
`@MethodSource("myFactory(java.lang.String[])` instead of
`@MethodSource("myFactory([Ljava.lang.String;)`.
* Exceptions thrown for undeletable files when cleaning up a temporary directory created
via `@TempDir` now include the root cause.
* ❓

==== Deprecations and Breaking Changes

* ❓
* The `dynamic` parallel execution strategy now allows the thread pool to be saturated by
default.

==== New Features and Improvements

* Promote various "experimental" APIs that have matured to "stable" including
`MethodOrderer`, `ClassOrderer`, `InvocationInterceptor`,
`LifecycleMethodExecutionExceptionHandler`, `@TempDir`, parallel execution annotations,
and others.
* `@RepeatedTest` can now be configured with a failure threshold which signifies the
number of failures after which remaining repetitions will be automatically skipped. See
the <<../user-guide/index.adoc#writing-tests-repeated-tests, User Guide>> for details.
* New `ArgumentsAccessor.getInvocationIndex()` method that supplies the index of a
`@ParameterizedTest` invocation.
* `DisplayNameGenerator` methods are now allowed to return `null`, in order to signal to
fall back to the default display name generator.
* `@EmptySource` now supports additional types, including `Collection` and `Map` subtypes
with a public no-arg constructor.
* `DisplayNameGenerator` methods are now allowed to return `null`, in order to signal to
fall back to the default display name generator.
* New `AnnotationBasedArgumentsProvider` convenience base class which implements both
`ArgumentsProvider` and `AnnotationConsumer`.
* New `AnnotationBasedArgumentConverter` convenience base class which implements both
`ArgumentConverter` and `AnnotationConsumer`.
* New `junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor` configuration
parameter to set the maximum pool size factor.
* New `junit.jupiter.execution.parallel.config.dynamic.saturate` configuration
parameter to disable pool saturation.


[[release-notes-5.10.0-M1-junit-vintage]]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
[[release-notes-5.9.3]]
== 5.9.3

*Date of Release:* ❓

*Scope:* ❓

For a complete list of all _closed_ issues and pull requests for this release, consult the
link:{junit5-repo}+/milestone/67?closed=1+[5.9.3] milestone page in the
JUnit repository on GitHub.


[[release-notes-5.9.3-junit-platform]]
=== JUnit Platform

==== Bug Fixes

* ❓

==== Deprecations and Breaking Changes

* ❓

==== New Features and Improvements

* ❓


[[release-notes-5.9.3-junit-jupiter]]
=== JUnit Jupiter

==== Bug Fixes

* Parameter types for _local_ `@MethodSource` factory method names are now validated. For
example, `@MethodSource("myFactory(example.NonexistentType)")` will now result in an
exception stating that `example.NonexistentType` cannot be resolved to a valid type.
* The syntax for parameter types in _local_ `@MethodSource` factory method names now
supports canonical array names -- for example, you may now specify `int[]` as in
`@MethodSource("myFactory(int[])"` instead of the _binary_ name `[I` as in
`@MethodSource("myFactory([I)"` (which was already supported) and
`@MethodSource("myFactory(java.lang.String[])` instead of
`@MethodSource("myFactory([Ljava.lang.String;)`.
* The search algorithm used to find `@MethodSource` factory methods now applies consistent
semantics for _local_ qualified method names and fully-qualified method names for
overloaded factory methods.
* Exceptions thrown for files that cannot be deleted when cleaning up a temporary
directory created via `@TempDir` now include the root cause.
* Lifecycle methods are allowed to be declared as `private` again for backwards
compatibility; however, using `private` visibility for lifecycle methods is strongly
discouraged and will be disallowed in a future release.

==== Deprecations and Breaking Changes

* ❓

==== New Features and Improvements

* The search algorithm used to find `@MethodSource` factory methods now falls back to
lenient search semantics when a factory method cannot be found by qualified name
(without a parameter list) and also provides better diagnostics when a unique factory
method cannot be found.


[[release-notes-5.9.3-junit-vintage]]
=== JUnit Vintage

==== Bug Fixes

* ❓

==== Deprecations and Breaking Changes

* ❓

==== New Features and Improvements

* ❓
94 changes: 76 additions & 18 deletions documentation/src/docs/asciidoc/user-guide/writing-tests.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -862,13 +862,13 @@ following demonstrates how to have `TestInfo` injected into a test constructor,
include::{testDir}/example/TestInfoDemo.java[tags=user_guide]
----

* `{RepetitionInfoParameterResolver}`: if a method parameter in a `@RepeatedTest`,
`@BeforeEach`, or `@AfterEach` method is of type `{RepetitionInfo}`, the
`RepetitionInfoParameterResolver` will supply an instance of `RepetitionInfo`.
`RepetitionInfo` can then be used to retrieve information about the current repetition
and the total number of repetitions for the corresponding `@RepeatedTest`. Note,
however, that `RepetitionInfoParameterResolver` is not registered outside the context
of a `@RepeatedTest`. See <<writing-tests-repeated-tests-examples>>.
* `{RepetitionExtension}`: if a method parameter in a `@RepeatedTest`, `@BeforeEach`, or
`@AfterEach` method is of type `{RepetitionInfo}`, the `RepetitionExtension` will supply
an instance of `RepetitionInfo`. `RepetitionInfo` can then be used to retrieve
information about the current repetition, the total number of repetitions, the number of
repetitions that have failed, and the failure threshold for the corresponding
`@RepeatedTest`. Note, however, that `RepetitionExtension` is not registered outside the
context of a `@RepeatedTest`. See <<writing-tests-repeated-tests-examples>>.

* `{TestReporterParameterResolver}`: if a constructor or method parameter is of type
`{TestReporter}`, the `TestReporterParameterResolver` will supply an instance of
Expand Down Expand Up @@ -1020,10 +1020,32 @@ void repeatedTest() {
}
----

In addition to specifying the number of repetitions, a custom display name can be
configured for each repetition via the `name` attribute of the `@RepeatedTest`
annotation. Furthermore, the display name can be a pattern composed of a combination of
static text and dynamic placeholders. The following placeholders are currently supported.
Since JUnit Jupiter 5.10, `@RepeatedTest` can be configured with a failure threshold which
signifies the number of failures after which remaining repetitions will be automatically
skipped. Set the `failureThreshold` attribute to a positive number less than the total
number of repetitions in order to skip the invocations of remaining repetitions after the
specified number of failures has been encountered.

For example, if you are using `@RepeatedTest` to repeatedly invoke a test that you suspect
to be _flaky_, a single failure is sufficient to demonstrate that the test is flaky, and
there is no need to invoke the remaining repetitions. To support that specific use case,
set `failureThreshold = 1`. You can alternatively set the threshold to a number greater
than 1 depending on your use case.

By default, the `failureThreshold` attribute is set to `Integer.MAX_VALUE`, signaling that
no failure threshold will be applied, which effectively means that the specified number of
repetitions will be invoked regardless of whether any repetitions fail.

WARNING: If the repetitions of a `@RepeatedTest` method are executed in parallel, no
guarantees can be made regarding the failure threshold. It is therefore recommended that a
`@RepeatedTest` method be annotated with `@Execution(SAME_THREAD)` when parallel execution
is configured. See <<writing-tests-parallel-execution>> for further details.

In addition to specifying the number of repetitions and failure threshold, a custom
display name can be configured for each repetition via the `name` attribute of the
`@RepeatedTest` annotation. Furthermore, the display name can be a pattern composed of a
combination of static text and dynamic placeholders. The following placeholders are
currently supported.

- `{displayName}`: display name of the `@RepeatedTest` method
- `{currentRepetition}`: the current repetition count
Expand All @@ -1039,21 +1061,26 @@ latter is equal to `"{displayName} :: repetition {currentRepetition} of
{totalRepetitions}"` which results in display names for individual repetitions like
`repeatedTest() :: repetition 1 of 10`, `repeatedTest() :: repetition 2 of 10`, etc.

In order to retrieve information about the current repetition and the total number of
repetitions programmatically, a developer can choose to have an instance of
`RepetitionInfo` injected into a `@RepeatedTest`, `@BeforeEach`, or `@AfterEach` method.
In order to retrieve information about the current repetition, the total number of
repetitions, the number of repetitions that have failed, and the failure threshold, a
developer can choose to have an instance of `{RepetitionInfo}` injected into a
`@RepeatedTest`, `@BeforeEach`, or `@AfterEach` method.

[[writing-tests-repeated-tests-examples]]
==== Repeated Test Examples

The `RepeatedTestsDemo` class at the end of this section demonstrates several examples of
repeated tests.

The `repeatedTest()` method is identical to example from the previous section; whereas,
The `repeatedTest()` method is identical to the example from the previous section; whereas,
`repeatedTestWithRepetitionInfo()` demonstrates how to have an instance of
`RepetitionInfo` injected into a test to access the total number of repetitions for the
current repeated test.

`repeatedTestWithFailureThreshold()` demonstrates how to set a failure threshold and
simulates an unexpected failure for every second repetition. The resulting behavior can be
viewed in the `ConsoleLauncher` output at the end of this section.

The next two methods demonstrate how to include a custom `@DisplayName` for the
`@RepeatedTest` method in the display name of each repetition. `customDisplayName()`
combines a custom display name with a custom pattern and then uses `TestInfo` to verify
Expand Down Expand Up @@ -1089,6 +1116,10 @@ INFO: About to execute repetition 2 of 5 for repeatedTestWithRepetitionInfo
INFO: About to execute repetition 3 of 5 for repeatedTestWithRepetitionInfo
INFO: About to execute repetition 4 of 5 for repeatedTestWithRepetitionInfo
INFO: About to execute repetition 5 of 5 for repeatedTestWithRepetitionInfo
INFO: About to execute repetition 1 of 8 for repeatedTestWithFailureThreshold
INFO: About to execute repetition 2 of 8 for repeatedTestWithFailureThreshold
INFO: About to execute repetition 3 of 8 for repeatedTestWithFailureThreshold
INFO: About to execute repetition 4 of 8 for repeatedTestWithFailureThreshold
INFO: About to execute repetition 1 of 1 for customDisplayName
INFO: About to execute repetition 1 of 1 for customDisplayNameWithLongPattern
INFO: About to execute repetition 1 of 5 for repeatedTestInGerman
Expand Down Expand Up @@ -1125,6 +1156,15 @@ When using the `ConsoleLauncher` with the unicode theme enabled, execution of
│ │ ├─ repetition 3 of 5 ✔
│ │ ├─ repetition 4 of 5 ✔
│ │ └─ repetition 5 of 5 ✔
│ ├─ repeatedTestWithFailureThreshold(RepetitionInfo) ✔
│ │ ├─ repetition 1 of 8 ✔
│ │ ├─ repetition 2 of 8 ✘ Boom!
│ │ ├─ repetition 3 of 8 ✔
│ │ ├─ repetition 4 of 8 ✘ Boom!
│ │ ├─ repetition 5 of 8 ↷ Failure threshold [2] exceeded
│ │ ├─ repetition 6 of 8 ↷ Failure threshold [2] exceeded
│ │ ├─ repetition 7 of 8 ↷ Failure threshold [2] exceeded
│ │ └─ repetition 8 of 8 ↷ Failure threshold [2] exceeded
│ ├─ Repeat! ✔
│ │ └─ Repeat! 1/1 ✔
│ ├─ Details... ✔
Expand Down Expand Up @@ -2355,6 +2395,8 @@ configuration parameter to one of the following options.
Computes the desired parallelism based on the number of available processors/cores
multiplied by the `junit.jupiter.execution.parallel.config.dynamic.factor`
configuration parameter (defaults to `1`).
The optional `junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor`
configuration parameter can be used to limit the maximum number of threads.

`fixed`::
Uses the mandatory `junit.jupiter.execution.parallel.config.fixed.parallelism`
Expand All @@ -2378,8 +2420,8 @@ of the synchronization mechanisms described in the next section, the `ForkJoinPo
is used behind the scenes may spawn additional threads to ensure execution continues with
sufficient parallelism.
If you require such guarantees, with Java 9+, it is possible to limit the maximum number
of concurrent threads by controlling the maximum pool size of the `fixed` and `custom`
strategies.
of concurrent threads by controlling the maximum pool size of the `dynamic`, `fixed` and
`custom` strategies.

[[writing-tests-parallel-execution-config-properties]]
===== Relevant properties
Expand Down Expand Up @@ -2426,6 +2468,22 @@ The following table lists relevant properties for configuring parallel execution
| a positive decimal number
| ```1.0```

| ```junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor```
| Factor to be multiplied by the number of available processors/cores and the value of
`junit.jupiter.execution.parallel.config.dynamic.factor` to determine the desired
parallelism for the ```dynamic``` configuration strategy
| a positive decimal number, must be greater than or equal to `1.0`
| 256 + the value of `junit.jupiter.execution.parallel.config.dynamic.factor` multiplied
by the number of available processors/cores

| ```junit.jupiter.execution.parallel.config.dynamic.saturate```
| Disable saturation of the underlying fork-join pool for the ```dynamic``` configuration
strategy
|
* `true`
* `false`
| ```true```

| ```junit.jupiter.execution.parallel.config.fixed.parallelism```
| Desired parallelism for the ```fixed``` configuration strategy
| a positive integer
Expand All @@ -2434,7 +2492,7 @@ The following table lists relevant properties for configuring parallel execution
| ```junit.jupiter.execution.parallel.config.fixed.max-pool-size```
| Desired maximum pool size of the underlying fork-join pool for the ```fixed```
configuration strategy
| a positive integer, must greater than or equal to `junit.jupiter.execution.parallel.config.fixed.parallelism`
| a positive integer, must be greater than or equal to `junit.jupiter.execution.parallel.config.fixed.parallelism`
| 256 + the value of `junit.jupiter.execution.parallel.config.fixed.parallelism`

| ```junit.jupiter.execution.parallel.config.fixed.saturate```
Expand Down
17 changes: 17 additions & 0 deletions documentation/src/test/java/example/RepeatedTestsDemo.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

// tag::user_guide[]
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

import java.util.logging.Logger;

Expand All @@ -21,6 +22,10 @@
import org.junit.jupiter.api.RepetitionInfo;
import org.junit.jupiter.api.TestInfo;

// end::user_guide[]
// Use fully-qualified names to avoid having them show up in the imports.
@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD)
// tag::user_guide[]
class RepeatedTestsDemo {

private Logger logger = // ...
Expand All @@ -47,6 +52,18 @@ void repeatedTestWithRepetitionInfo(RepetitionInfo repetitionInfo) {
assertEquals(5, repetitionInfo.getTotalRepetitions());
}

// end::user_guide[]
// Use fully-qualified name to avoid having it show up in the imports.
@org.junit.jupiter.api.Disabled("intentional failures would break the build")
// tag::user_guide[]
@RepeatedTest(value = 8, failureThreshold = 2)
void repeatedTestWithFailureThreshold(RepetitionInfo repetitionInfo) {
// Simulate unexpected failure every second repetition
if (repetitionInfo.getCurrentRepetition() % 2 == 0) {
fail("Boom!");
}
}

@RepeatedTest(value = 1, name = "{displayName} {currentRepetition}/{totalRepetitions}")
@DisplayName("Repeat!")
void customDisplayName(TestInfo testInfo) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,4 +82,7 @@ dependencies {
testRuntimeOnly(project(":junit-platform-reporting"))

testRuntimeOnly(bundleFromLibs("log4j"))
testRuntimeOnly(dependencyFromLibs("openTestReporting-events")) {
because("it's required to run tests via IntelliJ which does not consumed the shadowed jar of junit-platform-reporting")
}
}
Loading

0 comments on commit 22210ca

Please sign in to comment.