From a294b5117da6837644c00e8523e40454c90dedbd Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 4 Jun 2026 19:20:36 +0800 Subject: [PATCH 01/15] add config for specifying the number of execution for detecting flakiness --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 4 ++++ docs/options.md | 1 + 2 files changed, 5 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 382b612f21..5836ee3215 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2704,6 +2704,10 @@ class EMConfig { "Note that flakiness is now supported only for fuzzing REST APIs") var handleFlakiness = false + @Experimental + @Cfg("Specify the number of re-executions for detecting flakiness in tests") + var execNumForDetectFlaky = 1 + @Experimental @Cfg("Use environment variables to define the paths required by External Drivers. " + "This is necessary when the generated tests are executed on the different machine. " + diff --git a/docs/options.md b/docs/options.md index 4d6a04ef2c..5717df3b2e 100644 --- a/docs/options.md +++ b/docs/options.md @@ -272,6 +272,7 @@ There are 3 types of options: |`enableCustomizedMethodForScheduleTaskHandling`| __Boolean__. Whether to apply customized method (i.e., implement 'customizeScheduleTaskInvocation' for invoking schedule task) to invoke schedule task. *Default value*: `false`.| |`enableRPCCustomizedTestOutput`| __Boolean__. Whether to enable customized RPC Test output if 'customizeRPCTestOutput' is implemented. *Default value*: `false`.| |`enableWriteSnapshotTests`| __Boolean__. Enable to print snapshots of the generated tests during the search in an interval defined in snapshotsInterval. *Default value*: `false`.| +|`execNumForDetectFlaky`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Default value*: `1`.| |`executiveSummary`| __Boolean__. Generate an executive summary, containing an example of each category of potential faults found.NOTE: This option is only meaningful when used in conjunction with test suite splitting. *Default value*: `false`.| |`expectationsActive`| __Boolean__. Enable Expectation Generation. If enabled, expectations will be generated. A variable called expectationsMasterSwitch is added to the test suite, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail. *Default value*: `false`.| |`exportTestCasesDuringSeeding`| __Boolean__. Whether to export test cases during seeding as a separate file. *Default value*: `false`.| From e0c985b974a5e47676cd0b5e85ef62eeaa5979ad Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 4 Jun 2026 21:18:07 +0800 Subject: [PATCH 02/15] update config info --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 2 +- docs/options.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 5836ee3215..da0aa64d0c 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2706,7 +2706,7 @@ class EMConfig { @Experimental @Cfg("Specify the number of re-executions for detecting flakiness in tests") - var execNumForDetectFlaky = 1 + var execNumForDetectFlakiness = 0 @Experimental @Cfg("Use environment variables to define the paths required by External Drivers. " + diff --git a/docs/options.md b/docs/options.md index 5717df3b2e..cc17eed569 100644 --- a/docs/options.md +++ b/docs/options.md @@ -272,7 +272,7 @@ There are 3 types of options: |`enableCustomizedMethodForScheduleTaskHandling`| __Boolean__. Whether to apply customized method (i.e., implement 'customizeScheduleTaskInvocation' for invoking schedule task) to invoke schedule task. *Default value*: `false`.| |`enableRPCCustomizedTestOutput`| __Boolean__. Whether to enable customized RPC Test output if 'customizeRPCTestOutput' is implemented. *Default value*: `false`.| |`enableWriteSnapshotTests`| __Boolean__. Enable to print snapshots of the generated tests during the search in an interval defined in snapshotsInterval. *Default value*: `false`.| -|`execNumForDetectFlaky`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Default value*: `1`.| +|`execNumForDetectFlakiness`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Default value*: `0`.| |`executiveSummary`| __Boolean__. Generate an executive summary, containing an example of each category of potential faults found.NOTE: This option is only meaningful when used in conjunction with test suite splitting. *Default value*: `false`.| |`expectationsActive`| __Boolean__. Enable Expectation Generation. If enabled, expectations will be generated. A variable called expectationsMasterSwitch is added to the test suite, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail. *Default value*: `false`.| |`exportTestCasesDuringSeeding`| __Boolean__. Whether to export test cases during seeding as a separate file. *Default value*: `false`.| From 3ffcad2a41cd7a0566eca996046e55156ac99645 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 4 Jun 2026 21:18:16 +0800 Subject: [PATCH 03/15] start to handle flaky body --- .../main/kotlin/org/evomaster/core/Main.kt | 2 +- .../core/problem/httpws/HttpWsCallResult.kt | 36 ++++++++++++++++--- .../core/search/service/FlakinessDetector.kt | 31 +++++++++------- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 239cb1aedb..2d68e55a04 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -450,7 +450,7 @@ class Main { epc.markStartingFlakiness() val flakinessDetector = injector.getInstance(Key.get(object : TypeLiteral>() {})) - flakinessDetector.reexecuteToDetectFlakiness() + flakinessDetector.reexecuteToDetectFlakiness(config.execNumForDetectFlakiness) } else -> { LoggingUtil.getInfoLogger() .warn("Flakiness detection phase currently not handled for problem type: ${config.problemType}") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index 35c49d1147..428f3cf3f3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -37,6 +37,7 @@ abstract class HttpWsCallResult : EnterpriseActionResult { const val FLAKY_BODY = "FLAKY_BODY" const val FLAKY_BODY_TYPE = "FLAKY_BODY_TYPE" const val FLAKY_ERROR_MESSAGE = "FLAKY_ERROR_MESSAGE" + const val FLAKY_DETECTION_TIMES = "FLAKY_DETECTION_TIMES" } /** @@ -176,20 +177,32 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun setFlakyStatusCode(code: Int) = addResultValue(FLAKY_STATUS_CODE, code.toString()) fun getFlakyStatusCode() : Int? = getResultValue(FLAKY_STATUS_CODE)?.toInt() - fun setFlakyBody(body: String) = addResultValue(FLAKY_BODY, body) - fun getFlakyBody() : String? = getResultValue(FLAKY_BODY) + fun setFlakyBody(body: String, execIndex : Int?=null) = addResultValue(flakyInfoKey(FLAKY_BODY, execIndex), body) + fun getFlakyBody(execIndex : Int? = null) : String? = getResultValue(flakyInfoKey(FLAKY_BODY, execIndex)) + fun getFlakyBodies() : List? = getMultipleFlakyInfo(FLAKY_BODY) + fun containFlakyBody(flakyBody: String) : Boolean { + if (getFlakyBodies() == null) return false + return getFlakyBodies()!!.contains(flakyBody) + } fun setFlakyBodyType(type: MediaType) = addResultValue(FLAKY_BODY_TYPE, type.toString()) fun getFlakyBodyType() : MediaType? = getResultValue(FLAKY_BODY_TYPE)?.let { MediaType.valueOf(it) } - fun setFlakiness(previous: HttpWsCallResult){ + fun setFlakyDetectionTimes(times: Int) { + if (getFlakyDetectionTimes() == null || getFlakyDetectionTimes()!! < times) addResultValue(FLAKY_DETECTION_TIMES, times.toString()) + } + fun getFlakyDetectionTimes() : Int? = getResultValue(FLAKY_DETECTION_TIMES)?.toInt() + + fun setFlakiness(previous: HttpWsCallResult, execIndex: Int){ + setFlakyDetectionTimes(execIndex) + val pStatusCode = previous.getStatusCode() if (pStatusCode != null && pStatusCode != getStatusCode()) { setFlakyStatusCode(pStatusCode) } val pBody = previous.getBody() - if (pBody != null && pBody != getBody()) { + if (pBody != null && pBody != getBody() && !containFlakyBody(pBody)) { setFlakyBody(pBody) } @@ -203,4 +216,19 @@ abstract class HttpWsCallResult : EnterpriseActionResult { setFlakyErrorMessage(pMessage) } } + + private fun flakyInfoKey(infoKey: String, execIndex : Int? = null): String = "${infoKey}_${execIndex?.toString() ?: ""}" + + private fun getMultipleFlakyInfo(infoKey: String): List?{ + val repeat = getFlakyDetectionTimes() + + if (repeat != null && repeat > 0) { + return (1..repeat).map { + getResultValue(flakyInfoKey(infoKey, it)) + }.plus(getResultValue(infoKey)).filterNotNull().run { + ifEmpty { null } + } + } + return null + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt index 195e4c16f9..8aea638624 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt @@ -37,7 +37,7 @@ class FlakinessDetector : TimeBoxedPhase { private lateinit var epc: ExecutionPhaseController override fun applyPhase() { - reexecuteToDetectFlakiness() + reexecuteToDetectFlakiness(config.execNumForDetectFlakiness) } override fun hasPhaseTimedOut(): Boolean { @@ -47,7 +47,7 @@ class FlakinessDetector : TimeBoxedPhase { /** * re-execute individuals in archive for identifying flakiness */ - fun reexecuteToDetectFlakiness() : Solution{ + fun reexecuteToDetectFlakiness(execNum : Int) : Solution{ /* here we rely on extractSolution returning actual references of individuals in the archive, and not copies */ @@ -59,11 +59,13 @@ class FlakinessDetector : TimeBoxedPhase { if(hasPhaseTimedOut()) break - val ei = fitness.computeWholeAchievedCoverageForPostProcessing(ci.individual) - if(ei == null){ - log.warn("Failed to re-evaluate individual during flakiness analysis.") - }else{ - checkAndMarkConsistency(ei, ci) + for (execIndex in 1..execNum){ + val ei = fitness.computeWholeAchievedCoverageForPostProcessing(ci.individual) + if(ei == null){ + log.warn("Failed to re-evaluate individual at index ($execIndex) during flakiness analysis.") + }else{ + checkAndMarkConsistency(ei, ci, execIndex) + } } } @@ -74,8 +76,9 @@ class FlakinessDetector : TimeBoxedPhase { * This might have side-effects will be applied in the archive * compare [inArchive] with [other] to check if the action results are same, the inconsistent info will be saved in [inArchive] evaluated individual * @param inArchive the evaluated individual which saves info of flakiness + * @param indexExecN the index of execution times, eg, 1st, 2nd */ - fun checkAndMarkConsistency(other: EvaluatedIndividual, inArchive: EvaluatedIndividual){ + fun checkAndMarkConsistency(other: EvaluatedIndividual, inArchive: EvaluatedIndividual, indexExecN: Int){ val previousActions = other.evaluatedMainActions() val currentActions = inArchive.evaluatedMainActions() @@ -89,7 +92,7 @@ class FlakinessDetector : TimeBoxedPhase { val action = it.action if(action is HttpWsAction){ if (it.result is HttpWsCallResult){ - handleFlakinessInActionResult(it.result, previousActions[index].result as HttpWsCallResult) + handleFlakinessInActionResult(it.result, previousActions[index].result as HttpWsCallResult, indexExecN) } } @@ -98,9 +101,13 @@ class FlakinessDetector : TimeBoxedPhase { private fun handleFlakinessInActionResult( resultToUpdate: HttpWsCallResult, - other: HttpWsCallResult + other: HttpWsCallResult, + indexExecN: Int ) { + // set execution index + resultToUpdate.setFlakyDetectionTimes(indexExecN) + // handle status code val rStatus = resultToUpdate.getStatusCode() val oStatus = other.getStatusCode() @@ -114,11 +121,11 @@ class FlakinessDetector : TimeBoxedPhase { val oBody = other.getBody() if (oBody != null) { - if (rBody != oBody) { + if (rBody != oBody && !resultToUpdate.containFlakyBody(oBody)) { resultToUpdate.setFlakyBody(oBody) } else { val normO = derive(oBody) - if (rBody != normO) { + if (rBody != normO && !resultToUpdate.containFlakyBody(oBody)) { resultToUpdate.setFlakyBody(oBody) } } From 22543242da09a72e1a168a2b00f2798c1adbad46 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 09:31:43 +0800 Subject: [PATCH 04/15] fix compilation failure --- .../org/evomaster/core/problem/httpws/HttpWsCallResult.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index d87532d48b..dc39bb534e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -229,7 +229,7 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } fun getFlakyDetectionTimes() : Int? = getResultValue(FLAKY_DETECTION_TIMES)?.toInt() - fun setFlakiness(previous: HttpWsCallResult, execIndex: Int){ + fun setFlakiness(previous: HttpWsCallResult, execIndex: Int = 1){ setFlakyDetectionTimes(execIndex) val pStatusCode = previous.getStatusCode() From e6d84d1861b73c690089ec72e7f74b410195a035 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 15:41:14 +0800 Subject: [PATCH 05/15] fix execNumForDetectFlakiness config --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 3 ++- docs/options.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 984996e1f6..8b53da4225 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2794,8 +2794,9 @@ class EMConfig { var handleFlakiness = false @Experimental + @Min(1.0) @Cfg("Specify the number of re-executions for detecting flakiness in tests") - var execNumForDetectFlakiness = 0 + var execNumForDetectFlakiness = 1 @Experimental @Cfg("Use environment variables to define the paths required by External Drivers. " + diff --git a/docs/options.md b/docs/options.md index 9968c48f4a..2df2ef6f49 100644 --- a/docs/options.md +++ b/docs/options.md @@ -284,7 +284,7 @@ There are 3 types of options: |`enableCustomizedMethodForScheduleTaskHandling`| __Boolean__. Whether to apply customized method (i.e., implement 'customizeScheduleTaskInvocation' for invoking schedule task) to invoke schedule task. *Default value*: `false`.| |`enableRPCCustomizedTestOutput`| __Boolean__. Whether to enable customized RPC Test output if 'customizeRPCTestOutput' is implemented. *Default value*: `false`.| |`enableWriteSnapshotTests`| __Boolean__. Enable to print snapshots of the generated tests during the search in an interval defined in snapshotsInterval. *Default value*: `false`.| -|`execNumForDetectFlakiness`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Default value*: `0`.| +|`execNumForDetectFlakiness`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Constraints*: `min=1.0`. *Default value*: `1`.| |`executiveSummary`| __Boolean__. Generate an executive summary, containing an example of each category of potential faults found.NOTE: This option is only meaningful when used in conjunction with test suite splitting. *Default value*: `false`.| |`expectationsActive`| __Boolean__. Enable Expectation Generation. If enabled, expectations will be generated. A variable called expectationsMasterSwitch is added to the test suite, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail. *Default value*: `false`.| |`exportTestCasesDuringSeeding`| __Boolean__. Whether to export test cases during seeding as a separate file. *Default value*: `false`.| From 9113892fc938bb0b27554c6243afbaabef66dcf3 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 17:22:56 +0800 Subject: [PATCH 06/15] modify configs for flakiness handling --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 10 ++++++++-- docs/options.md | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 8b53da4225..09b1dff31a 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2794,8 +2794,14 @@ class EMConfig { var handleFlakiness = false @Experimental - @Min(1.0) - @Cfg("Specify the number of re-executions for detecting flakiness in tests") + @DependsOnTrueFor("handleFlakiness") + @Cfg("Specify whether to infer potential flakiness statically from response values, such as timestamps, UUIDs, hashes and runtime-specific messages.") + var enableStaticFlakyInference = true + + @Experimental + @Min(0.0) + @DependsOnTrueFor("handleFlakiness") + @Cfg("Specify the number of re-executions for detecting flakiness in tests. Set to 0 to disable re-execution based flakiness detection.") var execNumForDetectFlakiness = 1 @Experimental diff --git a/docs/options.md b/docs/options.md index 2df2ef6f49..e870a9a1c3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -283,8 +283,9 @@ There are 3 types of options: |`enableCustomizedMethodForMockObjectHandling`| __Boolean__. Whether to apply customized method (i.e., implement 'customizeMockingRPCExternalService' for external services or 'customizeMockingDatabase' for database) to handle mock object. *Default value*: `false`.| |`enableCustomizedMethodForScheduleTaskHandling`| __Boolean__. Whether to apply customized method (i.e., implement 'customizeScheduleTaskInvocation' for invoking schedule task) to invoke schedule task. *Default value*: `false`.| |`enableRPCCustomizedTestOutput`| __Boolean__. Whether to enable customized RPC Test output if 'customizeRPCTestOutput' is implemented. *Default value*: `false`.| +|`enableStaticFlakyInference`| __Boolean__. Specify whether to infer potential flakiness statically from response values, such as timestamps, UUIDs, hashes and runtime-specific messages. *Depends on*: `handleFlakiness=true`. *Default value*: `true`.| |`enableWriteSnapshotTests`| __Boolean__. Enable to print snapshots of the generated tests during the search in an interval defined in snapshotsInterval. *Default value*: `false`.| -|`execNumForDetectFlakiness`| __Int__. Specify the number of re-executions for detecting flakiness in tests. *Constraints*: `min=1.0`. *Default value*: `1`.| +|`execNumForDetectFlakiness`| __Int__. Specify the number of re-executions for detecting flakiness in tests. Set to 0 to disable re-execution based flakiness detection. *Constraints*: `min=0.0`. *Depends on*: `handleFlakiness=true`. *Default value*: `1`.| |`executiveSummary`| __Boolean__. Generate an executive summary, containing an example of each category of potential faults found.NOTE: This option is only meaningful when used in conjunction with test suite splitting. *Default value*: `false`.| |`expectationsActive`| __Boolean__. Enable Expectation Generation. If enabled, expectations will be generated. A variable called expectationsMasterSwitch is added to the test suite, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail. *Default value*: `false`.| |`exportTestCasesDuringSeeding`| __Boolean__. Whether to export test cases during seeding as a separate file. *Default value*: `false`.| From ada85fa59d112defa6e411d8ec52ac84d1083a1a Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 17:23:42 +0800 Subject: [PATCH 07/15] modify the entry of flakiness detection phase --- core/src/main/kotlin/org/evomaster/core/Main.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 77bb2805ca..c2aac6307a 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -451,7 +451,7 @@ class Main { epc.markStartingFlakiness() val flakinessDetector = injector.getInstance(Key.get(object : TypeLiteral>() {})) - flakinessDetector.reexecuteToDetectFlakiness(config.execNumForDetectFlakiness) + flakinessDetector.applyPhase() } else -> { LoggingUtil.getInfoLogger() .warn("Flakiness detection phase currently not handled for problem type: ${config.problemType}") From 76d4602afc94ef37ed0213b275f4c4ad9c94eee4 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 17:24:59 +0800 Subject: [PATCH 08/15] use flaky observation to record results of multiple execution and static inference --- .../core/problem/httpws/HttpWsCallResult.kt | 207 +++++++++++++----- .../core/search/service/FlakinessDetector.kt | 86 +++----- 2 files changed, 186 insertions(+), 107 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index dc39bb534e..c367c5573d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -5,6 +5,7 @@ import com.google.gson.Gson import com.google.gson.JsonSyntaxException import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.core.utils.FlakinessInferenceUtil import javax.ws.rs.core.MediaType abstract class HttpWsCallResult : EnterpriseActionResult { @@ -12,7 +13,9 @@ abstract class HttpWsCallResult : EnterpriseActionResult { constructor(sourceLocalId: String, stopping: Boolean = false) : super(sourceLocalId,stopping) @VisibleForTesting - internal constructor(other: HttpWsCallResult) : super(other) + internal constructor(other: HttpWsCallResult) : super(other) { + flakyObservations.addAll(other.flakyObservations) + } companion object { const val STATUS_CODE = "STATUS_CODE" @@ -34,13 +37,41 @@ abstract class HttpWsCallResult : EnterpriseActionResult { const val VULNERABLE_SQLI = "VULNERABLE_SQLI" - const val FLAKY_STATUS_CODE = "FLAKY_STATUS_CODE" - const val FLAKY_BODY = "FLAKY_BODY" - const val FLAKY_BODY_TYPE = "FLAKY_BODY_TYPE" - const val FLAKY_ERROR_MESSAGE = "FLAKY_ERROR_MESSAGE" - const val FLAKY_DETECTION_TIMES = "FLAKY_DETECTION_TIMES" } + enum class ResponseField { + STATUS_CODE, + BODY, + BODY_TYPE, + ERROR_MESSAGE, + TOO_LARGE_BODY, + INFINITE_LOOP, + TIMEDOUT, + TCP_PROBLEM, + INVALID_HTTP, + LOCATION, + ALLOW, + RESPONSE_TIME_MS + } + + enum class FlakyObservationSource { + RE_EXECUTION, + STATIC_INFERENCE + } + + data class FlakyObservation( + val source: FlakyObservationSource, + val execIndex: Int?, + val differences: Map + ) + + data class FieldVariation( + val field: ResponseField, + val valuesByExecIndex: Map + ) + + private val flakyObservations: MutableList = mutableListOf() + /** * In some cases (eg infinite loop redirection), a HTTP call * might fail, and, as such, we might not have an actual "result" @@ -207,64 +238,140 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun getResponseTimeMs(): Long? = getResultValue(RESPONSE_TIME_MS)?.toLong() - fun setFlakyErrorMessage(msg: String) = addResultValue(FLAKY_ERROR_MESSAGE, msg) - fun getFlakyErrorMessage() : String? = getResultValue(FLAKY_ERROR_MESSAGE) - - fun setFlakyStatusCode(code: Int) = addResultValue(FLAKY_STATUS_CODE, code.toString()) - fun getFlakyStatusCode() : Int? = getResultValue(FLAKY_STATUS_CODE)?.toInt() - - fun setFlakyBody(body: String, execIndex : Int?=null) = addResultValue(flakyInfoKey(FLAKY_BODY, execIndex), body) - fun getFlakyBody(execIndex : Int? = null) : String? = getResultValue(flakyInfoKey(FLAKY_BODY, execIndex)) - fun getFlakyBodies() : List? = getMultipleFlakyInfo(FLAKY_BODY) - fun containFlakyBody(flakyBody: String) : Boolean { - if (getFlakyBodies() == null) return false - return getFlakyBodies()!!.contains(flakyBody) - } + fun recordFlakyObservation(other: HttpWsCallResult, execIndex: Int) { + val differences = responseFieldExtractors() + .mapNotNull { spec -> + val baseline = spec.extract(this) + val observed = spec.extract(other) - fun setFlakyBodyType(type: MediaType) = addResultValue(FLAKY_BODY_TYPE, type.toString()) - fun getFlakyBodyType() : MediaType? = getResultValue(FLAKY_BODY_TYPE)?.let { MediaType.valueOf(it) } + if (observed != null && observed != baseline) { + spec.field to observed + } else { + null + } + } + .toMap() - fun setFlakyDetectionTimes(times: Int) { - if (getFlakyDetectionTimes() == null || getFlakyDetectionTimes()!! < times) addResultValue(FLAKY_DETECTION_TIMES, times.toString()) + if (differences.isNotEmpty()) { + flakyObservations.removeIf { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex == execIndex } + flakyObservations.add(FlakyObservation(FlakyObservationSource.RE_EXECUTION, execIndex, differences)) + } } - fun getFlakyDetectionTimes() : Int? = getResultValue(FLAKY_DETECTION_TIMES)?.toInt() - fun setFlakiness(previous: HttpWsCallResult, execIndex: Int = 1){ - setFlakyDetectionTimes(execIndex) + fun recordStaticFlakyInference() { + val differences = mutableMapOf() - val pStatusCode = previous.getStatusCode() - if (pStatusCode != null && pStatusCode != getStatusCode()) { - setFlakyStatusCode(pStatusCode) - } - - val pBody = previous.getBody() - if (pBody != null && pBody != getBody() && !containFlakyBody(pBody)) { - setFlakyBody(pBody) + getBody()?.let { + val normalized = FlakinessInferenceUtil.derive(it) + if (normalized != it) { + differences[ResponseField.BODY] = normalized + } } - val pBodyType = previous.getBodyType() - if (pBodyType != null && pBodyType != getBodyType()) { - setFlakyBodyType(pBodyType) + getErrorMessage()?.let { + val normalized = FlakinessInferenceUtil.derive(it) + if (normalized != it) { + differences[ResponseField.ERROR_MESSAGE] = normalized + } } - val pMessage = previous.getErrorMessage() - if (pMessage != null && pMessage != getErrorMessage()) { - setFlakyErrorMessage(pMessage) + if (differences.isNotEmpty()) { + flakyObservations.removeIf { it.source == FlakyObservationSource.STATIC_INFERENCE } + flakyObservations.add(FlakyObservation(FlakyObservationSource.STATIC_INFERENCE, null, differences)) } } - private fun flakyInfoKey(infoKey: String, execIndex : Int? = null): String = "${infoKey}_${execIndex?.toString() ?: ""}" + fun getFlakyObservations(): List = flakyObservations.toList() + + fun getFlakyObservation(execIndex: Int): FlakyObservation? = + flakyObservations.find { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex == execIndex } - private fun getMultipleFlakyInfo(infoKey: String): List?{ - val repeat = getFlakyDetectionTimes() + fun getStaticFlakyObservation(): FlakyObservation? = + flakyObservations.find { it.source == FlakyObservationSource.STATIC_INFERENCE } - if (repeat != null && repeat > 0) { - return (1..repeat).map { - getResultValue(flakyInfoKey(infoKey, it)) - }.plus(getResultValue(infoKey)).filterNotNull().run { - ifEmpty { null } + fun hasFlakyField(field: ResponseField): Boolean = + flakyObservations.any { it.differences.containsKey(field) } + + fun getFlakyValues(field: ResponseField): List = + flakyObservations + .sortedWith(compareBy { it.source }.thenBy { it.execIndex ?: Int.MAX_VALUE }) + .mapNotNull { observation -> + if (observation.differences.containsKey(field)) { + observation.differences[field] + } else { + null + } } - } - return null + + fun getFlakyVariation(field: ResponseField): FieldVariation? { + val values = flakyObservations + .filter { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex != null } + .sortedBy { it.execIndex } + .mapNotNull { observation -> + if (observation.differences.containsKey(field)) { + observation.execIndex!! to observation.differences[field] + } else { + null + } + } + .toMap() + + return if (values.isEmpty()) null else FieldVariation(field, values) + } + + fun setFlakyErrorMessage(msg: String) = addFlakyDifference(ResponseField.ERROR_MESSAGE, msg) + fun getFlakyErrorMessage() : String? = getFirstFlakyValue(ResponseField.ERROR_MESSAGE) + + fun setFlakyStatusCode(code: Int) = addFlakyDifference(ResponseField.STATUS_CODE, code.toString()) + fun getFlakyStatusCode() : Int? = getFirstFlakyValue(ResponseField.STATUS_CODE)?.toInt() + + fun setFlakyBody(body: String, execIndex : Int?=null) = addFlakyDifference(ResponseField.BODY, body, execIndex) + fun getFlakyBody(execIndex : Int? = null) : String? = + if (execIndex == null) getFirstFlakyValue(ResponseField.BODY) + else getFlakyObservation(execIndex)?.differences?.get(ResponseField.BODY) + + fun getFlakyBodies() : List? = getFlakyValues(ResponseField.BODY).filterNotNull().ifEmpty { null } + fun containFlakyBody(flakyBody: String) : Boolean = getFlakyBodies()?.contains(flakyBody) ?: false + + fun setFlakyBodyType(type: MediaType) = addFlakyDifference(ResponseField.BODY_TYPE, type.toString()) + fun getFlakyBodyType() : MediaType? = getFirstFlakyValue(ResponseField.BODY_TYPE)?.let { MediaType.valueOf(it) } + + fun setFlakiness(previous: HttpWsCallResult, execIndex: Int = 1) = + recordFlakyObservation(previous, execIndex) + + private fun addFlakyDifference(field: ResponseField, value: String?, execIndex: Int? = null) { + val index = execIndex ?: 1 + + val existing = flakyObservations.find { it.execIndex == index } + val differences = existing?.differences?.toMutableMap() ?: mutableMapOf() + differences[field] = value + + flakyObservations.removeIf { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex == index } + flakyObservations.add(FlakyObservation(FlakyObservationSource.RE_EXECUTION, index, differences)) } + + private fun getFirstFlakyValue(field: ResponseField): String? = + flakyObservations + .sortedBy { it.execIndex } + .firstNotNullOfOrNull { it.differences[field] } + + private data class ResponseFieldSpec( + val field: ResponseField, + val extract: (HttpWsCallResult) -> String? + ) + + private fun responseFieldExtractors(): List = listOf( + ResponseFieldSpec(ResponseField.STATUS_CODE) { it.getStatusCode()?.toString() }, + ResponseFieldSpec(ResponseField.BODY) { it.getBody() }, + ResponseFieldSpec(ResponseField.BODY_TYPE) { it.getBodyType()?.toString() }, + ResponseFieldSpec(ResponseField.ERROR_MESSAGE) { it.getErrorMessage() }, + ResponseFieldSpec(ResponseField.TOO_LARGE_BODY) { it.getTooLargeBody().toString() }, + ResponseFieldSpec(ResponseField.INFINITE_LOOP) { it.getInfiniteLoop().toString() }, + ResponseFieldSpec(ResponseField.TIMEDOUT) { it.getTimedout().toString() }, + ResponseFieldSpec(ResponseField.TCP_PROBLEM) { it.getTcpProblem().toString() }, + ResponseFieldSpec(ResponseField.INVALID_HTTP) { it.getInvalidHTTP().toString() }, + ResponseFieldSpec(ResponseField.LOCATION) { it.getLocation() }, + ResponseFieldSpec(ResponseField.ALLOW) { it.getAllow() }, + ResponseFieldSpec(ResponseField.RESPONSE_TIME_MS) { it.getResponseTimeMs()?.toString() } + ) } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt index 34d08513e0..55446a4908 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt @@ -10,7 +10,6 @@ import org.evomaster.core.search.Individual import org.evomaster.core.search.Solution import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.time.TimeBoxedPhase -import org.evomaster.core.utils.FlakinessInferenceUtil.derive import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -37,7 +36,12 @@ class FlakinessDetector : TimeBoxedPhase { private lateinit var epc: ExecutionPhaseController override fun applyPhase() { - reexecuteToDetectFlakiness(config.execNumForDetectFlakiness) + if (config.enableStaticFlakyInference) { + inferStaticFlakiness() + } + if (config.execNumForDetectFlakiness > 0) { + reexecuteToDetectFlakiness(config.execNumForDetectFlakiness) + } } override fun hasPhaseTimedOut(): Boolean { @@ -72,6 +76,27 @@ class FlakinessDetector : TimeBoxedPhase { return archive.extractSolution() } + /** + * Infer potential flaky response values without re-executing the SUT. + */ + fun inferStaticFlakiness(): Solution { + val currentIndividuals = archive.extractSolution().individuals + + LoggingUtil.getInfoLogger().info("Inferring static flakiness for ${currentIndividuals.size} individuals.") + + for (ci in currentIndividuals) { + if(hasPhaseTimedOut()) break + + ci.evaluatedMainActions() + .filter { it.action is HttpWsAction && it.result is HttpWsCallResult } + .forEach { + (it.result as HttpWsCallResult).recordStaticFlakyInference() + } + } + + return archive.extractSolution() + } + /** * This might have side-effects will be applied in the archive * compare [inArchive] with [other] to check if the action results are same, the inconsistent info will be saved in [inArchive] evaluated individual @@ -104,60 +129,7 @@ class FlakinessDetector : TimeBoxedPhase { other: HttpWsCallResult, indexExecN: Int ) { - - // set execution index - resultToUpdate.setFlakyDetectionTimes(indexExecN) - - // handle status code - val rStatus = resultToUpdate.getStatusCode() - val oStatus = other.getStatusCode() - - if (oStatus != null && rStatus != oStatus) { - resultToUpdate.setFlakyStatusCode(oStatus) - } - - // handle body - val rBody = resultToUpdate.getBody() - val oBody = other.getBody() - - if (oBody != null) { - if (rBody != oBody && !resultToUpdate.containFlakyBody(oBody)) { - resultToUpdate.setFlakyBody(oBody) - } else { - val normO = derive(oBody) - if (rBody != normO && !resultToUpdate.containFlakyBody(oBody)) { - resultToUpdate.setFlakyBody(normO) - } - } - } - - // handle body type - val rType = resultToUpdate.getBodyType() - val oType = other.getBodyType() - - if (oType != null && rType != oType) { - resultToUpdate.setFlakyBodyType(oType) - } - - // handle error message - val rMsg = resultToUpdate.getErrorMessage() - val oMsg = other.getErrorMessage() - - if (oMsg != null) { - if (rMsg != oMsg) { - resultToUpdate.setFlakyErrorMessage(oMsg) - } else { - /* - there may be chance that the flakiness is not identified with the near execution. - However, we use predefined regex to infer potential flaky value for known flaky source, - such as UUID, time. - */ - val normO = derive(oMsg) - if (rMsg != normO) { - resultToUpdate.setFlakyErrorMessage(normO) - } - } - } + resultToUpdate.recordFlakyObservation(other, indexExecN) } -} \ No newline at end of file +} From a35ebef65160611447432115951605765441b2fa Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 17:25:11 +0800 Subject: [PATCH 09/15] update unit tests --- .../core/problem/rest/RestCallResultTest.kt | 105 +++++++++++++----- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt index f017f027a8..d7cbd11203 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt @@ -1,6 +1,9 @@ package org.evomaster.core.problem.rest import org.evomaster.core.problem.rest.data.RestCallResult +import org.evomaster.core.problem.httpws.HttpWsCallResult.FlakyObservationSource +import org.evomaster.core.problem.httpws.HttpWsCallResult.ResponseField +import org.evomaster.core.utils.FlakinessInferenceUtil.UUID_MARKER import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull @@ -133,19 +136,24 @@ internal class RestCallResultTest { @ParameterizedTest @MethodSource("getFlakyBodyDataProvider") fun testSetFlakinessInBody(same: String, diff: String){ - val body = createCallResult(same) - val same = createCallResult(same) - val diff = createCallResult(diff) + val original = createCallResult(same) + val sameResult = createCallResult(same) + val differentResult = createCallResult(diff) - body.setFlakiness(same) - assertNotNull(body.getBodyType()) + original.recordFlakyObservation(sameResult, 1) + assertNotNull(original.getBodyType()) - assertNull(body.getFlakyBody()) - assertNull(body.getFlakyBodyType()) + assertNull(original.getFlakyObservation(1)) + assertNull(original.getFlakyVariation(ResponseField.BODY)) + assertNull(original.getFlakyVariation(ResponseField.BODY_TYPE)) - body.setFlakiness(diff) - assertEquals(diff.getBody(), body.getFlakyBody()) - assertNull(body.getFlakyBodyType()) + original.recordFlakyObservation(differentResult, 2) + + val observation = original.getFlakyObservation(2) + assertNotNull(observation) + assertEquals(differentResult.getBody(), observation!!.differences[ResponseField.BODY]) + assertEquals(mapOf(2 to differentResult.getBody()), original.getFlakyVariation(ResponseField.BODY)!!.valuesByExecIndex) + assertNull(original.getFlakyVariation(ResponseField.BODY_TYPE)) } @@ -157,25 +165,72 @@ internal class RestCallResultTest { val msg = "hello" val diffmsg = "hello!" - val body = RestCallResult("1") - val same = RestCallResult("2") - val diff = RestCallResult("3") + val original = RestCallResult("1") + val sameResult = RestCallResult("2") + val differentResult = RestCallResult("3") + + original.setStatusCode(code) + sameResult.setStatusCode(code) + differentResult.setStatusCode(diffcode) + + original.setErrorMessage(msg) + sameResult.setErrorMessage(msg) + differentResult.setErrorMessage(diffmsg) + + original.recordFlakyObservation(sameResult, 1) + assertNull(original.getFlakyObservation(1)) + assertNull(original.getFlakyVariation(ResponseField.STATUS_CODE)) + assertNull(original.getFlakyVariation(ResponseField.ERROR_MESSAGE)) + + original.recordFlakyObservation(differentResult, 2) - body.setStatusCode(code) - same.setStatusCode(code) - diff.setStatusCode(diffcode) + val observation = original.getFlakyObservation(2) + assertNotNull(observation) + assertEquals(diffcode.toString(), observation!!.differences[ResponseField.STATUS_CODE]) + assertEquals(diffmsg, observation.differences[ResponseField.ERROR_MESSAGE]) + assertEquals(mapOf(2 to diffcode.toString()), original.getFlakyVariation(ResponseField.STATUS_CODE)!!.valuesByExecIndex) + assertEquals(mapOf(2 to diffmsg), original.getFlakyVariation(ResponseField.ERROR_MESSAGE)!!.valuesByExecIndex) + + } + + @Test + fun testFlakinessIsRecordedAsExecutionDeltas(){ + val original = createCallResult("""{"id":"42"}""") + original.setStatusCode(200) - body.setErrorMessage(msg) - same.setErrorMessage(msg) - diff.setErrorMessage(diffmsg) + val same = createCallResult("""{"id":"42"}""") + same.setStatusCode(200) + + val different = createCallResult("""{"id":"735"}""") + different.setStatusCode(500) + + original.recordFlakyObservation(same, 1) + original.recordFlakyObservation(different, 2) + + assertNull(original.getFlakyObservation(1)) + + val observation = original.getFlakyObservation(2) + assertNotNull(observation) + assertEquals("500", observation!!.differences[ResponseField.STATUS_CODE]) + assertEquals("""{"id":"735"}""", observation.differences[ResponseField.BODY]) + + val statusVariation = original.getFlakyVariation(ResponseField.STATUS_CODE) + assertNotNull(statusVariation) + assertEquals(mapOf(2 to "500"), statusVariation!!.valuesByExecIndex) + } + + @Test + fun testStaticFlakinessInferenceIsStoredSeparatelyFromExecutionDeltas(){ + val original = createCallResult("""{"id":"550e8400-e29b-41d4-a716-446655440000"}""") - body.setFlakiness(same) - assertNull(body.getFlakyStatusCode()) - assertNull(body.getFlakyErrorMessage()) + original.recordStaticFlakyInference() - body.setFlakiness(diff) - assertEquals(diffcode, body.getFlakyStatusCode()) - assertEquals(diffmsg, body.getFlakyErrorMessage()) + assertNull(original.getFlakyObservation(1)) + val staticObservation = original.getStaticFlakyObservation() + assertNotNull(staticObservation) + assertEquals(FlakyObservationSource.STATIC_INFERENCE, staticObservation!!.source) + assertNull(staticObservation.execIndex) + assertEquals("""{"id":"$UUID_MARKER"}""", staticObservation.differences[ResponseField.BODY]) } } From 67e0032e25ef4839f4ed99b44fe40bbae46e5278 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 22:18:43 +0800 Subject: [PATCH 10/15] clean HttpWsCallResult --- .../core/problem/httpws/HttpWsCallResult.kt | 140 ++++++++++++------ 1 file changed, 98 insertions(+), 42 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index c367c5573d..eafe7add6d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -39,39 +39,6 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } - enum class ResponseField { - STATUS_CODE, - BODY, - BODY_TYPE, - ERROR_MESSAGE, - TOO_LARGE_BODY, - INFINITE_LOOP, - TIMEDOUT, - TCP_PROBLEM, - INVALID_HTTP, - LOCATION, - ALLOW, - RESPONSE_TIME_MS - } - - enum class FlakyObservationSource { - RE_EXECUTION, - STATIC_INFERENCE - } - - data class FlakyObservation( - val source: FlakyObservationSource, - val execIndex: Int?, - val differences: Map - ) - - data class FieldVariation( - val field: ResponseField, - val valuesByExecIndex: Map - ) - - private val flakyObservations: MutableList = mutableListOf() - /** * In some cases (eg infinite loop redirection), a HTTP call * might fail, and, as such, we might not have an actual "result" @@ -237,7 +204,73 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun setResponseTimeMs(responseTime: Long) = addResultValue(RESPONSE_TIME_MS, responseTime.toString()) fun getResponseTimeMs(): Long? = getResultValue(RESPONSE_TIME_MS)?.toLong() + // -------------------- Flakiness handling -------------------- + /** + * Fields in responses compared across multiple executions of the http call action + * when handling flakiness. + */ + enum class ResponseField { + STATUS_CODE, + BODY, + BODY_TYPE, + ERROR_MESSAGE //, + // do not consider the following fields in flakiness handling +// TOO_LARGE_BODY, +// INFINITE_LOOP, +// TIMEDOUT, +// TCP_PROBLEM, +// INVALID_HTTP, +// LOCATION, +// ALLOW, +// RESPONSE_TIME_MS + } + + /** + * specify how flakiness was observed or identified. + */ + enum class FlakyObservationSource { + /** + * flakiness was observed by re-executing the HTTP action. + */ + RE_EXECUTION, + + /** + * flakiness was inferred through static analysis. + */ + STATIC_INFERENCE + } + + /** + * A set of response fields whose values differ from the original call result. + * + * [execIndex] is index of re-execution to detect flakiness + * and is null when the observation comes from static inference. + * [source] is the source how this flaky was observed + */ + data class FlakyObservation( + val source: FlakyObservationSource, + val execIndex: Int?, + val differences: Map + ) + + /** + * field value differences between the original result and re-execution results. + */ + data class FieldVariation( + val field: ResponseField, + val valuesByExecIndex: Map + ) + + private val flakyObservations: MutableList = mutableListOf() + + /** + * Compare [other] result with [this] original call result + * the response fields whose observed values differ from the [this] original call result. + * + * @param other result from the re-execution to compare against this one + * @param execIndex index identifying the re-execution that produced [other] + */ fun recordFlakyObservation(other: HttpWsCallResult, execIndex: Int) { val differences = responseFieldExtractors() .mapNotNull { spec -> @@ -258,6 +291,10 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } } + /** + * Infer unstable values directly from this result, for example by + * normalizing timestamps or generated identifiers found in response text. + */ fun recordStaticFlakyInference() { val differences = mutableMapOf() @@ -281,17 +318,33 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } } + /** + * Return all recorded flaky observations. + */ fun getFlakyObservations(): List = flakyObservations.toList() + /** + * Return the flaky observation recorded for a specific re-execution. + */ fun getFlakyObservation(execIndex: Int): FlakyObservation? = flakyObservations.find { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex == execIndex } + /** + * Return the flaky observation derived without re-executing the action. + */ fun getStaticFlakyObservation(): FlakyObservation? = flakyObservations.find { it.source == FlakyObservationSource.STATIC_INFERENCE } + /** + * Check whether any flaky observation contains a difference for [field]. + */ fun hasFlakyField(field: ResponseField): Boolean = flakyObservations.any { it.differences.containsKey(field) } + /** + * Return the observed flaky values for [field], ordered by observation + * source and execution index. + */ fun getFlakyValues(field: ResponseField): List = flakyObservations .sortedWith(compareBy { it.source }.thenBy { it.execIndex ?: Int.MAX_VALUE }) @@ -303,6 +356,9 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } } + /** + * Return the values observed for [field] across concrete re-executions. + */ fun getFlakyVariation(field: ResponseField): FieldVariation? { val values = flakyObservations .filter { it.source == FlakyObservationSource.RE_EXECUTION && it.execIndex != null } @@ -364,14 +420,14 @@ abstract class HttpWsCallResult : EnterpriseActionResult { ResponseFieldSpec(ResponseField.STATUS_CODE) { it.getStatusCode()?.toString() }, ResponseFieldSpec(ResponseField.BODY) { it.getBody() }, ResponseFieldSpec(ResponseField.BODY_TYPE) { it.getBodyType()?.toString() }, - ResponseFieldSpec(ResponseField.ERROR_MESSAGE) { it.getErrorMessage() }, - ResponseFieldSpec(ResponseField.TOO_LARGE_BODY) { it.getTooLargeBody().toString() }, - ResponseFieldSpec(ResponseField.INFINITE_LOOP) { it.getInfiniteLoop().toString() }, - ResponseFieldSpec(ResponseField.TIMEDOUT) { it.getTimedout().toString() }, - ResponseFieldSpec(ResponseField.TCP_PROBLEM) { it.getTcpProblem().toString() }, - ResponseFieldSpec(ResponseField.INVALID_HTTP) { it.getInvalidHTTP().toString() }, - ResponseFieldSpec(ResponseField.LOCATION) { it.getLocation() }, - ResponseFieldSpec(ResponseField.ALLOW) { it.getAllow() }, - ResponseFieldSpec(ResponseField.RESPONSE_TIME_MS) { it.getResponseTimeMs()?.toString() } + ResponseFieldSpec(ResponseField.ERROR_MESSAGE) { it.getErrorMessage() }//, +// ResponseFieldSpec(ResponseField.TOO_LARGE_BODY) { it.getTooLargeBody().toString() }, +// ResponseFieldSpec(ResponseField.INFINITE_LOOP) { it.getInfiniteLoop().toString() }, +// ResponseFieldSpec(ResponseField.TIMEDOUT) { it.getTimedout().toString() }, +// ResponseFieldSpec(ResponseField.TCP_PROBLEM) { it.getTcpProblem().toString() }, +// ResponseFieldSpec(ResponseField.INVALID_HTTP) { it.getInvalidHTTP().toString() }, +// ResponseFieldSpec(ResponseField.LOCATION) { it.getLocation() }, +// ResponseFieldSpec(ResponseField.ALLOW) { it.getAllow() }, +// ResponseFieldSpec(ResponseField.RESPONSE_TIME_MS) { it.getResponseTimeMs()?.toString() } ) } From 2472df83c0b77a467d8209aa6db9d4923ed14c42 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 22:28:46 +0800 Subject: [PATCH 11/15] add getMergedFlakyBody --- .../output/service/HttpWsTestCaseWriter.kt | 4 +- .../core/problem/httpws/HttpWsCallResult.kt | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 56221bd711..9bab1d585b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -843,7 +843,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("JsonConvert.DeserializeObject(await $responseVariableName.Content.ReadAsStringAsync());") } - handleJsonStringAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName, res.getTooLargeBody()) + handleJsonStringAssertion(bodyString, res.getFlakyBodies()?.let { res.getMergedFlakyBody() }, lines, bodyVarName, res.getTooLargeBody()) } else if (type.isCompatible(MediaType.TEXT_PLAIN_TYPE)) { @@ -851,7 +851,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") } - handleTextPlainTextAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName) + handleTextPlainTextAssertion(bodyString, res.getFlakyBodies()?.let { res.getMergedFlakyBody() }, lines, bodyVarName) } else { if (format.isCsharp()) { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index eafe7add6d..cd0b3371aa 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -1,5 +1,10 @@ package org.evomaster.core.problem.httpws +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.common.annotations.VisibleForTesting import com.google.gson.Gson import com.google.gson.JsonSyntaxException @@ -18,6 +23,8 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } companion object { + private val mapper = ObjectMapper() + const val STATUS_CODE = "STATUS_CODE" const val BODY = "BODY" const val BODY_TYPE = "BODY_TYPE" @@ -389,6 +396,92 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun getFlakyBodies() : List? = getFlakyValues(ResponseField.BODY).filterNotNull().ifEmpty { null } fun containFlakyBody(flakyBody: String) : Boolean = getFlakyBodies()?.contains(flakyBody) ?: false + /** + * Merge all observed flaky body values into one representative body. + * + * For JSON objects and arrays, the original body is used as the baseline and each + * observed flaky body contributes only the fields or elements that differ from it. + * This allows different flaky fields observed in different executions to be handled + * together when generating assertions. + * + * If there is no flaky body, the original body is returned. If the body is not + * mergeable as JSON, the first observed flaky body is returned. + */ + fun getMergedFlakyBody() : String{ + val originalBody = getBody() + val flakyBodies = getFlakyBodies() + + if (flakyBodies.isNullOrEmpty()) { + return originalBody ?: "" + } + + if (originalBody == null) { + return flakyBodies.first() + } + + return try { + val originalJson = mapper.readTree(originalBody) + if (!originalJson.isObject && !originalJson.isArray) { + return flakyBodies.first() + } + + flakyBodies + .map { mapper.readTree(it) } + .fold(originalJson.deepCopy()) { merged, observed -> + mergeJsonDiffFromOriginal(originalJson, observed, merged) + } + .let { mapper.writeValueAsString(it) } + } catch (e: JsonProcessingException) { + flakyBodies.first() + } catch (e: IllegalStateException) { + flakyBodies.first() + } + } + + private fun mergeJsonDiffFromOriginal( + original: JsonNode, + observed: JsonNode, + merged: JsonNode + ): JsonNode { + if (original == observed) { + return merged + } + + if (original is ObjectNode && observed is ObjectNode && merged is ObjectNode) { + val observedFieldNames = observed.fieldNames().asSequence().toSet() + + original.fieldNames().asSequence() + .filter { !observedFieldNames.contains(it) } + .forEach { merged.remove(it) } + + observed.fields().asSequence().forEach { (field, observedValue) -> + val originalValue = original.get(field) + if (originalValue == null) { + merged.set(field, observedValue.deepCopy()) + } else { + val mergedValue = merged.get(field) ?: originalValue.deepCopy() + merged.set(field, mergeJsonDiffFromOriginal(originalValue, observedValue, mergedValue)) + } + } + + return merged + } + + if (original is ArrayNode && observed is ArrayNode && merged is ArrayNode) { + if (original.size() != observed.size()) { + return observed.deepCopy() + } + + for (i in 0 until observed.size()) { + merged.set(i, mergeJsonDiffFromOriginal(original[i], observed[i], merged[i])) + } + + return merged + } + + return observed.deepCopy() + } + fun setFlakyBodyType(type: MediaType) = addFlakyDifference(ResponseField.BODY_TYPE, type.toString()) fun getFlakyBodyType() : MediaType? = getFirstFlakyValue(ResponseField.BODY_TYPE)?.let { MediaType.valueOf(it) } From 1e6371ea2d767b1467b305d3ac96d932077d2040 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 22:29:31 +0800 Subject: [PATCH 12/15] add unit test --- .../core/problem/rest/RestCallResultTest.kt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt index d7cbd11203..8876f415b8 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt @@ -233,4 +233,40 @@ internal class RestCallResultTest { assertNull(staticObservation.execIndex) assertEquals("""{"id":"$UUID_MARKER"}""", staticObservation.differences[ResponseField.BODY]) } + + @Test + fun testMergedFlakyBodyCombinesDifferentExecutionDeltas(){ + val original = createCallResult("""{"stable":"same","first":"a","second":"b"}""") + val firstDiff = createCallResult("""{"stable":"same","first":"x","second":"b"}""") + val secondDiff = createCallResult("""{"stable":"same","first":"a","second":"y"}""") + + original.recordFlakyObservation(firstDiff, 1) + original.recordFlakyObservation(secondDiff, 2) + + assertEquals("""{"stable":"same","first":"x","second":"y"}""", original.getMergedFlakyBody()) + } + + @Test + fun testMergedFlakyBodyCombinesNestedExecutionDeltas(){ + val original = createCallResult("""{"data":{"stable":"same","first":"a","second":"b"},"items":[{"id":"1"},{"id":"2"}]}""") + val firstDiff = createCallResult("""{"data":{"stable":"same","first":"x","second":"b"},"items":[{"id":"1"},{"id":"2"}]}""") + val secondDiff = createCallResult("""{"data":{"stable":"same","first":"a","second":"b"},"items":[{"id":"1"},{"id":"3"}]}""") + + original.recordFlakyObservation(firstDiff, 1) + original.recordFlakyObservation(secondDiff, 2) + + assertEquals("""{"data":{"stable":"same","first":"x","second":"b"},"items":[{"id":"1"},{"id":"3"}]}""", original.getMergedFlakyBody()) + } + + @Test + fun testMergedFlakyBodyFallsBackToFirstObservedBodyForNonJson(){ + val original = createCallResult("original") + val firstDiff = createCallResult("first") + val secondDiff = createCallResult("second") + + original.recordFlakyObservation(firstDiff, 1) + original.recordFlakyObservation(secondDiff, 2) + + assertEquals("first", original.getMergedFlakyBody()) + } } From d63d3bc4a62f476d2be59a273a362e613b5a27c6 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 22:42:58 +0800 Subject: [PATCH 13/15] add e2e test --- .../v3/flakinessdetect/FlakinessDetectRest.kt | 23 +++++++++- .../FlakinessDetectWithMultiExecEMTest.kt | 44 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectWithMultiExecEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt index c8e00ccdd6..df2fb3484e 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt @@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RestController import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger import kotlin.math.abs import kotlin.math.max import kotlin.math.min @@ -21,6 +22,7 @@ class FlakinessDetectRest { companion object{ val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww") private val START_UP_OBJECT_TAG = arrayOf(javax.validation.constraints.Pattern.Flag.CASE_INSENSITIVE).toString() + private val MULTI_EXECUTION_COUNTER = AtomicInteger(0) } @GetMapping(path = ["/objectFlag"]) fun getEmptyFlag() : ResponseEntity { @@ -54,6 +56,19 @@ class FlakinessDetectRest { return ResponseEntity.ok(FlakinessDetectData(msg, num)) } + @GetMapping(path = ["/multiexecution"]) + fun getMultiExecution() : ResponseEntity { + val index = MULTI_EXECUTION_COUNTER.getAndIncrement() % 3 + + return ResponseEntity.ok( + when (index) { + 0 -> MultiExecutionData("same", "a", "b") + 1 -> MultiExecutionData("same", "x", "b") + else -> MultiExecutionData("same", "a", "y") + } + ) + } + @GetMapping("/price/estimate") fun estimatePrice(@RequestParam base: Int): Map { val randomJitter = randomInt(1000) @@ -103,4 +118,10 @@ data class TimeAgoData( val calculatedPastTime: String ) -class EmptyForFlag(val flag: String) \ No newline at end of file +data class MultiExecutionData( + val stable: String, + val first: String, + val second: String +) + +class EmptyForFlag(val flag: String) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectWithMultiExecEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectWithMultiExecEMTest.kt new file mode 100644 index 0000000000..190a0e7ad4 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectWithMultiExecEMTest.kt @@ -0,0 +1,44 @@ +package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FlakinessDetectWithMultiExecEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FlakinessDetectController()) + } + } + + @Test + fun testRunEMWithMultipleFlakinessExecutions() { + + val outputFolder = "FlakinessDetectMultipleExecutionsEM" + val outputClass = "org.foo.FlakinessDetectMultipleExecutionsEM" + + runTestHandlingFlakyAndCompilation( + outputFolder, + outputClass, + 30, + true + ) { args: MutableList -> + + setOption(args, "minimize", "true") + setOption(args, "handleFlakiness", "true") + setOption(args, "execNumForDetectFlakiness", "2") + setOption(args, "endpointFocus", "/api/flakinessdetect/multiexecution") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.isNotEmpty()) + assertTextInTests(outputFolder, outputClass, "Flaky value of field \"'first'\"") + assertTextInTests(outputFolder, outputClass, "Flaky value of field \"'second'\"") + } + } +} From da0280914e30b114253aa0113d33bed3731f1bdb Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Tue, 23 Jun 2026 22:45:10 +0800 Subject: [PATCH 14/15] add execNumForDetectFlakiness 1 --- .../spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt | 3 ++- .../v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt | 3 ++- .../spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt index d82303ca66..efa5551010 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt @@ -37,6 +37,7 @@ class FlakinessDetectBBEMTest : SpringTestBase() { ){ args: MutableList -> setOption(args, "handleFlakiness", "true") + setOption(args, "execNumForDetectFlakiness", "1") val solution = initAndRun(args) @@ -44,4 +45,4 @@ class FlakinessDetectBBEMTest : SpringTestBase() { } } -} \ No newline at end of file +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt index 5a78d64422..4780f80aa6 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt @@ -36,6 +36,7 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() { setOption(args, "handleFlakiness", "true") + setOption(args, "execNumForDetectFlakiness", "1") // we may still need to specify info in non bb-e2etest setOption(args, "blackBox", "true") @@ -52,4 +53,4 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() { } } -} \ No newline at end of file +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt index 50c13bab10..a5ad241e9e 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt @@ -37,6 +37,7 @@ class FlakinessDetectEMTest : SpringTestBase() { setOption(args, "minimize", "true") setOption(args, "handleFlakiness", "true") + setOption(args, "execNumForDetectFlakiness", "1") val solution = initAndRun(args) From 346f5c3c65359d9df96be039ff86539daa9756d7 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Mon, 29 Jun 2026 18:44:38 +0800 Subject: [PATCH 15/15] address the comments --- .../org/evomaster/core/problem/httpws/HttpWsCallResult.kt | 7 ++++++- .../org/evomaster/core/search/service/FlakinessDetector.kt | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index cd0b3371aa..af710dce6c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -46,6 +46,12 @@ abstract class HttpWsCallResult : EnterpriseActionResult { } + /** + * this is used to record observed flakiness during flakiness handling + */ + private val flakyObservations: MutableList = mutableListOf() + + /** * In some cases (eg infinite loop redirection), a HTTP call * might fail, and, as such, we might not have an actual "result" @@ -269,7 +275,6 @@ abstract class HttpWsCallResult : EnterpriseActionResult { val valuesByExecIndex: Map ) - private val flakyObservations: MutableList = mutableListOf() /** * Compare [other] result with [this] original call result diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt index 55446a4908..fea0dd66ac 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt @@ -36,6 +36,10 @@ class FlakinessDetector : TimeBoxedPhase { private lateinit var epc: ExecutionPhaseController override fun applyPhase() { + if (!config.handleFlakiness) { + throw IllegalStateException("handleFlakiness must be enabled before applying this phase of flakiness detection and handing with FlakinessDetector") + } + if (config.enableStaticFlakyInference) { inferStaticFlakiness() }