From 89ec54a043038c733bcefe9ca9827d89d161ea88 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Tue, 23 Jun 2026 22:32:15 +0300 Subject: [PATCH 1/4] timeout --- .../timeout/HttpTimeoutApplication.kt | 45 ++++++++++++++++ .../timeout/HttpTimeoutController.kt | 12 +++++ .../httporacle/timeout/HttpTimeoutEMTest.kt | 52 +++++++++++++++++++ .../enterprise/ExperimentalFaultCategory.kt | 2 + .../service/fitness/AbstractRestFitness.kt | 26 ++++++++++ 5 files changed, 137 insertions(+) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt new file mode 100644 index 0000000000..f448b4c246 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt @@ -0,0 +1,45 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.timeout + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/timeout"]) +@RestController +open class HttpTimeoutApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(HttpTimeoutApplication::class.java, *args) + } + + fun reset() {} + } + + @GetMapping(path = ["/slow/{id}"]) + open fun slow(@PathVariable("id") id: Int): ResponseEntity { + val deadline = System.currentTimeMillis() + 10_000 + while (System.currentTimeMillis() < deadline) { + try { + Thread.sleep(deadline - System.currentTimeMillis()) + } catch (e: InterruptedException) { + // ignore and keep blocking + } + } + return ResponseEntity.status(200).body("$id") + } + + // clean + @GetMapping(path = ["/fast/{id}"]) + open fun fast(@PathVariable("id") id: Int): ResponseEntity { + return ResponseEntity.status(200).body("$id") + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt new file mode 100644 index 0000000000..fbf2962116 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt @@ -0,0 +1,12 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.timeout + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.timeout.HttpTimeoutApplication + + +class HttpTimeoutController : SpringController(HttpTimeoutApplication::class.java) { + + override fun resetStateOfSUT() { + HttpTimeoutApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt new file mode 100644 index 0000000000..6ce19f91a4 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt @@ -0,0 +1,52 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.timeout + +import com.foo.rest.examples.spring.openapi.v3.httporacle.timeout.HttpTimeoutController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class HttpTimeoutEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(HttpTimeoutController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "HttpTimeoutEM", + 5 + ) { args: MutableList -> + + setOption(args, "blackBox", "true") + setOption(args, "bbTargetUrl", baseUrlOfSut) + setOption(args, "bbSwaggerUrl", "$baseUrlOfSut/v3/api-docs") + setOption(args, "useExperimentalOracles", "true") + setOption(args, "tcpTimeoutMs", "2000") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertTrue(ExperimentalFaultCategory.HTTP_TIMEOUT in faults) + + val timeoutFaults = DetectedFaultUtils.getDetectedFaults(solution) + .filter { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + + // fault on the slow path + assertTrue(timeoutFaults.any { it.operationId.contains("/api/timeout/slow/") }) + // no false positive on the fast path + assertTrue(timeoutFaults.none { it.operationId.contains("/api/timeout/fast/") }) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 08094a7f42..86b3d6dff8 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -39,6 +39,8 @@ enum class ExperimentalFaultCategory( "TODO"), HTTP_INVALID_ALLOW(919, "Invalid allow", "invalidAllow", "TODO"), + HTTP_TIMEOUT(921, "Timeout", "timeout", + "TODO"), HTTP_STATUS_NO_NON_STANDARD_CODES(950, "no-non-standard-codes", "invalidStatusCode", "TODO"), HTTP_STATUS_NO_201_IF_DELETE(951, "no-201-if-delete", "201OnDelete", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index a8f8c491b6..84ee948587 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1322,6 +1322,32 @@ abstract class AbstractRestFitness : HttpWsFitness() { analyzeHttpSemantics(individual, actionResults, fv) } + if(config.blackBox && config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_TIMEOUT)){ + handleTimeout(individual, actionResults, fv) + } + } + + /** + * A timeout is treated as a fault: the SUT should rather answer quickly (eg 202 with a + * Location header for long computations) instead of hanging until the client gives up. + */ + private fun handleTimeout( + individual: RestIndividual, + actionResults: List, + fv: FitnessValue + ) { + val actions = individual.seeMainExecutableActions() + + for (index in actions.indices) { + val a = actions[index] + val r = actionResults.find { it.sourceLocalId == a.getLocalId() } as RestCallResult? ?: continue + if (!r.getTimedout()) continue + + val category = ExperimentalFaultCategory.HTTP_TIMEOUT + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, a.getName())) + fv.updateTarget(scenarioId, 1.0, index) + r.addFault(DetectedFault(category, a.getName(), null)) + } } private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List, fv: FitnessValue) { From e7cabb86dc5a49ed89d34e66126f8651a74eb32a Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 30 Jun 2026 13:52:22 +0300 Subject: [PATCH 2/4] setting timeout in tests --- .../output/service/HttpWsTestCaseWriter.kt | 2 ++ .../core/output/service/RestTestCaseWriter.kt | 7 ++++--- .../core/output/service/TestSuiteWriter.kt | 20 ++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 56221bd711..f571f253bb 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -510,6 +510,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.indent(2) //in SuperAgent, verb must be first handleVerbEndpoint(baseUrlOfSut, call, lines) + //client timeout, same source as fuzzing tcpTimeoutMs + lines.add(".timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS})") lines.append(getAcceptHeader(call, res)) handleHeaders(call, lines) handleBody(call, lines) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 9ae514e51d..e110421ec1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -303,11 +303,12 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { if (bodyParam != null) { lines.append(", data=body") } - if(config.testTimeout > 0) { + if(config.tcpTimeoutMs > 0) { /* - As timeout at test level does not work reliably in Python, we do timeout as well in each HTTP call. + Client timeout per HTTP call, same source as fuzzing tcpTimeoutMs. + Also, timeout at test level does not work reliably in Python. */ - lines.append(", timeout=${config.testTimeout}") + lines.append(", timeout=EM_HTTP_TIMEOUT") } } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 705f162571..50d52d1a98 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -479,6 +479,7 @@ class TestSuiteWriter { addImport("io.restassured.RestAssured", lines) addImport("io.restassured.RestAssured.given", lines, true) addImport("io.restassured.response.ValidatableResponse", lines) + addImport("io.restassured.config.HttpClientConfig", lines) } if ((config.isEnabledExternalServiceMocking() && solution.needWireMockServers()) @@ -545,6 +546,8 @@ class TestSuiteWriter { if (format.isJavaScript()) { lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';") lines.add("const superagent = require(\"superagent\");") + // HTTP client timeout (ms) + lines.add("const EM_HTTP_TIMEOUT_MS = ${config.tcpTimeoutMs};") val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -599,6 +602,8 @@ class TestSuiteWriter { } } lines.add("from $pythonUtilsFilenameNoExtension import *") + // HTTP client timeout (seconds) + lines.add("EM_HTTP_TIMEOUT = ${config.tcpTimeoutMs / 1000.0}") val pythonUtils = PyLoader::class.java.getResource("/$pythonUtilsFilename").readText() saveToDisk(pythonUtils, Paths.get(config.outputFolder, pythonUtilsFilename)) } @@ -852,11 +857,20 @@ class TestSuiteWriter { addStatement("RestAssured.urlEncodingEnabled = false", lines) } - if (config.enableBasicAssertions && format.isJavaOrKotlin()) { + if (format.isJavaOrKotlin()) { + // global HTTP client config. The socket timeout MUST match the one used during + // fuzzing (tcpTimeoutMs), so that timeout faults are reproduced consistently lines.add("RestAssured.config = RestAssured.config()") lines.indented { - lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))") - lines.add(".redirect(redirectConfig().followRedirects(false))") + if (config.enableBasicAssertions) { + lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))") + lines.add(".redirect(redirectConfig().followRedirects(false))") + } + lines.add(".httpClient(HttpClientConfig.httpClientConfig()") + lines.indented { + lines.add(".setParam(\"http.socket.timeout\", ${config.tcpTimeoutMs})") + lines.add(".setParam(\"http.connection.timeout\", ${config.tcpTimeoutMs}))") + } } lines.appendSemicolon() } From 6b6353f6ebbf9cd3b82842144853828f876ac7ab Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 30 Jun 2026 15:33:50 +0300 Subject: [PATCH 3/4] http timeout write test case --- .../httptimeout/BBHttpTimeoutApplication.kt} | 12 +++--- .../bb/httptimeout/BBHttpTimeoutController.kt | 5 +++ .../bb/httptimeout/BBHttpTimeoutEMTest.kt} | 41 +++++++++++-------- .../timeout/HttpTimeoutController.kt | 12 ------ .../output/service/HttpWsTestCaseWriter.kt | 36 +++++++++++++++- .../core/output/service/TestCaseWriter.kt | 29 +++++++++---- 6 files changed, 90 insertions(+), 45 deletions(-) rename core-tests/e2e-tests/spring/{spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt => spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt} (76%) create mode 100644 core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt rename core-tests/e2e-tests/spring/{spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt => spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt} (56%) delete mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt similarity index 76% rename from core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt rename to core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt index f448b4c246..5db9b9e307 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutApplication.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt @@ -1,5 +1,6 @@ -package com.foo.rest.examples.spring.openapi.v3.httporacle.timeout +package com.foo.rest.examples.bb.httptimeout +import org.evomaster.e2etests.utils.CoveredTargets import org.springframework.boot.SpringApplication import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration @@ -13,19 +14,20 @@ import org.springframework.web.bind.annotation.RestController @SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) @RequestMapping(path = ["/api/timeout"]) @RestController -open class HttpTimeoutApplication { +open class BBHttpTimeoutApplication { companion object { @JvmStatic fun main(args: Array) { - SpringApplication.run(HttpTimeoutApplication::class.java, *args) + SpringApplication.run(BBHttpTimeoutApplication::class.java, *args) } - - fun reset() {} } + // slow endpoint: blocks longer than the client timeout, triggering a HTTP_TIMEOUT fault. + // the target is covered as soon as the request is handled, before the client gives up. @GetMapping(path = ["/slow/{id}"]) open fun slow(@PathVariable("id") id: Int): ResponseEntity { + CoveredTargets.cover("timeout") val deadline = System.currentTimeMillis() + 10_000 while (System.currentTimeMillis() < deadline) { try { diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt new file mode 100644 index 0000000000..f009c98207 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.bb.httptimeout + +import com.foo.rest.examples.bb.SpringController + +class BBHttpTimeoutController : SpringController(BBHttpTimeoutApplication::class.java) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt similarity index 56% rename from core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt rename to core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt index 6ce19f91a4..ccde279536 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/timeout/HttpTimeoutEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt @@ -1,35 +1,43 @@ -package org.evomaster.e2etests.spring.openapi.v3.httporacle.timeout +package org.evomaster.e2etests.spring.rest.bb.httptimeout -import com.foo.rest.examples.spring.openapi.v3.httporacle.timeout.HttpTimeoutController +import com.foo.rest.examples.bb.httptimeout.BBHttpTimeoutController +import org.evomaster.core.output.OutputFormat import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory -import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.evomaster.e2etests.spring.rest.bb.SpringTestBase +import org.evomaster.e2etests.utils.EnterpriseTestBase import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource -class HttpTimeoutEMTest : SpringTestBase() { +class BBHttpTimeoutEMTest : SpringTestBase() { companion object { + + init { + EnterpriseTestBase.shouldApplyInstrumentation = false + } + @BeforeAll @JvmStatic fun init() { - initClass(HttpTimeoutController()) + initClass(BBHttpTimeoutController()) } } + @ParameterizedTest + @EnumSource + fun testBlackBoxOutput(outputFormat: OutputFormat) { - @Test - fun testRunEM() { - - runTestHandlingFlakyAndCompilation( - "HttpTimeoutEM", - 5 + executeAndEvaluateBBTest( + outputFormat, + "bbhttptimeout", + 5, + 6, + "timeout" ) { args: MutableList -> - setOption(args, "blackBox", "true") - setOption(args, "bbTargetUrl", baseUrlOfSut) - setOption(args, "bbSwaggerUrl", "$baseUrlOfSut/v3/api-docs") setOption(args, "useExperimentalOracles", "true") setOption(args, "tcpTimeoutMs", "2000") @@ -37,9 +45,6 @@ class HttpTimeoutEMTest : SpringTestBase() { assertTrue(solution.individuals.size >= 1) - val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) - assertTrue(ExperimentalFaultCategory.HTTP_TIMEOUT in faults) - val timeoutFaults = DetectedFaultUtils.getDetectedFaults(solution) .filter { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt deleted file mode 100644 index fbf2962116..0000000000 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/timeout/HttpTimeoutController.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.foo.rest.examples.spring.openapi.v3.httporacle.timeout - -import com.foo.rest.examples.spring.openapi.v3.SpringController -import com.foo.rest.examples.spring.openapi.v3.httporacle.timeout.HttpTimeoutApplication - - -class HttpTimeoutController : SpringController(HttpTimeoutApplication::class.java) { - - override fun resetStateOfSUT() { - HttpTimeoutApplication.reset() - } -} diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index f571f253bb..a07426a996 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -13,6 +13,8 @@ import org.evomaster.core.output.dto.DtoCall import org.evomaster.core.output.dto.GeneToDto import org.evomaster.core.output.formatter.OutputFormatter import org.evomaster.core.problem.enterprise.EnterpriseActionGroup +import org.evomaster.core.problem.enterprise.EnterpriseActionResult +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.HttpWsCallResult @@ -57,6 +59,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return !(result as HttpWsCallResult).getTimedout() } + /** + * For a HTTP_TIMEOUT fault, the call is expected to fail, and the assertion is expressed + * directly on the call: + * - JS: await expect(...).rejects.toThrow() + * - Python: with self.assertRaises(Exception): ... + */ + protected fun hasTimeoutFault(res: ActionResult): Boolean { + return res is EnterpriseActionResult + && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + } + + protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && hasTimeoutFault(res) + + protected fun expectsAssertRaises(res: ActionResult) = format.isPython() && hasTimeoutFault(res) + fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") @@ -111,7 +128,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + // for a call expected to reject, wrap it in await expect(...).rejects.toThrow() + format.isJavaScript() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -392,7 +410,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { timeStartName = handleExecutionTimePrologue(lines); } - if (res.invalidCall()) { + if (res.invalidCall() && !expectsRejection(res) && !expectsAssertRaises(res)) { addActionInTryCatch(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) } else { addActionLines(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) @@ -496,6 +514,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { dtoVar = writeDto(call, lines) } + val pyAssertRaises = expectsAssertRaises(res) + if (pyAssertRaises) { + lines.add("with self.assertRaises(Exception):") + lines.indent() + } + handleFirstLine(call, lines, res, responseVariableName) when { @@ -533,6 +557,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { handleResponseDirectlyInTheCall(call, res, lines) } handleLastLine(call, res, lines, responseVariableName) + + if (pyAssertRaises) { + lines.deindent() + } return responseVariableName } @@ -884,6 +912,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { so, here we make it passes as long as a status was present */ lines.add(".ok(res => res.status)") + if (expectsRejection(res)) { + //close the await expect(...) and assert the call rejected + lines.append(").rejects.toThrow()") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7f286c394a..fbfbaac649 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -9,6 +9,7 @@ import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.output.TestWriterUtils.getWireMockVariableName import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.enterprise.EnterpriseIndividual +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.externalservice.HostnameResolutionAction import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.externalservice.httpws.param.HttpWsResponseParam @@ -44,6 +45,10 @@ abstract class TestCaseWriter { companion object { private val log = LoggerFactory.getLogger(TestCaseWriter::class.java) + + // message for the assertion that flags a missing expected timeout (Java/Kotlin/C#) + // JS uses await expect(...).rejects.toThrow() and Python uses with self.assertRaises(...) + private const val EXPECTED_TIMEOUT_MSG = "Expected a timeout" } @@ -320,10 +325,19 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("try:") } + // a HTTP_TIMEOUT fault means the call is expected to time out (client timeout == fuzzing + // tcpTimeoutMs). if no timeout exception is thrown, the fault did not reproduce -> fail + val timeoutFault = res is EnterpriseActionResult + && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + lines.indented { addActionLines(call,index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) - if (shouldFailIfExceptionNotThrown(res)) { + if (timeoutFault) { + // only Java/Kotlin/C# reach here; JS uses expect(...).rejects.toThrow() and + // Python uses with self.assertRaises(...), neither wrapped in this try/catch + lines.add("fail(\"$EXPECTED_TIMEOUT_MSG\");") + } else if (shouldFailIfExceptionNotThrown(res)) { if (!format.isJavaScript()) { /* TODO need a way to do it for JS, see @@ -372,17 +386,16 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } - res.getErrorMessage()?.let { - lines.indented { + lines.indented { + res.getErrorMessage()?.let { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") } - } - - if (format.isPython()) { - lines.indented { + if (format.isPython()) { lines.add("pass") } - } else { + } + + if (!format.isPython()) { lines.add("}") } } From 6c23581dbefd4ba6b51e15d08a83e2c27b0b08b2 Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 30 Jun 2026 17:00:58 +0300 Subject: [PATCH 4/4] add timeout param --- .../evomaster/core/output/PythonTestCaseWriterTest.kt | 8 ++++---- .../org/evomaster/core/output/TestCaseWriterTest.kt | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt index 8ac07a22bb..d7a390da03 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt @@ -96,7 +96,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ indent() add(".get(self.baseUrlOfSut + \"/\",") indent() - add("headers=headers, verify=False)") + add("headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False)") deindent() deindent() deindent() @@ -167,7 +167,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] @@ -241,7 +241,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] @@ -300,7 +300,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index d2454da456..588cad0fe1 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -1331,7 +1331,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200); @@ -1404,7 +1405,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200); @@ -1462,7 +1464,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200);