diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt new file mode 100644 index 0000000000..5db9b9e307 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt @@ -0,0 +1,47 @@ +package com.foo.rest.examples.bb.httptimeout + +import org.evomaster.e2etests.utils.CoveredTargets +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/timeout"]) +@RestController +open class BBHttpTimeoutApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(BBHttpTimeoutApplication::class.java, *args) + } + } + + // slow endpoint: blocks longer than the client timeout, triggering a HTTP_TIMEOUT fault. + // the target is covered as soon as the request is handled, before the client gives up. + @GetMapping(path = ["/slow/{id}"]) + open fun slow(@PathVariable("id") id: Int): ResponseEntity { + CoveredTargets.cover("timeout") + val deadline = System.currentTimeMillis() + 10_000 + while (System.currentTimeMillis() < deadline) { + try { + Thread.sleep(deadline - System.currentTimeMillis()) + } catch (e: InterruptedException) { + // ignore and keep blocking + } + } + return ResponseEntity.status(200).body("$id") + } + + // clean + @GetMapping(path = ["/fast/{id}"]) + open fun fast(@PathVariable("id") id: Int): ResponseEntity { + return ResponseEntity.status(200).body("$id") + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt new file mode 100644 index 0000000000..f009c98207 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.bb.httptimeout + +import com.foo.rest.examples.bb.SpringController + +class BBHttpTimeoutController : SpringController(BBHttpTimeoutApplication::class.java) diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt new file mode 100644 index 0000000000..ccde279536 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt @@ -0,0 +1,57 @@ +package org.evomaster.e2etests.spring.rest.bb.httptimeout + +import com.foo.rest.examples.bb.httptimeout.BBHttpTimeoutController +import org.evomaster.core.output.OutputFormat +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.rest.bb.SpringTestBase +import org.evomaster.e2etests.utils.EnterpriseTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource + +class BBHttpTimeoutEMTest : SpringTestBase() { + + companion object { + + init { + EnterpriseTestBase.shouldApplyInstrumentation = false + } + + @BeforeAll + @JvmStatic + fun init() { + initClass(BBHttpTimeoutController()) + } + } + + @ParameterizedTest + @EnumSource + fun testBlackBoxOutput(outputFormat: OutputFormat) { + + executeAndEvaluateBBTest( + outputFormat, + "bbhttptimeout", + 5, + 6, + "timeout" + ) { args: MutableList -> + + setOption(args, "useExperimentalOracles", "true") + setOption(args, "tcpTimeoutMs", "2000") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val timeoutFaults = DetectedFaultUtils.getDetectedFaults(solution) + .filter { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + + // fault on the slow path + assertTrue(timeoutFaults.any { it.operationId.contains("/api/timeout/slow/") }) + // no false positive on the fast path + assertTrue(timeoutFaults.none { it.operationId.contains("/api/timeout/fast/") }) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 79a1bd6ce7..6a13cce490 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -13,6 +13,8 @@ import org.evomaster.core.output.dto.DtoCall import org.evomaster.core.output.dto.GeneToDto import org.evomaster.core.output.formatter.OutputFormatter import org.evomaster.core.problem.enterprise.EnterpriseActionGroup +import org.evomaster.core.problem.enterprise.EnterpriseActionResult +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.HttpWsCallResult @@ -60,6 +62,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return !(result as HttpWsCallResult).getTimedout() } + /** + * For a HTTP_TIMEOUT fault, the call is expected to fail, and the assertion is expressed + * directly on the call: + * - JS: await expect(...).rejects.toThrow() + * - Python: with self.assertRaises(Exception): ... + */ + protected fun hasTimeoutFault(res: ActionResult): Boolean { + return res is EnterpriseActionResult + && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + } + + protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && hasTimeoutFault(res) + + protected fun expectsAssertRaises(res: ActionResult) = format.isPython() && hasTimeoutFault(res) + fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") @@ -114,7 +131,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + // for a call expected to reject, wrap it in await expect(...).rejects.toThrow() + format.isJavaScript() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -398,7 +416,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { timeStartName = handleExecutionTimePrologue(lines); } - if (res.invalidCall()) { + if (res.invalidCall() && !expectsRejection(res) && !expectsAssertRaises(res)) { addActionInTryCatch(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) } else { addActionLines(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) @@ -502,6 +520,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { dtoVar = writeDto(call, lines) } + val pyAssertRaises = expectsAssertRaises(res) + if (pyAssertRaises) { + lines.add("with self.assertRaises(Exception):") + lines.indent() + } + handleFirstLine(call, lines, res, responseVariableName) when { @@ -516,6 +540,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.indent(2) //in SuperAgent, verb must be first handleVerbEndpoint(baseUrlOfSut, call, lines) + //client timeout, same source as fuzzing tcpTimeoutMs + lines.add(".timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS})") lines.append(getAcceptHeader(call, res)) handleHeaders(call, lines) handleBody(call, lines) @@ -537,6 +563,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { handleResponseDirectlyInTheCall(call, res, lines) } handleLastLine(call, res, lines, responseVariableName) + + if (pyAssertRaises) { + lines.deindent() + } return responseVariableName } @@ -900,6 +930,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { so, here we make it passes as long as a status was present */ lines.add(".ok(res => res.status)") + if (expectsRejection(res)) { + //close the await expect(...) and assert the call rejected + lines.append(").rejects.toThrow()") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 9ae514e51d..e110421ec1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -303,11 +303,12 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { if (bodyParam != null) { lines.append(", data=body") } - if(config.testTimeout > 0) { + if(config.tcpTimeoutMs > 0) { /* - As timeout at test level does not work reliably in Python, we do timeout as well in each HTTP call. + Client timeout per HTTP call, same source as fuzzing tcpTimeoutMs. + Also, timeout at test level does not work reliably in Python. */ - lines.append(", timeout=${config.testTimeout}") + lines.append(", timeout=EM_HTTP_TIMEOUT") } } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7f286c394a..fbfbaac649 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -9,6 +9,7 @@ import org.evomaster.core.output.TestWriterUtils import org.evomaster.core.output.TestWriterUtils.getWireMockVariableName import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.enterprise.EnterpriseIndividual +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.externalservice.HostnameResolutionAction import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.externalservice.httpws.param.HttpWsResponseParam @@ -44,6 +45,10 @@ abstract class TestCaseWriter { companion object { private val log = LoggerFactory.getLogger(TestCaseWriter::class.java) + + // message for the assertion that flags a missing expected timeout (Java/Kotlin/C#) + // JS uses await expect(...).rejects.toThrow() and Python uses with self.assertRaises(...) + private const val EXPECTED_TIMEOUT_MSG = "Expected a timeout" } @@ -320,10 +325,19 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("try:") } + // a HTTP_TIMEOUT fault means the call is expected to time out (client timeout == fuzzing + // tcpTimeoutMs). if no timeout exception is thrown, the fault did not reproduce -> fail + val timeoutFault = res is EnterpriseActionResult + && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } + lines.indented { addActionLines(call,index, testCaseName, lines, res, testSuitePath, baseUrlOfSut) - if (shouldFailIfExceptionNotThrown(res)) { + if (timeoutFault) { + // only Java/Kotlin/C# reach here; JS uses expect(...).rejects.toThrow() and + // Python uses with self.assertRaises(...), neither wrapped in this try/catch + lines.add("fail(\"$EXPECTED_TIMEOUT_MSG\");") + } else if (shouldFailIfExceptionNotThrown(res)) { if (!format.isJavaScript()) { /* TODO need a way to do it for JS, see @@ -372,17 +386,16 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } - res.getErrorMessage()?.let { - lines.indented { + lines.indented { + res.getErrorMessage()?.let { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") } - } - - if (format.isPython()) { - lines.indented { + if (format.isPython()) { lines.add("pass") } - } else { + } + + if (!format.isPython()) { lines.add("}") } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 38d791c9d8..6b73975905 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -465,6 +465,7 @@ class TestSuiteWriter { addImport("io.restassured.RestAssured", lines) addImport("io.restassured.RestAssured.given", lines, true) addImport("io.restassured.response.ValidatableResponse", lines) + addImport("io.restassured.config.HttpClientConfig", lines) } if ((config.isEnabledExternalServiceMocking() && solution.needWireMockServers()) @@ -532,6 +533,8 @@ class TestSuiteWriter { if (format.isJavaScript()) { lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';") lines.add("const superagent = require(\"superagent\");") + // HTTP client timeout (ms) + lines.add("const EM_HTTP_TIMEOUT_MS = ${config.tcpTimeoutMs};") val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -586,6 +589,8 @@ class TestSuiteWriter { } } lines.add("from $pythonUtilsFilenameNoExtension import *") + // HTTP client timeout (seconds) + lines.add("EM_HTTP_TIMEOUT = ${config.tcpTimeoutMs / 1000.0}") val pythonUtils = PyLoader::class.java.getResource("/$pythonUtilsFilename").readText() saveToDisk(pythonUtils, Paths.get(config.outputFolder, pythonUtilsFilename)) } @@ -839,9 +844,20 @@ class TestSuiteWriter { addStatement("RestAssured.urlEncodingEnabled = false", lines) } - if (config.enableBasicAssertions && format.isJavaOrKotlin()) { + if (format.isJavaOrKotlin()) { + // global HTTP client config. The socket timeout MUST match the one used during + // fuzzing (tcpTimeoutMs), so that timeout faults are reproduced consistently lines.add("RestAssured.config = RestAssured.config()") lines.indented { + if (config.enableBasicAssertions) { + lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))") + lines.add(".redirect(redirectConfig().followRedirects(false))") + } + lines.add(".httpClient(HttpClientConfig.httpClientConfig()") + lines.indented { + lines.add(".setParam(\"http.socket.timeout\", ${config.tcpTimeoutMs})") + lines.add(".setParam(\"http.connection.timeout\", ${config.tcpTimeoutMs}))") + } lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))") lines.add(".redirect(redirectConfig().followRedirects(false))") lines.add(".encoderConfig(EncoderConfig.encoderConfig().encodeContentTypeAs(\"application/octet-stream\", ContentType.TEXT))") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 4ed49a4bef..90a7e6b96c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -39,6 +39,8 @@ enum class ExperimentalFaultCategory( "TODO"), HTTP_INVALID_ALLOW(919, "Invalid allow", "invalidAllow", "TODO"), + HTTP_TIMEOUT(921, "Timeout", "timeout", + "TODO"), HTTP_STATUS_NO_NON_STANDARD_CODES(950, "no-non-standard-codes", "invalidStatusCode", "TODO"), HTTP_STATUS_NO_201_IF_DELETE(951, "no-201-if-delete", "201OnDelete", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index ae5a63def9..514e88a664 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1312,6 +1312,32 @@ abstract class AbstractRestFitness : HttpWsFitness() { analyzeHttpSemantics(individual, actionResults, fv) } + if(config.blackBox && config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_TIMEOUT)){ + handleTimeout(individual, actionResults, fv) + } + } + + /** + * A timeout is treated as a fault: the SUT should rather answer quickly (eg 202 with a + * Location header for long computations) instead of hanging until the client gives up. + */ + private fun handleTimeout( + individual: RestIndividual, + actionResults: List, + fv: FitnessValue + ) { + val actions = individual.seeMainExecutableActions() + + for (index in actions.indices) { + val a = actions[index] + val r = actionResults.find { it.sourceLocalId == a.getLocalId() } as RestCallResult? ?: continue + if (!r.getTimedout()) continue + + val category = ExperimentalFaultCategory.HTTP_TIMEOUT + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, a.getName())) + fv.updateTarget(scenarioId, 1.0, index) + r.addFault(DetectedFault(category, a.getName(), null)) + } } private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List, fv: FitnessValue) { diff --git a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt index 8ac07a22bb..d7a390da03 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt @@ -96,7 +96,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ indent() add(".get(self.baseUrlOfSut + \"/\",") indent() - add("headers=headers, verify=False)") + add("headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False)") deindent() deindent() deindent() @@ -167,7 +167,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] @@ -241,7 +241,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] @@ -300,7 +300,7 @@ class PythonTestCaseWriterTest : WriterTestBase(){ headers['Accept'] = "*/*" res_0 = requests \ .get(self.baseUrlOfSut + "/foo", - headers=headers, verify=False) + headers=headers, timeout=EM_HTTP_TIMEOUT, verify=False) assert res_0.status_code == 200 assert "application/json" in res_0.headers["content-type"] diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index 96245239c5..6c4bf6fd15 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -1397,7 +1397,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200); @@ -1470,7 +1471,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200); @@ -1528,7 +1530,8 @@ public void test() throws Exception { test("test", async () => { const res_0 = await superagent - .get(baseUrlOfSut + "/foo").set('Accept', "*/*") + .get(baseUrlOfSut + "/foo") + .timeout({response: EM_HTTP_TIMEOUT_MS, deadline: EM_HTTP_TIMEOUT_MS}).set('Accept', "*/*") .ok(res => res.status); expect(res_0.status).toBe(200);