handling shitshow of content-type on empty bodies

client-java/controller/pom.xml

@@ -323,8 +323,8 @@
                                     <shadedPattern>shaded.javassist</shadedPattern>
                                 </relocation>
                                 <relocation>
-                                    <pattern>jersey</pattern>
-                                    <shadedPattern>shaded.jersey</shadedPattern>
+                                    <pattern>jersey.repackaged</pattern>
+                                    <shadedPattern>shaded.jersey.repackaged</shadedPattern>
                                 </relocation>
                                 <relocation>
                                     <pattern>org.aopalliance</pattern>

core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/emptybody/BBEmptyBodyApplication.kt

@@ -0,0 +1,73 @@
+package com.foo.rest.examples.bb.emptybody
+
+import org.evomaster.e2etests.utils.CoveredTargets
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+import org.springframework.http.HttpHeaders
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.*
+
+
+@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
+@RequestMapping(path = ["/api/bbemptybody"])
+@RestController
+open class BBEmptyBodyApplication {
+
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) {
+            SpringApplication.run(BBEmptyBodyApplication::class.java, *args)
+        }
+    }
+
+    private fun verifyHeader(headers: HttpHeaders) {
+
+        /*
+            tricky...
+            for sure we don't want content-type on empty request, but, in theory,
+            we might want to have
+            content-length: 0
+            to handle some frameworks that might have issues with it.
+            but didn't manage to get Jersey to handle it :(
+            also, it seems like even if we can control Jersey, the HTTP libraries in the
+            generated tests might have their own different "opinions" of what to send...
+            what the fucking mess...
+            TODO if we find fix, would need to update here
+         */
+
+        if (!headers[HttpHeaders.CONTENT_LENGTH].isNullOrEmpty()) {
+
+            headers[HttpHeaders.CONTENT_LENGTH]!!.forEach {
+                val size = it.substringAfterLast(':').toInt()
+                if(size != 0){
+                    throw IllegalArgumentException("Content-Length must be null or 0 value: $headers")
+                }
+            }
+        }
+        if (!headers[HttpHeaders.CONTENT_TYPE].isNullOrEmpty()) {
+            throw IllegalArgumentException("Content-Length must be null: $headers")
+        }
+    }
+
+    @PatchMapping(path = ["/patch"])
+    fun patch(@RequestHeader headers: HttpHeaders) : ResponseEntity<String> {
+        verifyHeader(headers)
+        CoveredTargets.cover("PATCH")
+        return ResponseEntity.ok("OK")
+    }
+
+    @PutMapping(path = ["/put"])
+    fun put(@RequestHeader headers: HttpHeaders) : ResponseEntity<String> {
+        verifyHeader(headers)
+        CoveredTargets.cover("PUT")
+        return ResponseEntity.ok("OK")
+    }
+
+    @PostMapping(path = ["/post"])
+    fun post(@RequestHeader headers: HttpHeaders) : ResponseEntity<String> {
+        verifyHeader(headers)
+        CoveredTargets.cover("POST")
+        return ResponseEntity.ok("OK")
+    }
+}

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/emptybody/BBEmptyBodyController.kt

@@ -0,0 +1,5 @@
+package com.foo.rest.examples.bb.emptybody
+
+import com.foo.rest.examples.bb.SpringController
+
+class BBEmptyBodyController : SpringController(BBEmptyBodyApplication::class.java)

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/emptybody/BBEmptyBodyEMTest.kt

@@ -0,0 +1,57 @@
+package org.evomaster.e2etests.spring.rest.bb.emptybody
+
+import com.foo.rest.examples.bb.emptybody.BBEmptyBodyController
+import org.evomaster.core.output.OutputFormat
+import org.evomaster.core.problem.rest.data.HttpVerb
+import org.evomaster.e2etests.spring.rest.bb.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.EnumSource
+
+class BBEmptyBodyEMTest : SpringTestBase() {
+
+    companion object {
+        init {
+            shouldApplyInstrumentation = false
+        }
+
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(BBEmptyBodyController())
+        }
+    }
+
+
+
+    @ParameterizedTest
+    @EnumSource
+    fun testBlackBoxOutput(outputFormat: OutputFormat) {
+
+        executeAndEvaluateBBTest(
+            outputFormat,
+            "emptybody",
+            100,
+            3,
+            listOf("PATCH","PUT","POST")
+        ){ args: MutableList<String> ->
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.size >= 1)
+            assertHasAtLeastOne(solution, HttpVerb.PATCH, 200, "/api/bbemptybody/patch", "OK")
+            assertNone(solution, HttpVerb.PATCH, 415, "/api/bbemptybody/patch", null)
+            assertNone(solution, HttpVerb.PATCH, 500, "/api/bbemptybody/patch", null)
+
+
+            assertHasAtLeastOne(solution, HttpVerb.PUT, 200, "/api/bbemptybody/put", "OK")
+            assertNone(solution, HttpVerb.PUT, 415, "/api/bbemptybody/put", null)
+            assertNone(solution, HttpVerb.PUT, 500, "/api/bbemptybody/put", null)
+
+            assertHasAtLeastOne(solution, HttpVerb.POST, 200, "/api/bbemptybody/post", "OK")
+            assertNone(solution, HttpVerb.POST, 415, "/api/bbemptybody/post", null)
+            assertNone(solution, HttpVerb.POST, 500, "/api/bbemptybody/post", null)
+        }
+    }
+}

core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt

@@ -16,6 +16,8 @@ import org.evomaster.core.problem.enterprise.EnterpriseActionGroup
 import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction
 import org.evomaster.core.problem.httpws.HttpWsAction
 import org.evomaster.core.problem.httpws.HttpWsCallResult
+import org.evomaster.core.problem.rest.data.HttpVerb
+import org.evomaster.core.problem.rest.data.RestCallAction
 import org.evomaster.core.problem.rest.param.BodyParam
 import org.evomaster.core.problem.rest.param.HeaderParam
 import org.evomaster.core.problem.security.data.ActionStubMapping
@@ -560,92 +562,107 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
             return
         }
 
-        if (bodyParam != null) {
+        if (bodyParam == null) {
 
-            val send = sendBodyCommand()
-
-            when {
-                format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")")
-                format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')")
-                format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"")
+            if(call is RestCallAction && call.verb == HttpVerb.POST && format.isJavaOrKotlin()){
+                //   RestAssured automatically add content-type for forms on POST without body :(
+                lines.add(".noContentType()")
             }
 
-            if (bodyParam.isJson()) {
+            return
+        }
 
-                if (format.isPython()) {
-                    lines.add("body = {}")
-                }
+        val send = sendBodyCommand()
 
-                val json = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = format)
+        when {
+            format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")")
+            format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')")
+            format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"")
+        }
 
-                printSendJsonBody(json, lines, dtoVar)
+        if (bodyParam.isJson()) {
 
-            } else if (bodyParam.isTextPlain()) {
+            if (format.isPython()) {
+                lines.add("body = {}")
+            }
 
-                val body = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.TEXT, targetFormat = format)
-                // handle body only if it is not black
-                if (body.isNotBlank()){
-                    if (body != "\"\"") {
-                        when {
-                            format.isCsharp() -> {
-                                lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
-                            }
-                            format.isPython() -> {
-                                if (body.trim().isNullOrBlank()) {
-                                    lines.add("body = \"\"")
-                                } else {
-                                    lines.add("body = $body")
-                                }
-                            }
-                            else -> lines.add(".$send($body)")
+            val json = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = format)
+
+            printSendJsonBody(json, lines, dtoVar)
+
+        } else if (bodyParam.isTextPlain()) {
+
+            val body = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.TEXT, targetFormat = format)
+            // handle body only if it is not black
+            if (body.isNotBlank()) {
+                if (body != "\"\"") {
+                    when {
+                        format.isCsharp() -> {
+                            lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
                         }
-                    } else {
-                        when {
-                            format.isCsharp() -> {
-                                lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
-                            }
-                            format.isPython() -> {
+
+                        format.isPython() -> {
+                            if (body.trim().isNullOrBlank()) {
                                 lines.add("body = \"\"")
+                            } else {
+                                lines.add("body = $body")
                             }
-                            else -> lines.add(".$send(\"${"""\"\""""}\")")
                         }
+
+                        else -> lines.add(".$send($body)")
                     }
-                }
+                } else {
+                    when {
+                        format.isCsharp() -> {
+                            lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
+                        }
 
-                //BMR: this is needed because, if the string is empty, it causes a 400 (bad request) code on the test end.
-                // inserting \"\" should prevent that problem
-                // TODO: get some tests done of this
+                        format.isPython() -> {
+                            lines.add("body = \"\"")
+                        }
 
-            } else if (bodyParam.isForm()) {
-                val body = bodyParam.gene.getValueAsPrintableString(
-                        mode = GeneUtils.EscapeMode.X_WWW_FORM_URLENCODED,
-                        targetFormat = format
-                )
-                when {
-                    format.isCsharp() -> {
-                        lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
-                    }
-                    format.isPython() -> {
-                        lines.add("body = \"$body\"")
+                        else -> lines.add(".$send(\"${"""\"\""""}\")")
                     }
-                    else -> lines.add(".$send(\"$body\")")
                 }
-            } else if (bodyParam.isXml()) {
+            }
 
-                val xml = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.XML, targetFormat = format)
-                // Escape quotes for string literal in generated code
-                val escapedXml = xml.replace("\\", "\\\\").replace("\"", "\\\"")
+            //BMR: this is needed because, if the string is empty, it causes a 400 (bad request) code on the test end.
+            // inserting \"\" should prevent that problem
+            // TODO: get some tests done of this
 
-                when {
-                    format.isPython() -> {
-                        lines.add("body = \"$escapedXml\"")
-                    }
-                    else -> lines.add(".$send(\"$escapedXml\")")
+        } else if (bodyParam.isForm()) {
+            val body = bodyParam.gene.getValueAsPrintableString(
+                mode = GeneUtils.EscapeMode.X_WWW_FORM_URLENCODED,
+                targetFormat = format
+            )
+            when {
+                format.isCsharp() -> {
+                    lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")")
                 }
-            } else {
-                LoggingUtil.uniqueWarn(log, "Unhandled type for body payload: " + bodyParam.contentType())
+
+                format.isPython() -> {
+                    lines.add("body = \"$body\"")
+                }
+
+                else -> lines.add(".$send(\"$body\")")
             }
+        } else if (bodyParam.isXml()) {
+
+            val xml = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.XML, targetFormat = format)
+            // Escape quotes for string literal in generated code
+            val escapedXml = xml.replace("\\", "\\\\").replace("\"", "\\\"")
+
+            when {
+                format.isPython() -> {
+                    lines.add("body = \"$escapedXml\"")
+                }
+
+                else -> lines.add(".$send(\"$escapedXml\")")
+            }
+        } else {
+            LoggingUtil.uniqueWarn(log, "Unhandled type for body payload: " + bodyParam.contentType())
         }
+
     }
 
     fun printSendJsonBody(json: String, lines: Lines, dtoVar: String? = null) {

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt

@@ -1103,37 +1103,22 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
             )
         } else if (forms != null) {
             Entity.entity(forms, MediaType.APPLICATION_FORM_URLENCODED_TYPE)
-        } else if (a.verb == HttpVerb.PUT || a.verb == HttpVerb.PATCH) {
-            /*
-                PUT and PATCH must have a payload. But it might happen that it is missing in the Swagger schema
-                when objects like WebRequest are used.
-             */
-            Entity.entity("", MediaType.APPLICATION_FORM_URLENCODED_TYPE)
-            //null //cannot be left null, Jersey crash
-        } else if (a.verb == HttpVerb.POST) {
-            /*
-                POST does not enforce payload (isn't it?). However seen issues with Dotnet that gives
-                411 if  Content-Length is missing...
-             */
-            //builder.header("Content-Length", 0)
-            // null
-            /*
-                yet another critical bug in Jersey that it ignores that header (verified with WireShark)
-             */
-            Entity.entity("", MediaType.APPLICATION_FORM_URLENCODED_TYPE)
-            //null //cannot be left null, Jersey crash
         } else {
             null
         }
 
-        if(bodyEntity != null) {
-            if(bodyEntity.entity.isEmpty()){
-                // Jersey overwrite it...
-                //builder.header("Content-Type", "")
-            } else {
-                builder.header("Content-Type", bodyEntity.mediaType)
-            }
-        }
+        /*
+            TODO
+            handling of empty body has been a shit show.
+            Before, Jersey would crash if left emtpy on PUT/PATCH/POST (which is wrong).
+            so, had to force empty bodies, which would lead to 415 when wrong type.
+            but, after upgrading, removing that wrong code was possible, but it leads to another issue:
+            for some server frameworks, might still expect 'Content-length: 0', which doesn't
+            seem possible to force in Jersey... :(
+            in those cases, then some servers might return a 411 :(
+            this happened when we were supporting C# in WB.
+            TODO should check if still a problem. if so, should reproduce and try fix
+         */
 
         val invocation = when (a.verb) {
             /*

pom.xml

@@ -835,7 +835,7 @@
             <dependency>
                 <groupId>io.rest-assured</groupId>
                 <artifactId>rest-assured</artifactId>
-                <version>4.3.0</version>
+                <version>5.5.7</version>
                 <scope>test</scope>
             </dependency>
             <dependency>