setRecord(
new QueryParams(),
value,
contentType,
+ extraHeaders,
timeout,
options.doNotRetryTimeoutsValue());
}
diff --git a/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java b/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java
index 8f3b493..24891cd 100644
--- a/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java
+++ b/src/main/java/com/apify/client/keyvalue/SetRecordOptions.java
@@ -3,11 +3,12 @@
/**
* Write options for {@link KeyValueStoreClient#setRecord(String, byte[], String,
* SetRecordOptions)}, mirroring the reference client's {@code timeoutSecs}/{@code
- * doNotRetryTimeouts}.
+ * doNotRetryTimeouts}/{@code contentEncoding}.
*/
public final class SetRecordOptions {
private Long timeoutSecs;
private Boolean doNotRetryTimeouts;
+ private String contentEncoding;
/**
* Per-request timeout for the upload, in seconds. Use it to shorten the wait for this upload;
@@ -25,6 +26,26 @@ public SetRecordOptions doNotRetryTimeouts(Boolean doNotRetryTimeouts) {
return this;
}
+ /**
+ * Encoding already applied to the record's {@code value}, sent as the {@code Content-Encoding}
+ * header (trimmed first; it must not be blank). Set it to upload a pre-compressed value (e.g. a
+ * gzipped file read from disk): the client then sends the bytes as they are and does not compress
+ * them again. The API accepts {@code gzip}, {@code br}, {@code deflate}, and {@code identity}
+ * (matched case-insensitively) and stores the record exactly as uploaded; since this client does
+ * not advertise support for a compressed response ({@code Accept-Encoding}), a later {@link
+ * KeyValueStoreClient#getRecord(String) getRecord} transparently receives the decompressed value
+ * rather than the stored compressed bytes. Passing {@code identity} turns the client's own
+ * automatic compression off for this upload without changing how the client is configured.
+ *
+ * Unlike the reference JS client, this has no corresponding type check: {@code value} here is
+ * always raw bytes (never a string or an object the client would serialize to JSON), so there is
+ * no non-binary case to reject.
+ */
+ public SetRecordOptions contentEncoding(String contentEncoding) {
+ this.contentEncoding = contentEncoding;
+ return this;
+ }
+
Long timeoutSecsValue() {
return timeoutSecs;
}
@@ -32,4 +53,8 @@ Long timeoutSecsValue() {
boolean doNotRetryTimeoutsValue() {
return Boolean.TRUE.equals(doNotRetryTimeouts);
}
+
+ String contentEncodingValue() {
+ return contentEncoding;
+ }
}
diff --git a/src/main/java/com/apify/client/log/LogClient.java b/src/main/java/com/apify/client/log/LogClient.java
index d7bcced..7d5edae 100644
--- a/src/main/java/com/apify/client/log/LogClient.java
+++ b/src/main/java/com/apify/client/log/LogClient.java
@@ -4,7 +4,6 @@
import com.apify.client.internal.HttpClientCore;
import com.apify.client.internal.QueryParams;
import com.apify.client.internal.ResourceContext;
-import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
@@ -58,6 +57,24 @@ public CompletableFuture> get(LogOptions options) {
: Optional.of(new String(resp.body(), StandardCharsets.UTF_8)));
}
+ /**
+ * As {@link #get()}, but resolves to the log's raw bytes instead of decoding them as text -
+ * mirroring the reference clients' {@code buffer}/{@code get_as_bytes} option. Useful when the
+ * log is not UTF-8 text, or the caller wants to avoid the decoding step. See {@link #get()} for
+ * 404 handling.
+ */
+ public CompletableFuture> getAsBytes() {
+ return getAsBytes(new LogOptions());
+ }
+
+ /** As {@link #getAsBytes()}, with explicit options (raw, download). */
+ public CompletableFuture> getAsBytes(LogOptions options) {
+ QueryParams params = new QueryParams();
+ options.apply(params);
+ return ctx.getRawUnlessAmbiguous("", params)
+ .thenApply(resp -> resp == null ? Optional.empty() : Optional.of(resp.body()));
+ }
+
/**
* Opens a live, streaming connection to the log and completes with a stream over the log bytes.
* The caller is responsible for closing the returned {@link InputStream}.
@@ -76,25 +93,6 @@ public CompletableFuture stream(LogOptions options) {
params.addBool("stream", true);
options.apply(params);
String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl(""));
-
- return ctx.http
- .streamAsync(url)
- .thenApply(
- resp -> {
- if (resp.statusCode() >= HttpClientCore.MAX_SUCCESS_STATUS) {
- byte[] body = drain(resp.body());
- throw HttpClientCore.buildApiError(
- resp.statusCode(), body, 1, "GET", HttpClientCore.extractPath(url));
- }
- return resp.body();
- });
- }
-
- private static byte[] drain(InputStream in) {
- try (InputStream stream = in) {
- return stream.readAllBytes();
- } catch (IOException e) {
- return new byte[0];
- }
+ return ctx.http.streamOrThrow(url);
}
}
diff --git a/src/test/java/com/apify/client/AmbiguousNotFoundTest.java b/src/test/java/com/apify/client/AmbiguousNotFoundTest.java
index 1fcaee9..5d8451a 100644
--- a/src/test/java/com/apify/client/AmbiguousNotFoundTest.java
+++ b/src/test/java/com/apify/client/AmbiguousNotFoundTest.java
@@ -59,6 +59,11 @@ void logByIdSwallows404ToEmpty() {
assertTrue(client(404).log("missing").get().join().isEmpty());
}
+ @Test
+ void logGetAsBytesByIdSwallows404ToEmpty() {
+ assertTrue(client(404).log("missing").getAsBytes().join().isEmpty());
+ }
+
// ---- Reached through a run/build with no id of its own: throw -------------------------
@Test
diff --git a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java
index 3233912..7d19607 100644
--- a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java
+++ b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java
@@ -1,5 +1,6 @@
package com.apify.client;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -946,4 +947,93 @@ void createItemsPublicUrlWithoutFormatOmitsQueryParam() {
.join();
assertFalse(url.contains("format="), url);
}
+
+ @Test
+ void setRecordForwardsContentEncodingAndSkipsDoubleCompression() {
+ MockTransport backend = MockTransport.ofConstant(200, "");
+ // Larger than HttpClientCore's auto-compression threshold (1024 bytes) and trivially
+ // compressible, so sending it verbatim (rather than gzipping it again) is what proves the
+ // explicit header suppressed the client's own compression.
+ byte[] precompressed = "a".repeat(2000).getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ client(backend)
+ .keyValueStore("s")
+ .setRecord(
+ "k",
+ precompressed,
+ "application/octet-stream",
+ new SetRecordOptions().contentEncoding("gzip"))
+ .join();
+ assertEquals("gzip", backend.lastHeaders.firstValue("Content-Encoding").orElse(null));
+ assertArrayEquals(precompressed, backend.lastBodyBytes);
+ }
+
+ @Test
+ void setRecordRejectsBlankContentEncoding() {
+ MockTransport backend = MockTransport.ofConstant(200, "");
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ client(backend)
+ .keyValueStore("s")
+ .setRecord(
+ "k",
+ new byte[] {1},
+ "application/octet-stream",
+ new SetRecordOptions().contentEncoding(" ")));
+ assertEquals(0, backend.calls, "no request should be sent for a blank contentEncoding");
+ }
+
+ @Test
+ void setRecordTrimsContentEncodingBeforeForwarding() {
+ MockTransport backend = MockTransport.ofConstant(200, "");
+ client(backend)
+ .keyValueStore("s")
+ .setRecord(
+ "k",
+ new byte[] {1},
+ "application/octet-stream",
+ new SetRecordOptions().contentEncoding(" gzip "))
+ .join();
+ assertEquals(
+ "gzip",
+ backend.lastHeaders.firstValue("Content-Encoding").orElse(null),
+ "the header value must be trimmed, matching the reference client's schema");
+ }
+
+ @Test
+ void streamItemsResolvesToReadableStreamOfExport() throws java.io.IOException {
+ MockTransport backend = MockTransport.ofConstant(200, "");
+ backend.scriptStream(200, "a;b\n1;2\n");
+ java.io.InputStream stream =
+ TestAsync.await(
+ client(backend)
+ .dataset("d1")
+ .streamItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions()));
+ byte[] bytes = stream.readAllBytes();
+ assertEquals("a;b\n1;2\n", new String(bytes, java.nio.charset.StandardCharsets.UTF_8));
+ assertTrue(backend.lastUrl.contains("format=csv"), backend.lastUrl);
+ }
+
+ @Test
+ void streamItemsThrowsOnNonSuccessStatus() {
+ MockTransport backend = MockTransport.ofConstant(200, "");
+ backend.scriptStream(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"no\"}}");
+ ApifyApiException ex =
+ assertThrows(
+ ApifyApiException.class,
+ () ->
+ TestAsync.await(
+ client(backend)
+ .dataset("d1")
+ .streamItems(DownloadItemsFormat.JSON, new DatasetDownloadOptions())));
+ assertEquals(404, ex.getStatusCode());
+ }
+
+ @Test
+ void logGetAsBytesReturnsRawBytes() {
+ MockTransport backend = MockTransport.ofConstant(200, "log line 1\nlog line 2\n");
+ byte[] bytes = client(backend).log("l1").getAsBytes().join().orElseThrow();
+ assertEquals(
+ "log line 1\nlog line 2\n", new String(bytes, java.nio.charset.StandardCharsets.UTF_8));
+ }
}
diff --git a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java
index c4cf1a2..0f9a872 100644
--- a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java
+++ b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java
@@ -61,6 +61,9 @@ void runActorAndReadOutputs() {
Optional log = client.run(run.getId()).log().get().join();
assertTrue(log.isPresent() && !log.get().isEmpty());
+ Optional logBytes = client.run(run.getId()).log().getAsBytes().join();
+ assertTrue(logBytes.isPresent() && logBytes.get().length > 0);
+
client.run(run.getId()).dataset().listItems(new DatasetListItemsOptions()).join();
client.run(run.getId()).keyValueStore().getRecord("OUTPUT").join();
client.run(run.getId()).requestQueue().listHead(null).join();
diff --git a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java
index 2414ffe..5a83cf4 100644
--- a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java
+++ b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java
@@ -11,6 +11,7 @@
import com.apify.client.dataset.DatasetDownloadOptions;
import com.apify.client.dataset.DatasetListItemsOptions;
import com.apify.client.dataset.DownloadItemsFormat;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -39,7 +40,7 @@ void getDataset() {
}
@Test
- void datasetCrudFlow() {
+ void datasetCrudFlow() throws java.io.IOException {
ApifyClient client = requireClient();
Dataset ds = client.datasets().getOrCreate(uniqueName("ds-crud")).join();
try {
@@ -65,6 +66,13 @@ void datasetCrudFlow() {
.join();
assertTrue(new String(csv, StandardCharsets.UTF_8).contains("url"));
+ byte[] streamed;
+ try (InputStream stream =
+ dataset.streamItems(DownloadItemsFormat.JSONL, new DatasetDownloadOptions()).join()) {
+ streamed = stream.readAllBytes();
+ }
+ assertTrue(new String(streamed, StandardCharsets.UTF_8).contains("https://a.com"));
+
String url = dataset.createItemsPublicUrl(new DatasetListItemsOptions(), null).join();
assertTrue(url != null && !url.isEmpty());
diff --git a/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java b/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java
index b4417b4..7374e26 100644
--- a/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java
+++ b/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java
@@ -10,6 +10,8 @@
import com.apify.client.keyvalue.KeyValueStoreClient;
import com.apify.client.keyvalue.KeyValueStoreRecord;
import com.apify.client.keyvalue.ListKeysOptions;
+import com.apify.client.keyvalue.SetRecordOptions;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
@@ -18,6 +20,7 @@
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Optional;
+import java.util.zip.GZIPOutputStream;
import org.junit.jupiter.api.Test;
class KeyValueStoreIntegrationTest extends IntegrationBase {
@@ -99,6 +102,42 @@ void keyValueStoreCrudFlow() throws Exception {
}
}
+ @Test
+ void setRecordWithContentEncodingStoresPrecompressedValueAsIs() throws IOException {
+ ApifyClient client = requireClient();
+ KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-gzip")).join();
+ try {
+ KeyValueStoreClient kvs = client.keyValueStore(store.getId());
+ // Large and trivially compressible, so a double-gzip (the explicit header failing to
+ // suppress the client's own automatic compression) would silently corrupt it into different
+ // bytes rather than merely failing to shrink.
+ String original = "{\"hello\":\"world\",\"padding\":\"" + "x".repeat(4096) + "\"}";
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ try (GZIPOutputStream gzip = new GZIPOutputStream(buffer)) {
+ gzip.write(original.getBytes(StandardCharsets.UTF_8));
+ }
+ byte[] gzipped = buffer.toByteArray();
+
+ kvs.setRecord(
+ "report.json",
+ gzipped,
+ "application/json",
+ new SetRecordOptions().contentEncoding("gzip"))
+ .join();
+
+ // This client sends no `Accept-Encoding`, so the platform transparently decompresses the
+ // stored gzip record on the way back out - reading back the exact original plaintext proves
+ // the upload carried a single layer of gzip (the explicit header suppressed the client's own
+ // automatic compression); a double-gzip would leave one undecoded layer behind instead,
+ // failing this exact comparison.
+ Optional rec = kvs.getRecord("report.json").join();
+ assertTrue(rec.isPresent());
+ assertEquals(original, new String(rec.get().getValue(), StandardCharsets.UTF_8));
+ } finally {
+ client.keyValueStore(store.getId()).delete().join();
+ }
+ }
+
@Test
void recordPublicUrlIsFetchable() throws IOException, InterruptedException {
ApifyClient client = requireClient();