Repository navigation
Expand file tree
/
Copy pathcompressed_payload.py
More file actions
271 lines (226 loc) · 10.2 KB
/
Copy pathcompressed_payload.py
File metadata and controls
271 lines (226 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
from dataclasses import dataclass, field
from enum import Enum
from io import BytesIO
import ctypes
import struct
import sys
from uarp_payload import UarpPayload
# TODO(spotlightishere): Replace libcompression.dylib with a cross-platform implementation
if sys.platform == "darwin":
try:
libcompression = ctypes.CDLL("libcompression.dylib")
except OSError:
libcompression = None
else:
# For now, we cannot use it on non-darwin platforms.
libcompression = None
# Our header's length is 10 bytes in length.
COMPRESSED_HEADER_LENGTH = 10
class CompressionTypes(Enum):
"""Possible values for a payload's compression type."""
# Simply copies data - no compression is applied.
PASSTHROUGH = 0
# Internally known as "LZBitmapFast2"
LZBITMAPFAST = 1
# Internally known as "LZBitmap2"
LZBITMAP = 2
LZ4 = 3
def get_compression_algorithm(self) -> int:
"""Returns a usable value for compression algorithms with libcompression."""
# TODO(spotlightishere): This should be removed once usage can be
# migrated to a platform-agnostic library.
if self == CompressionTypes.PASSTHROUGH:
# This is a special, uncompressed type.
#
# If this is erroneously handled as compressed data,
# libcompression should error elsewhere.
return 0
elif self == CompressionTypes.LZBITMAPFAST:
# Undocumented compression type.
# It's 0x100 less than its sibling, LZBITMAP.
return 0x602
elif self == CompressionTypes.LZBITMAP:
# https://developer.apple.com/documentation/compression/compression_algorithm/compression_lzbitmap?language=objc
return 0x702
elif self == CompressionTypes.LZ4:
# https://developer.apple.com/documentation/compression/compression_algorithm/compression_lz4?language=objc
return 0x100
else:
raise AssertionError("Unknown compression type!")
def decompress_lzbitmap_fast(src: bytes, dst_len: int) -> bytes:
"""Decompresses Apple's LZBitmapFast2 algorithm (libcompression algorithm 0x602).
Each block emits up to 128 bytes (16 groups of 8 bytes).
Format of each block:
- 32-bit header (little endian):
low 16 bits: literal_mask (1 bit per group indicating if literal flags byte exists)
high 16 bits: distance_mask (1 bit per group indicating if uint16 distance exists)
- Uncompressed block special case:
If literal_mask == 0xFFFF and 5th byte == 0x00:
distance_mask is length to copy; if 0, marks end of stream.
- Compressed block:
popcount(literal_mask) bytes of group literal masks (1 byte per set bit)
popcount(distance_mask) uint16_le distance words (1 word per set bit)
literal stream follows
"""
dst = bytearray()
src_idx = 0
distance = 8
src_len = len(src)
while len(dst) < dst_len and src_idx < src_len:
if src_idx + 4 > src_len:
break
header = struct.unpack_from("<I", src, src_idx)[0]
literal_mask = header & 0xFFFF
distance_mask = (header >> 16) & 0xFFFF
# Check for uncompressed block:
# Literal mask is 0xFFFF and 5th byte is 0x00.
if literal_mask == 0xFFFF and src_idx + 5 <= src_len and src[src_idx + 4] == 0:
copy_len = distance_mask
if copy_len == 0:
# End of stream marker
break
src_idx += 5
to_copy = min(copy_len, dst_len - len(dst))
dst.extend(src[src_idx : src_idx + to_copy])
src_idx += copy_len
continue
src_idx += 4
# Compressed block (produces up to 128 bytes: 16 groups of 8 bytes)
# 1. Read literal masks (1 byte per set bit in literal_mask)
n_lit_masks = literal_mask.bit_count()
lit_masks = src[src_idx : src_idx + n_lit_masks]
src_idx += n_lit_masks
# 2. Read distances (uint16_le per set bit in distance_mask)
n_dists = distance_mask.bit_count()
dist_bytes = src[src_idx : src_idx + n_dists * 2]
src_idx += n_dists * 2
dists = struct.unpack_from(f"<{n_dists}H", dist_bytes) if n_dists else ()
lit_mask_idx = 0
dist_idx = 0
for g in range(16):
if len(dst) >= dst_len:
break
if distance_mask & (1 << g):
distance = dists[dist_idx]
dist_idx += 1
if literal_mask & (1 << g):
byte_mask = lit_masks[lit_mask_idx]
lit_mask_idx += 1
else:
byte_mask = 0
for b in range(8):
if len(dst) >= dst_len:
break
if byte_mask & (1 << b):
# Literal byte
dst.append(src[src_idx])
src_idx += 1
else:
# Match byte from past output
if distance > len(dst):
raise ValueError(
f"Invalid match distance: {distance} > {len(dst)}"
)
dst.append(dst[-distance])
return bytes(dst)
@dataclass
class CompressedChunk(object):
"""Parses and decompresses a chunk of a compressed payload."""
# Compression type - the only observed value is LZBitmapFast2.
# (However, this appears to match with CoreUARP's handling of such.)
raw_compression_type: int
# Offset of this chunk within the decompressed file.
# We can't use this.
decompressed_offset: int
# Amount of compressed data within this chunk.
compressed_length: int
# Size of uncompressed data this chunk produces.
#
# May be smaller than the configured block size. Payload length, rather
# than a short chunk, determines the end of the stream.
decompressed_length: int
# Our raw data to decompress.
compressed_data: bytes = field(repr=False)
def __init__(self, raw_data: BytesIO):
# Packed, big-endian uint16/uint32/uint16/uint16 fields.
header = raw_data.read(COMPRESSED_HEADER_LENGTH)
if len(header) != COMPRESSED_HEADER_LENGTH:
raise ValueError("Truncated compression chunk header")
(
raw_compression_type,
self.decompressed_offset,
self.compressed_length,
self.decompressed_length,
) = struct.unpack(">HIHH", header)
self.compression_type = CompressionTypes(raw_compression_type)
# Passthrough chunks must have the same length for compressed and decompressed data.
if self.compression_type == CompressionTypes.PASSTHROUGH:
assert (
self.compressed_length == self.decompressed_length
), "Invalid passthrough chunk lengths!"
# Our raw data is immediately beyond our header.
self.compressed_data = raw_data.read(self.compressed_length)
if len(self.compressed_data) != self.compressed_length:
raise ValueError("Truncated compressed chunk data")
def decompress(self) -> bytes:
"""Leverages libcompression from macOS or native fallback to decompress contents."""
# If this is passthrough, we simply return our "compressed" data's length.
if self.compression_type == CompressionTypes.PASSTHROUGH:
return self.compressed_data
if self.compression_type == CompressionTypes.LZBITMAPFAST:
if libcompression is not None:
try:
decompressed_buf = ctypes.create_string_buffer(self.decompressed_length)
buffer_size = libcompression.compression_decode_buffer(
decompressed_buf,
self.decompressed_length,
self.compressed_data,
self.compressed_length,
None,
self.compression_type.get_compression_algorithm(),
)
if buffer_size > 0:
return decompressed_buf[0:buffer_size]
except Exception:
pass
return decompress_lzbitmap_fast(self.compressed_data, self.decompressed_length)
# Otherwise, decompress via libcompression!
if libcompression is not None:
compression_algorithm = self.compression_type.get_compression_algorithm()
decompressed_buf = ctypes.create_string_buffer(self.decompressed_length)
buffer_size = libcompression.compression_decode_buffer(
decompressed_buf,
self.decompressed_length,
self.compressed_data,
self.compressed_length,
None,
compression_algorithm,
)
return decompressed_buf[0:buffer_size]
raise NotImplementedError(
f"Decompression for {self.compression_type.name} is not yet supported without libcompression."
)
def decompress_payload_chunks(payload: UarpPayload) -> bytes:
"""Decompresses a compressed payload within a SuperBinary."""
data = BytesIO(payload.contents)
decompressed_data = BytesIO()
# We're not presented with the resulting size of this content.
# As such, we'll need to iterate through this entire file, handling chunks as we go.
while data.tell() < len(payload.contents):
# Read and decompress the current chunk.
current_chunk = CompressedChunk(data)
decompressed = current_chunk.decompress()
# Ensure we've fully decompressed this data as expected.
expected_length = current_chunk.decompressed_length
actual_length = len(decompressed)
if expected_length != actual_length:
# We need to step back by this current chunk's size in order to get its starting offset.
chunk_offset = (
data.tell() - current_chunk.compressed_length - COMPRESSED_HEADER_LENGTH
)
raise AssertionError(
"Data did not fully decompress! "
f"(chunk offset {chunk_offset}; expected {expected_length}, but only read {actual_length})"
)
decompressed_data.write(decompressed)
return decompressed_data.getvalue()