From 30246e067b4596fe9f470cb833ec117b890b290f Mon Sep 17 00:00:00 2001 From: Daniel Ng Date: Tue, 16 Jun 2026 10:17:05 -0700 Subject: [PATCH] Implement host-pinned transfer microbenchmarks and PyTree checkpoint integrations for Orbax benchmarks. PiperOrigin-RevId: 933158757 --- .../_src/serialization/async_io_engine.py | 22 +- .../_src/serialization/jax_array_handlers.py | 219 +- .../serialization/jax_array_restore_args.py | 18 +- .../_src/serialization/tensorstore_utils.py | 23 + .../serialization/tensorstore_utils_test.py | 135 + .../checkpoint/_src/serialization/types.py | 5 + .../host_pinned_transfer_benchmark.py | 233 ++ .../host_pinned_transfer_benchmark_test.py | 114 + .../benchmarks/pytree_checkpoint_benchmark.py | 3 + .../pytree_checkpoint_benchmark_test.py | 4 + .../testing/benchmarks/xpk/parse_results.py | 133 + .../tiering_service/gcp_storage_client.py | 455 ++++ .../gcp_storage_client_test.py | 281 ++ .../v1/_src/layout/orbax_layout.py | 2 +- checkpoint/pyproject.toml | 1 + .../checkpoint/v1/async_checkpointing.ipynb | 4 +- .../checkpoint/v1/checkpoint_format.ipynb | 2 +- .../checkpoint/v1/checkpointing_pytrees.ipynb | 2346 +++++++++-------- .../v1/maximizing_performance.ipynb | 848 ++++++ .../checkpoint/v1/orbax_checkpoint_101.ipynb | 2 +- .../v1/orbax_v0_to_v1_migration.ipynb | 4 +- docs/guides/checkpoint/v1/training.ipynb | 12 +- docs/index.rst | 1 + 23 files changed, 3668 insertions(+), 1199 deletions(-) create mode 100644 checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark.py create mode 100644 checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark_test.py create mode 100644 checkpoint/orbax/checkpoint/_src/testing/benchmarks/xpk/parse_results.py create mode 100644 checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py create mode 100644 checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py create mode 100644 docs/guides/checkpoint/v1/maximizing_performance.ipynb diff --git a/checkpoint/orbax/checkpoint/_src/serialization/async_io_engine.py b/checkpoint/orbax/checkpoint/_src/serialization/async_io_engine.py index c5844ba78b..4e19edff8c 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/async_io_engine.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/async_io_engine.py @@ -50,6 +50,7 @@ from orbax.checkpoint._src.futures import future from orbax.checkpoint._src.multihost import multihost from orbax.checkpoint._src.serialization import memory_regulator +from orbax.checkpoint._src.serialization import tensorstore_utils as ts_utils from orbax.checkpoint._src.serialization import type_handlers from orbax.checkpoint._src.serialization import types import tensorstore as ts @@ -88,19 +89,6 @@ def get_batch_memory_size( return sum(write_sizes), sum(read_sizes) -def _get_total_bytes_written_from_tensorstore( - metrics: Sequence[dict[str, Any]], -) -> int: - total = 0 - for m in metrics: - if m['name'].startswith('/tensorstore/kvstore/') and m['name'].endswith( - '/bytes_written' - ): - for val in m['values']: - total += val['value'] - return total - - def log_io_metrics( size: int, start_time: float, @@ -130,10 +118,12 @@ def log_io_metrics( jax.monitoring.record_scalar(gbytes_metric, value=size / (1024**3)) if initial_ts_metrics is not None: final_ts_metrics = ts.experimental_collect_matching_metrics('/tensorstore/') - initial_bytes = _get_total_bytes_written_from_tensorstore( - initial_ts_metrics + initial_bytes = ts_utils.get_total_bytes_from_tensorstore( + initial_ts_metrics, types.IoDirection.WRITE + ) + final_bytes = ts_utils.get_total_bytes_from_tensorstore( + final_ts_metrics, types.IoDirection.WRITE ) - final_bytes = _get_total_bytes_written_from_tensorstore(final_ts_metrics) compressed_bytes = final_bytes - initial_bytes if compressed_bytes > 0 and size > 0: diff --git a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_handlers.py b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_handlers.py index 9890125ff5..74e8836e2f 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_handlers.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_handlers.py @@ -25,6 +25,7 @@ import warnings from absl import logging +from etils import epath import humanize import jax import jax.numpy as jnp @@ -236,6 +237,137 @@ def _get_replica_slices( ] +def _record_logical_metrics( + direction: types.IoDirection, + logical_bytes: int, + duration: float, + storage_type: str, +): + """Records logical bytes, throughput, and duration to JAX monitoring.""" + logical_throughput = logical_bytes / duration if duration > 0 else 0 + + logging.info( + '[process=%d] %s throughput: %s/s (total gbytes: %s) (time elapsed: %s s)' + ' (per-host)', + multihost.process_index(), + f'/jax/orbax/{direction.value}/worker/io/requested', + humanize.naturalsize(logical_throughput, binary=True, format='%.3f'), + humanize.naturalsize(logical_bytes, binary=True), + duration, + ) + + jax.monitoring.record_event_duration_secs( + f'/jax/orbax/{direction.value}/worker/total_duration_secs', + duration, + storage_type=storage_type, + ) + + jax.monitoring.record_scalar( + f'/jax/orbax/{direction.value}/worker/io/requested/gbytes', + logical_bytes / (1024**3), + storage_type=storage_type, + ) + jax.monitoring.record_scalar( + f'/jax/orbax/{direction.value}/worker/io/requested/throughput/gbytes_per_sec', + logical_throughput / (1024**3), + storage_type=storage_type, + ) + + +def _record_raw_metrics( + direction: types.IoDirection, + logical_bytes: int, + duration: float, + storage_type: str, + initial_ts_metrics: Sequence[dict[str, Any]] | None = None, +): + """Records raw metrics collected from TensorStore.""" + if initial_ts_metrics is None: + return + + try: + final_ts_metrics = ts.experimental_collect_matching_metrics('/tensorstore/') + except Exception: # pylint: disable=broad-except + final_ts_metrics = None + + if final_ts_metrics is None: + return + + initial_bytes = ts_utils.get_total_bytes_from_tensorstore( + initial_ts_metrics, direction + ) + final_bytes = ts_utils.get_total_bytes_from_tensorstore( + final_ts_metrics, direction + ) + raw_bytes = final_bytes - initial_bytes + + if raw_bytes <= 0: + return + + raw_throughput = raw_bytes / duration if duration > 0 else 0 + logging.info( + '[process=%d] Raw %s throughput: %s/s (total gbytes: %s) (time elapsed:' + ' %s s) (per-host)', + multihost.process_index(), + f'/jax/orbax/{direction.value}/worker/io/raw', + humanize.naturalsize(raw_throughput, binary=True, format='%.3f'), + humanize.naturalsize(raw_bytes, binary=True), + duration, + ) + jax.monitoring.record_scalar( + f'/jax/orbax/{direction.value}/worker/io/raw/gbytes', + raw_bytes / (1024**3), + storage_type=storage_type, + ) + jax.monitoring.record_scalar( + f'/jax/orbax/{direction.value}/worker/io/raw/throughput/gbytes_per_sec', + raw_throughput / (1024**3), + storage_type=storage_type, + ) + + if logical_bytes > 0: + ratio = float(raw_bytes) / logical_bytes + logging.info( + '[process=%d] %s ratio (raw/logical): %.3f (%s / %s)', + multihost.process_index(), + direction.value.capitalize(), + ratio, + humanize.naturalsize(raw_bytes, binary=True), + humanize.naturalsize(logical_bytes, binary=True), + ) + jax.monitoring.record_scalar( + f'/jax/orbax/{direction.value}/worker/io/compression_ratio', + ratio, + storage_type=storage_type, + ) + + +def _log_io_metrics( + direction: types.IoDirection, + logical_bytes: int, + start_time: float, + parent_dir: epath.Path, + initial_ts_metrics: Sequence[dict[str, Any]] | None = None, +): + """Logs and records IO telemetry metrics for array serialization/deserialization.""" + duration = time.time() - start_time + storage_type = path_utils.get_storage_type(parent_dir) + + _record_logical_metrics( + direction, + logical_bytes, + duration, + storage_type, + ) + _record_raw_metrics( + direction, + logical_bytes, + duration, + storage_type, + initial_ts_metrics=initial_ts_metrics, + ) + + def _worker_serialize_arrays( arrays: Sequence[jax.Array], infos: Sequence[types.ParamInfo], @@ -251,6 +383,13 @@ def _worker_serialize_arrays( ext_metadata: Dict[str, Any], ): """Worker function to serialize arrays.""" + try: + initial_ts_metrics = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + except Exception: # pylint: disable=broad-except + initial_ts_metrics = None + total_start_time = time.time() rslices_per_array = _get_replica_slices( arrays, replica_id, @@ -272,6 +411,15 @@ def _worker_serialize_arrays( ext_metadata=ext_metadata, ) ) + if infos: + total_io_bytes = sum(v.nbytes for v in rslices_per_array) + _log_io_metrics( + direction=types.IoDirection.WRITE, + logical_bytes=total_io_bytes, + start_time=total_start_time, + parent_dir=infos[0].parent_dir, + initial_ts_metrics=initial_ts_metrics, + ) def _get_deprioritized_batches_to_serialize( @@ -380,7 +528,19 @@ def _serialize_arrays_batches_without_dispatcher( ) async def _serialize_without_dispatcher(): + if not prioritized and not deprioritized: + return + try: + initial_ts_metrics = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + except Exception: # pylint: disable=broad-except + initial_ts_metrics = None + total_start_time = time.time() + total_io_bytes = 0 + if prioritized_values_on_host: + total_io_bytes += sum(v.nbytes for v in prioritized_values_on_host) await async_serialize_replica_slices_batch( prioritized_values_on_host, prioritized_infos, @@ -404,6 +564,7 @@ async def _serialize_without_dispatcher(): ): b_arrays_on_host = replica_slices_transfer_arrays_to_host(b_arrays) _on_batch_callback(b_infos, callback.on_transfer_end) + total_io_bytes += sum(v.nbytes for v in b_arrays_on_host) await async_serialize_replica_slices_batch( b_arrays_on_host, b_infos, @@ -411,6 +572,15 @@ async def _serialize_without_dispatcher(): ) _on_batch_callback(b_infos, callback.on_write_end) + info_sample = prioritized[0][1] if prioritized else deprioritized[0][1] + _log_io_metrics( + direction=types.IoDirection.WRITE, + logical_bytes=total_io_bytes, + start_time=total_start_time, + parent_dir=info_sample.parent_dir, + initial_ts_metrics=initial_ts_metrics, + ) + return future.CommitFutureAwaitingContractedSignals( _serialize_without_dispatcher(), name='array_type_handler', @@ -787,6 +957,12 @@ async def _deserialize_arrays( array_metadata_store: array_metadata_store_lib.Store | None, ) -> Sequence[jax.Array]: """Deserializes arrays and applies array_metadata if available.""" + try: + initial_ts_metrics = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + except Exception: # pylint: disable=broad-except + initial_ts_metrics = None total_start_time = time.time() async def _async_deserialize( @@ -884,41 +1060,14 @@ async def _async_deserialize( metadata_key=metadata_key, ) - total_duration = time.time() - total_start_time - io_throughput = total_io_bytes / total_duration if total_duration > 0 else 0 - - storage_type = path_utils.get_storage_type(infos[0].parent_dir) - - logging.info( - '[process=%d] %s throughput: %s/s (total gbytes: %s) (time elapsed: %s s)' - ' (per-host)', - multihost.process_index(), - '/jax/orbax/read/worker/io/requested', - humanize.naturalsize(io_throughput, binary=True, format='%.3f'), - humanize.naturalsize(total_io_bytes, binary=True), - total_duration, - ) - - # Record total duration of the read operation. Note that for McJAX, it - # includes IO time and H2D transfer time. For Pathways Remote Python, - # it includes only IO time. - jax.monitoring.record_event_duration_secs( - '/jax/orbax/read/worker/total_duration_secs', - total_duration, - storage_type=storage_type, - ) - - # record total bytes requested to be read from IO - jax.monitoring.record_scalar( - '/jax/orbax/read/worker/io/requested/gbytes', - total_io_bytes / (1024**3), - storage_type=storage_type, - ) - jax.monitoring.record_scalar( - '/jax/orbax/read/worker/io/requested/throughput/gbytes_per_sec', - io_throughput / (1024**3), - storage_type=storage_type, - ) + if infos: + _log_io_metrics( + direction=types.IoDirection.READ, + logical_bytes=total_io_bytes, + start_time=total_start_time, + parent_dir=infos[0].parent_dir, + initial_ts_metrics=initial_ts_metrics, + ) return ret diff --git a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py index 20b7dab544..768bf8d26a 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py @@ -101,11 +101,13 @@ class SingleReplicaArrayRestoreArgs(ArrayRestoreArgs): def __post_init__(self): super().__post_init__() - logging.log_first_n( - logging.WARNING, - '`single_replica_sharding` is deprecated and will be removed in a' - ' future version. It is not needed, as Orbax code will automatically' - ' construct a single-replica sharding used for restoring before' - ' broadcasting.', - 1, - ) + if self.single_replica_sharding is not None: + logging.log_first_n( + logging.WARNING, + '`single_replica_sharding` is deprecated and will be removed in a' + ' future version. It is not needed, as Orbax code will automatically' + ' construct a single-replica sharding used for restoring before' + ' broadcasting.', + 1, + ) + diff --git a/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils.py b/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils.py index 382986c35d..8601057288 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils.py @@ -926,6 +926,29 @@ def array_metadata_from_tensorstore( ) +def get_total_bytes_from_tensorstore( + metrics: Sequence[dict[str, Any]], direction: types.IoDirection +) -> int: + """Sums bytes_read or bytes_written from all kvstore drivers in metrics.""" + total = 0 + if direction == types.IoDirection.WRITE: + suffix = '/bytes_written' + elif direction == types.IoDirection.READ: + suffix = '/bytes_read' + else: + raise ValueError(f'Invalid direction: {direction}') + + for m in metrics: + if not isinstance(m, dict): + continue + name = m.get('name', '') + if name.startswith('/tensorstore/kvstore/') and name.endswith(suffix): + for val in m.get('values', []): + if isinstance(val, dict): + total += val.get('value', 0) + return total + + def print_ts_debug_data(key: str | None, infos: Sequence[types.ParamInfo]): """Log Tensorstore related metrics.""" ts_metrics = ts.experimental_collect_matching_metrics('/tensorstore') diff --git a/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils_test.py b/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils_test.py index 3231cfa8fe..e4386042c9 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils_test.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/tensorstore_utils_test.py @@ -19,10 +19,13 @@ from absl.testing import absltest from absl.testing import parameterized +from etils import epath import numpy as np from orbax.checkpoint._src.arrays import subchunking from orbax.checkpoint._src.arrays import types from orbax.checkpoint._src.serialization import tensorstore_utils as ts_utils +from orbax.checkpoint._src.serialization import types as serialization_types +import tensorstore as ts GIB = 1024**3 @@ -1026,5 +1029,137 @@ def test_get_ts_context( self.assertDictEqual(expected_spec, context.spec.to_json()) +class GetTotalBytesFromTensorstoreTest(parameterized.TestCase): + + def test_get_total_bytes_written(self): + metrics = [ + { + 'name': '/tensorstore/kvstore/gcs/bytes_written', + 'values': [{'value': 100}, {'value': 200}], + }, + { + 'name': '/tensorstore/kvstore/gfile/bytes_written', + 'values': [{'value': 50}], + }, + { + 'name': '/tensorstore/kvstore/gcs/bytes_read', + 'values': [{'value': 500}], + }, + { + 'name': '/other/metric/bytes_written', + 'values': [{'value': 1000}], + }, + ] + self.assertEqual( + ts_utils.get_total_bytes_from_tensorstore( + metrics, serialization_types.IoDirection.WRITE + ), + 350, + ) + + def test_get_total_bytes_read(self): + metrics = [ + { + 'name': '/tensorstore/kvstore/gcs/bytes_written', + 'values': [{'value': 100}], + }, + { + 'name': '/tensorstore/kvstore/gcs/bytes_read', + 'values': [{'value': 500}, {'value': 250}], + }, + { + 'name': '/tensorstore/kvstore/gfile/bytes_read', + 'values': [{'value': 50}], + }, + ] + self.assertEqual( + ts_utils.get_total_bytes_from_tensorstore( + metrics, serialization_types.IoDirection.READ + ), + 800, + ) + + @parameterized.named_parameters( + ('with_compression', True), + ('without_compression', False), + ) + def test_get_total_bytes_with_real_ops(self, use_compression): + initial_metrics_write = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + + tempdir = self.create_tempdir().full_path + info = serialization_types.ParamInfo( + name='arr', + parent_dir=epath.Path(tempdir), + use_compression=use_compression, + ) + write_spec = ts_utils.build_array_write_spec( + info, + global_shape=(100000,), + local_shape=(100000,), + dtype=np.dtype(np.int32), + use_ocdbt=True, + ) + write_context = ts_utils.get_ts_context(use_ocdbt=True) + store = ts.open( + write_spec.json, + context=write_context, + create=True, + delete_existing=True, + dtype=np.int32, + shape=(100000,), + ).result() + store.write(np.arange(100000, dtype=np.int32)).result() + + final_metrics_write = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + + bytes_written = ts_utils.get_total_bytes_from_tensorstore( + final_metrics_write, serialization_types.IoDirection.WRITE + ) - ts_utils.get_total_bytes_from_tensorstore( + initial_metrics_write, serialization_types.IoDirection.WRITE + ) + + self.assertGreater(bytes_written, 0) + + initial_metrics_read = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + + read_spec = ts_utils.build_array_read_spec(info, use_ocdbt=True) + read_context = ts_utils.get_ts_context(use_ocdbt=True) + read_store = ts.open( + read_spec.json, + open=True, + context=read_context, + dtype=np.int32, + shape=(100000,), + ).result() + read_store.read().result() + + final_metrics_read = ts.experimental_collect_matching_metrics( + '/tensorstore/' + ) + + bytes_read = ts_utils.get_total_bytes_from_tensorstore( + final_metrics_read, serialization_types.IoDirection.READ + ) - ts_utils.get_total_bytes_from_tensorstore( + initial_metrics_read, serialization_types.IoDirection.READ + ) + + self.assertGreater(bytes_read, 0) + + if not use_compression: + # Logical size is 100000 * 4 bytes = 400000 bytes. + self.assertLess(abs(bytes_written - 400000) / 400000, 0.01) + self.assertLess(abs(bytes_read - 400000) / 400000, 0.01) + else: + # Verify that compression actually reduced the bytes written/read. + self.assertLess(bytes_written, 300000) + self.assertLess(bytes_read, 300000) + + if __name__ == '__main__': absltest.main() diff --git a/checkpoint/orbax/checkpoint/_src/serialization/types.py b/checkpoint/orbax/checkpoint/_src/serialization/types.py index 3cf223e8e9..6994b47ddd 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/types.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/types.py @@ -64,6 +64,11 @@ class TransferPriority(enum.Enum): UNKNOWN = 3 +class IoDirection(enum.Enum): + READ = 'read' + WRITE = 'write' + + class SerializationStatusCallback(Protocol): """Callback for tracking serialization status of PyTree parameters. diff --git a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark.py b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark.py new file mode 100644 index 0000000000..523f26e55a --- /dev/null +++ b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark.py @@ -0,0 +1,233 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2024 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microbenchmarks for Host Pinned Transfer.""" + +from collections.abc import Sequence +import dataclasses +import time +import jax +import jax.numpy as jnp +import numpy as np +from orbax.checkpoint._src.testing.benchmarks.core import core as benchmarks_core +from orbax.checkpoint._src.testing.benchmarks.core import metric as metric_lib + + +@dataclasses.dataclass(frozen=True) +class HostPinnedTransferOptions(benchmarks_core.BenchmarkOptions): + """Options for HostPinnedTransferBenchmark. + + Each attribute can be a single value or a list of values to create + a parameter sweep. + """ + + # Shape of the array to transfer. + shape: tuple[int, ...] | Sequence[tuple[int, ...]] = (8192, 128, 128) + # Data type. + dtype: str | Sequence[str] = "float32" + # Target transfer mode: eager (host-side Python loop + block_until_ready) or + # jit (jax.jit + jax.lax.scan). + mode: str | Sequence[str] = "eager" + # Number of transfer iterations to time. + iterations: int | Sequence[int] = 20 + # Number of warmup iterations. + warmup: int | Sequence[int] = 5 + + def is_valid(self) -> bool: + assert isinstance(self.shape, tuple) + assert isinstance(self.dtype, str) + assert isinstance(self.mode, str) + assert isinstance(self.iterations, int) + assert isinstance(self.warmup, int) + if self.mode not in ("eager", "jit"): + return False + return True + + +@benchmarks_core.benchmark_options(HostPinnedTransferOptions) +class HostPinnedTransferBenchmark(benchmarks_core.BenchmarksGenerator): + """Microbenchmark for raw TPU H2D and D2H transfer speeds.""" + + def test_fn( + self, context: benchmarks_core.TestContext + ) -> benchmarks_core.TestResult: + metrics = metric_lib.Metrics() + options = context.options + assert isinstance(options, HostPinnedTransferOptions) + mesh = context.mesh + if mesh is None: + devices = jax.devices() + mesh = jax.sharding.Mesh(np.array(devices), ("y",)) + + # Resolve shape and dtype + shape = options.shape + dtype = jnp.dtype(options.dtype) + size_bytes = jnp.zeros(shape, dtype=dtype).nbytes + size_gb = size_bytes / 1e9 + + # Shardings + device_sharding = jax.sharding.NamedSharding( + mesh, jax.sharding.PartitionSpec("y"), memory_kind="device" + ) + pinned_host_sharding = jax.sharding.NamedSharding( + mesh, jax.sharding.PartitionSpec("y"), memory_kind="pinned_host" + ) + unpinned_cpu_sharding = jax.sharding.NamedSharding( + mesh, jax.sharding.PartitionSpec("y"), memory_kind="unpinned_host" + ) + + # Generate source data on CPU/Host first, then put/run + np_array = jax.random.normal(jax.random.PRNGKey(0), shape, dtype=dtype) + + if options.mode == "eager": + # ---------------------------------------------------- + # 1. H2D (Host -> Device) + # ---------------------------------------------------- + # Setup Host arrays (Pinned and Unpinned) + host_pinned_array = jax.device_put(np_array, pinned_host_sharding) + host_unpinned_array = jax.device_put(np_array, unpinned_cpu_sharding) + + # Benchmark: H2D Pinned Host -> Device + for _ in range(options.warmup): + jax.device_put(host_pinned_array, device_sharding).block_until_ready() + + t0 = time.perf_counter() + for _ in range(options.iterations): + jax.device_put(host_pinned_array, device_sharding).block_until_ready() + h2d_pinned_s = (time.perf_counter() - t0) / options.iterations + metrics.results["h2d_pinned_latency_s"] = (h2d_pinned_s, "s") + metrics.results["h2d_pinned_throughput_gbps"] = ( + size_gb / h2d_pinned_s, + "Gbps", + ) + + # Benchmark: H2D Unpinned -> Device + for _ in range(options.warmup): + jax.device_put(host_unpinned_array, device_sharding).block_until_ready() + + t0 = time.perf_counter() + for _ in range(options.iterations): + jax.device_put(host_unpinned_array, device_sharding).block_until_ready() + h2d_unpinned_s = (time.perf_counter() - t0) / options.iterations + metrics.results["h2d_unpinned_latency_s"] = (h2d_unpinned_s, "s") + metrics.results["h2d_unpinned_throughput_gbps"] = ( + size_gb / h2d_unpinned_s, + "Gbps", + ) + + # ---------------------------------------------------- + # 2. D2H (Device -> Host) + # ---------------------------------------------------- + device_array = jax.device_put(np_array, device_sharding) + + # Benchmark: D2H Device -> Pinned Host + for _ in range(options.warmup): + jax.device_put(device_array, pinned_host_sharding).block_until_ready() + + t0 = time.perf_counter() + for _ in range(options.iterations): + jax.device_put(device_array, pinned_host_sharding).block_until_ready() + d2h_pinned_s = (time.perf_counter() - t0) / options.iterations + metrics.results["d2h_pinned_latency_s"] = (d2h_pinned_s, "s") + metrics.results["d2h_pinned_throughput_gbps"] = ( + size_gb / d2h_pinned_s, + "Gbps", + ) + + # Benchmark: D2H Device -> Unpinned CPU + for _ in range(options.warmup): + jax.device_put(device_array, unpinned_cpu_sharding).block_until_ready() + + t0 = time.perf_counter() + for _ in range(options.iterations): + jax.device_put(device_array, unpinned_cpu_sharding).block_until_ready() + d2h_unpinned_s = (time.perf_counter() - t0) / options.iterations + metrics.results["d2h_unpinned_latency_s"] = (d2h_unpinned_s, "s") + metrics.results["d2h_unpinned_throughput_gbps"] = ( + size_gb / d2h_unpinned_s, + "Gbps", + ) + + elif options.mode == "jit": + # JIT Mode: We run compute + transfer in a JIT compiled scan loop to + # measure compiler pipeline speed. + device_array_init = jax.device_put(np_array, device_sharding) + + def offload_pinned_step(carry, _): + # Dummy compute + HBM-to-Pinned-Host transfer + res = carry * -1 + res_host = jax.device_put(res, pinned_host_sharding) + return res, res_host + + def offload_unpinned_step(carry, _): + # Dummy compute + HBM-to-Unpinned-CPU transfer + res = carry * -1 + res_host = jax.device_put(res, unpinned_cpu_sharding) + return res, res_host + + # Compile loops + jit_pinned = jax.jit( + lambda x: jax.lax.scan( + offload_pinned_step, x, None, length=options.iterations + ) + ) + jit_unpinned = jax.jit( + lambda x: jax.lax.scan( + offload_unpinned_step, x, None, length=options.iterations + ) + ) + + # Warmup + _, res_hosts = jit_pinned(device_array_init) + res_hosts.block_until_ready() + _, res_hosts = jit_unpinned(device_array_init) + res_hosts.block_until_ready() + + # Time Pinned JIT offload + t0 = time.perf_counter() + _, res_hosts = jit_pinned(device_array_init) + res_hosts.block_until_ready() + d2h_pinned_jit_s = (time.perf_counter() - t0) / options.iterations + metrics.results["d2h_pinned_jit_latency_s"] = (d2h_pinned_jit_s, "s") + metrics.results["d2h_pinned_jit_throughput_gbps"] = ( + size_gb / d2h_pinned_jit_s, + "Gbps", + ) + + # Time Unpinned JIT offload + t0 = time.perf_counter() + _, res_hosts = jit_unpinned(device_array_init) + res_hosts.block_until_ready() + d2h_unpinned_jit_s = (time.perf_counter() - t0) / options.iterations + metrics.results["d2h_unpinned_jit_latency_s"] = (d2h_unpinned_jit_s, "s") + metrics.results["d2h_unpinned_jit_throughput_gbps"] = ( + size_gb / d2h_unpinned_jit_s, + "Gbps", + ) + + return benchmarks_core.TestResult(metrics=metrics) diff --git a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark_test.py b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark_test.py new file mode 100644 index 0000000000..0a72d3836a --- /dev/null +++ b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/host_pinned_transfer_benchmark_test.py @@ -0,0 +1,114 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2024 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests verifying host-pinned D2H/H2D memory transfer microbenchmarks.""" + +from absl.testing import absltest +from absl.testing import parameterized +from etils import epath +from orbax.checkpoint._src.testing.benchmarks import host_pinned_transfer_benchmark +from orbax.checkpoint._src.testing.benchmarks.core import configs as benchmarks_configs +from orbax.checkpoint._src.testing.benchmarks.core import core as benchmarks_core + +HostPinnedTransferOptions = ( + host_pinned_transfer_benchmark.HostPinnedTransferOptions +) +HostPinnedTransferBenchmark = ( + host_pinned_transfer_benchmark.HostPinnedTransferBenchmark +) + + +class HostPinnedTransferBenchmarkTest(parameterized.TestCase): + + @parameterized.parameters( + dict( + options=HostPinnedTransferOptions(mode='eager'), + expected_len=1, + ), + dict( + options=HostPinnedTransferOptions(mode=['eager', 'jit']), + expected_len=2, + ), + ) + def test_generate_benchmarks(self, options, expected_len): + generator = HostPinnedTransferBenchmark( + checkpoint_configs=[benchmarks_configs.CheckpointConfig(spec={})], + options=options, + ) + benchmarks = generator.generate() + self.assertLen(benchmarks, expected_len) + for benchmark in benchmarks: + self.assertIsInstance(benchmark.options, HostPinnedTransferOptions) + + @parameterized.parameters('eager', 'jit') + def test_benchmark_test_fn(self, mode): + generator = HostPinnedTransferBenchmark( + checkpoint_configs=[benchmarks_configs.CheckpointConfig(spec={})], + options=HostPinnedTransferOptions(), + ) + test_path = epath.Path(self.create_tempdir().full_path) + test_options = HostPinnedTransferOptions( + shape=(128, 128), + iterations=2, + warmup=1, + mode=mode, + ) + context = benchmarks_core.TestContext( + pytree=None, path=test_path, options=test_options + ) + + result = generator.test_fn(context) + self.assertIsInstance(result, benchmarks_core.TestResult) + if mode == 'eager': + self.assertContainsSubset( + { + 'h2d_pinned_latency_s', + 'h2d_pinned_throughput_gbps', + 'h2d_unpinned_latency_s', + 'h2d_unpinned_throughput_gbps', + 'd2h_pinned_latency_s', + 'd2h_pinned_throughput_gbps', + 'd2h_unpinned_latency_s', + 'd2h_unpinned_throughput_gbps', + }, + result.metrics.results.keys(), + ) + elif mode == 'jit': + self.assertContainsSubset( + { + 'd2h_pinned_jit_latency_s', + 'd2h_pinned_jit_throughput_gbps', + 'd2h_unpinned_jit_latency_s', + 'd2h_unpinned_jit_throughput_gbps', + }, + result.metrics.results.keys(), + ) + + +if __name__ == '__main__': + absltest.main() diff --git a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark.py b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark.py index 73af9afb74..46d1d30626 100644 --- a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark.py +++ b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark.py @@ -77,11 +77,13 @@ class PyTreeCheckpointOptions(benchmarks_core.BenchmarkOptions): enable_replica_parallel_separate_folder: bool | Sequence[bool] = False use_colocated_python: bool | Sequence[bool] = False save_device_host_concurrent_gb: int | None | Sequence[int | None] = None + enable_pinned_host_transfer: bool | Sequence[bool] = True def is_valid(self): assert isinstance(self.use_replica_parallel, bool) assert isinstance(self.enable_replica_parallel_separate_folder, bool) assert isinstance(self.use_colocated_python, bool) + assert isinstance(self.enable_pinned_host_transfer, bool) if self.enable_replica_parallel_separate_folder and ( not self.use_replica_parallel or not self.use_ocdbt @@ -167,6 +169,7 @@ def test_fn( save_concurrent_gb=options.save_concurrent_gb, restore_concurrent_gb=options.restore_concurrent_gb, save_device_host_concurrent_gb=options.save_device_host_concurrent_gb, + enable_pinned_host_transfer=options.enable_pinned_host_transfer, is_prioritized_key_fn=lambda key: "a" in ocp.tree.str_keypath(key), ) diff --git a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark_test.py b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark_test.py index ebf3fed607..e7314f1ee9 100644 --- a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark_test.py +++ b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/pytree_checkpoint_benchmark_test.py @@ -84,6 +84,7 @@ def test_generate_benchmarks(self, options, expected_len): use_replica_parallel=(True,), enable_replica_parallel_separate_folder=(False,), use_colocated_python=(False,), + enable_pinned_host_transfer=(False, True), ) def test_benchmark_test_fn( self, @@ -96,6 +97,7 @@ def test_benchmark_test_fn( use_replica_parallel, enable_replica_parallel_separate_folder, use_colocated_python, + enable_pinned_host_transfer, ): generator = PyTreeCheckpointBenchmark( checkpoint_configs=[benchmarks_configs.CheckpointConfig(spec={})], @@ -119,6 +121,7 @@ def test_benchmark_test_fn( use_replica_parallel=use_replica_parallel, enable_replica_parallel_separate_folder=enable_replica_parallel_separate_folder, use_colocated_python=use_colocated_python, + enable_pinned_host_transfer=enable_pinned_host_transfer, ) context = benchmarks_core.TestContext( pytree=pytree, path=test_path, options=test_options @@ -133,6 +136,7 @@ def test_benchmark_test_fn( save_concurrent_gb=save_concurrent_gb, restore_concurrent_gb=restore_concurrent_gb, save_device_host_concurrent_gb=save_device_host_concurrent_gb, + enable_pinned_host_transfer=enable_pinned_host_transfer, is_prioritized_key_fn=mock.ANY, ) self.mock_checkpointer.assert_called_once_with( diff --git a/checkpoint/orbax/checkpoint/_src/testing/benchmarks/xpk/parse_results.py b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/xpk/parse_results.py new file mode 100644 index 0000000000..e7c3754370 --- /dev/null +++ b/checkpoint/orbax/checkpoint/_src/testing/benchmarks/xpk/parse_results.py @@ -0,0 +1,133 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2024 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parses Orbax benchmark run results from GCS.""" + +import argparse +import json +from etils import epath +import numpy as np + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--output_dir', + type=str, + required=True, + help=( + 'GCS root directory of the runs (e.g.' + ' gs://bucket/runs/20260616-1703UTC)' + ), + ) + return parser.parse_args() + + +def parse_run_results(run_dir_path: epath.Path): + """Parses and aggregates means from host sidecar JSON files in a run.""" + print('\n==================================================') + print(f'Parsing results in: {run_dir_path}') + print('==================================================') + + tb_dir = run_dir_path / 'tensorboard' + if not tb_dir.exists(): + print('No tensorboard directory found!') + return + + for benchmark_dir in tb_dir.iterdir(): + if not benchmark_dir.is_dir(): + continue + + benchmark_name = benchmark_dir.name + print(f'\nBenchmark: {benchmark_name}') + + # Locate per-host means JSON files + means_dir = benchmark_dir / '_per_host_means' + if not means_dir.exists(): + # Check if per-host metrics were enabled directly + means_dir = benchmark_dir + + # Gather all hosts + hosts_data = [] + for host_dir in means_dir.iterdir(): + if host_dir.name.startswith('host_'): + sidecar = host_dir / '_per_host_means.json' + if sidecar.exists(): + try: + hosts_data.append(json.loads(sidecar.read_text())) + except Exception as e: # pylint: disable=broad-exception-caught + print(f'Failed to read {sidecar}: {e}') + + if not hosts_data: + print('No host metrics found.') + continue + + # Aggregate across hosts + all_keys = sorted(list(set().union(*(d['keys'] for d in hosts_data)))) + + # Print metrics + print(f'Aggregated Metrics (across {len(hosts_data)} hosts):') + for key in all_keys: + values = [] + for d in hosts_data: + if key in d['keys']: + idx = d['keys'].index(key) + values.append(d['means'][idx]) + + if values: + mean_val = np.mean(values) + max_val = np.max(values) + min_val = np.min(values) + + # Highlight important bandwidth/throughput metrics + if 'throughput' in key or 'ratio' in key or 'gbps' in key: + print( + f' \033[1;32m{key:60s}: mean={mean_val:10.4f} |' + f' max={max_val:10.4f} | min={min_val:10.4f}\033[0m' + ) + else: + print(f' {key:60s}: mean={mean_val:10.4f} s') + + +def main(): + args = parse_args() + root_path = epath.Path(args.output_dir) + if not root_path.exists(): + print(f'Error: {root_path} does not exist!') + return + + # Walk subdirectories (e.g. v5e/micro, v5e/bandwidth, v5p/micro...) + for tpu_gen_dir in root_path.iterdir(): + if tpu_gen_dir.is_dir(): + for run_type_dir in tpu_gen_dir.iterdir(): + if run_type_dir.is_dir(): + parse_run_results(run_type_dir) + + +if __name__ == '__main__': + main() diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py new file mode 100644 index 0000000000..6971b69ea1 --- /dev/null +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py @@ -0,0 +1,455 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GCP storage clients for Checkpoint Tiering Service (CTS).""" + +import abc +import asyncio +import dataclasses +import datetime +import enum +import os +from typing import Any +import google.auth +from google.auth import exceptions as auth_exceptions +from google.auth import impersonated_credentials +from google.auth import transport +import httpx + + +class OperationStatus(enum.Enum): + """The status of a job or operation.""" + + IN_PROGRESS = "IN_PROGRESS" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + + +@dataclasses.dataclass +class Result: + status: OperationStatus + detail_info: dict[str, Any] + + +@dataclasses.dataclass +class TransferContext: + job_request_id: str + source_path: str + destination_path: str + transfer_status: dict[str, Any] + + +class HttpxRequest(transport.Request): + """A google-auth compatible transport request using HTTPX (sync).""" + + def __init__(self, client: httpx.Client): + self._client = client + + def __call__( + self, + url: str, + method: str = "GET", + body: bytes | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + **kwargs, + ) -> transport.Response: + try: + response = self._client.request( + method=method, + url=url, + headers=headers, + content=body, + timeout=timeout, + **kwargs, + ) + + class HttpxResponse(transport.Response): + """A google-auth compatible transport response using HTTPX.""" + + def __init__(self, resp): + self._resp = resp + + @property + def status(self) -> int: + return self._resp.status_code + + @property + def headers(self) -> dict[str, str]: + return dict(self._resp.headers) + + @property + def data(self) -> bytes: + return self._resp.content + + return HttpxResponse(response) + except httpx.TimeoutException as e: + raise auth_exceptions.TransportError(f"Timeout: {e}") + except httpx.RequestError as e: + raise auth_exceptions.TransportError(f"Request error: {e}") + + +class GCPStorageClient(abc.ABC): + """Client interface to interact with GCP storage backend (e.g. + + Lustre, GCS). + """ + + def __init__( + self, + project: str | None = None, + location: str | None = None, + instance: str | None = None, + service_account: str | None = None, + ): + self.project = project + self.location = location + self.instance = instance + self.service_account = service_account + self._credentials = None + self._async_client = None + + @property + def async_client(self) -> httpx.AsyncClient: + if self._async_client is None: + self._async_client = httpx.AsyncClient() + return self._async_client + + async def close(self): + if self._async_client is not None: + await self._async_client.aclose() + self._async_client = None + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _get_token_and_project(self) -> tuple[str, str]: + """Gets authentication credentials and projects.""" + if not self._credentials: + base_credentials, detected_project = await asyncio.to_thread( + google.auth.default, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + if not self.project: + self.project = detected_project + + if self.service_account: + self._credentials = impersonated_credentials.Credentials( + source_credentials=base_credentials, + target_principal=self.service_account, + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + else: + self._credentials = base_credentials + + if not self._credentials.valid: + with httpx.Client() as client: + await asyncio.to_thread(self._credentials.refresh, HttpxRequest(client)) + + if not self.project: + raise ValueError("GCP Project ID must be specified or auto-detected.") + + return self._credentials.token, self.project + + @abc.abstractmethod + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers copy and returns operation name.""" + pass + + @abc.abstractmethod + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls operation status and returns a Result object.""" + pass + + +def _parse_gcs_path(gcs_path: str) -> tuple[str, str]: + """Parses a GCS path like gs://bucket/prefix/file into (bucket, prefix).""" + path_no_scheme = gcs_path.replace("gs://", "") + parts = path_no_scheme.split("/", 1) + bucket = parts[0] + prefix = parts[1] if len(parts) > 1 else "" + return bucket, prefix + + +class GcsToGcsClient(GCPStorageClient): + """Client implementation for GCS-to-GCS operations using Storage Transfer Service.""" + + def __init__( + self, + project: str | None = None, + service_account: str | None = None, + ): + super().__init__(project=project, service_account=service_account) + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers GCS-to-GCS transfer using GCP Storage Transfer Service (STS).""" + token, project = await self._get_token_and_project() + url = "https://storagetransfer.googleapis.com/v1/transferJobs" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + src_bucket, src_prefix = _parse_gcs_path(source_path) + dest_bucket, dest_prefix = _parse_gcs_path(destination_path) + + now = datetime.datetime.now(datetime.timezone.utc) + payload = { + "projectId": project, + "transferSpec": { + "gcsDataSource": { + "bucketName": src_bucket, + "path": src_prefix, + }, + "gcsDataSink": { + "bucketName": dest_bucket, + "path": dest_prefix, + }, + "transferOptions": { + "overwriteObjectsAlreadyExistingInSink": True, + }, + }, + "schedule": { + "scheduleStartDate": { + "year": now.year, + "month": now.month, + "day": now.day, + }, + }, + "status": "DISABLED", + } + + response = await self.async_client.post(url, json=payload, headers=headers) + + if response.status_code != 200: + raise RuntimeError( + f"Failed to create Storage Transfer Job: {response.status_code} -" + f" {response.text}" + ) + + job_name = response.json()["name"] + + # Run the job immediately. This returns the operation name directly. + run_url = f"https://storagetransfer.googleapis.com/v1/{job_name}:run" + run_payload = {"projectId": project} + run_response = await self.async_client.post( + run_url, json=run_payload, headers=headers + ) + + if run_response.status_code != 200: + raise RuntimeError( + f"Failed to run Storage Transfer Job {job_name}:" + f" {run_response.status_code} - {run_response.text}" + ) + + operation_name = run_response.json()["name"] + return operation_name + + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls Storage Transfer Service operation status.""" + token, _ = await self._get_token_and_project() + url = f"https://storagetransfer.googleapis.com/v1/{operation_name}" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + response = await self.async_client.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to poll Storage Transfer operation: {response.status_code} -" + f" {response.text}" + ) + + data = response.json() + done = data.get("done", False) + + if not done: + metadata = data.get("metadata", {}) + counters = metadata.get("counters", {}) + bytes_transferred = int(counters.get("bytesTransferredToSink", 0)) + bytes_found = int(counters.get("bytesFoundToTransfer", 0)) + return Result( + status=OperationStatus.IN_PROGRESS, + detail_info={ + "bytes_copied": bytes_transferred, + "total_bytes": bytes_found, + }, + ) + + if "error" in data: + return Result( + status=OperationStatus.FAILED, + detail_info={"error": data["error"]}, + ) + + metadata = data.get("metadata", {}) + op_status = metadata.get("status") + + if op_status == "SUCCESS": + return Result( + status=OperationStatus.SUCCESS, + detail_info={}, + ) + else: + error_msg = f"STS Operation ended with status: {op_status}" + return Result( + status=OperationStatus.FAILED, + detail_info={"error": error_msg}, + ) + + +class GcpLustreBaseClient(GCPStorageClient): + """Base client interface to interact with GCP Managed Lustre API via REST.""" + + def __init__( + self, + project: str | None = None, + location: str | None = None, + instance: str | None = None, + service_account: str | None = None, + ): + location = location or os.environ.get("CTS_LUSTRE_LOCATION") + instance = instance or os.environ.get("CTS_LUSTRE_INSTANCE") + + if not location or not instance: + raise ValueError("Lustre location and instance must be specified.") + + super().__init__( + project=project, + location=location, + instance=instance, + service_account=service_account, + ) + + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls operation status and returns a Result object.""" + del context # Unused for Lustre + token, _ = await self._get_token_and_project() + url = f"https://lustre.googleapis.com/v1/{operation_name}" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + response = await self.async_client.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to poll operation: {response.status_code} - {response.text}" + ) + data = response.json() + done = data.get("done", False) + if done: + if "error" in data: + return Result( + status=OperationStatus.FAILED, + detail_info={"error": data["error"]}, + ) + else: + return Result( + status=OperationStatus.SUCCESS, + detail_info=data.get("response", {}), + ) + else: + return Result( + status=OperationStatus.IN_PROGRESS, + detail_info=data.get("metadata", {}), + ) + + +class GcsToLustreClient(GcpLustreBaseClient): + """Client implementation to trigger GCS-to-Lustre imports.""" + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers import from GCS to Lustre and returns the Operation name.""" + token, project = await self._get_token_and_project() + url = ( + f"https://lustre.googleapis.com/v1/projects/{project}" + f"/locations/{self.location}/instances/{self.instance}:importData" + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + payload = { + "gcsPath": {"uri": source_path}, + "lustrePath": {"path": destination_path}, + "requestId": request_id, + } + response = await self.async_client.post(url, json=payload, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to trigger import: {response.status_code} - {response.text}" + ) + return response.json()["name"] + + +class LustreToGcsClient(GcpLustreBaseClient): + """Client implementation to trigger Lustre-to-GCS exports.""" + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers export from Lustre to GCS and returns the Operation name.""" + token, project = await self._get_token_and_project() + url = ( + f"https://lustre.googleapis.com/v1/projects/{project}" + f"/locations/{self.location}/instances/{self.instance}:exportData" + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + payload = { + "lustrePath": {"path": source_path}, + "gcsPath": {"uri": destination_path}, + "requestId": request_id, + } + response = await self.async_client.post(url, json=payload, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to trigger export: {response.status_code} - {response.text}" + ) + return response.json()["name"] diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py new file mode 100644 index 0000000000..99caea1778 --- /dev/null +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py @@ -0,0 +1,281 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for GCP Storage Clients.""" + +import os +import unittest +from unittest import mock +import httpx +from orbax.checkpoint.experimental.tiering_service import gcp_storage_client + + +class GCPStorageClientTest(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + super().setUp() + self.mock_creds = mock.MagicMock() + self.mock_creds.valid = True + self.mock_creds.token = "dummy_auth_token" + self.mock_default_auth = mock.patch( + "google.auth.default", return_value=(self.mock_creds, "dummy-project") + ) + self.mock_default_auth.start() + + # Mock httpx.AsyncClient + self.mock_client = mock.AsyncMock(spec=httpx.AsyncClient) + self.mock_client_patcher = mock.patch( + "httpx.AsyncClient", return_value=self.mock_client + ) + self.mock_client_patcher.start() + + def tearDown(self): + self.mock_default_auth.stop() + self.mock_client_patcher.stop() + super().tearDown() + + async def test_gcs_to_gcs_trigger_copy_success(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + # 1. Mock the first POST call to create transfer job + mock_post_resp_1 = mock.MagicMock(spec=httpx.Response) + mock_post_resp_1.status_code = 200 + mock_post_resp_1.json.return_value = {"name": "transferJobs/job-123"} + + # 2. Mock the second POST call to run transfer job + mock_post_resp_2 = mock.MagicMock(spec=httpx.Response) + mock_post_resp_2.status_code = 200 + mock_post_resp_2.json.return_value = {"name": "transferOperations/op-456"} + + self.mock_client.post.side_effect = [mock_post_resp_1, mock_post_resp_2] + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path/to/src", + destination_path="gs://dest-bucket/path/to/dest", + ) + + self.assertEqual(op_name, "transferOperations/op-456") + self.assertEqual(self.mock_client.post.call_count, 2) + + async def test_gcs_to_gcs_trigger_copy_sts_fail(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 400 + mock_post_resp.text = "Invalid arguments" + self.mock_client.post.return_value = mock_post_resp + + with self.assertRaises(RuntimeError) as ctx: + await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path", + destination_path="gs://dest-bucket/path", + ) + self.assertIn("Failed to create Storage Transfer Job", str(ctx.exception)) + + async def test_gcs_to_gcs_poll_operation_in_progress(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": False, + "metadata": { + "counters": { + "bytesTransferredToSink": "500", + "bytesFoundToTransfer": "1000", + } + }, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual( + result.status, gcp_storage_client.OperationStatus.IN_PROGRESS + ) + self.assertEqual(result.detail_info["bytes_copied"], 500) + self.assertEqual(result.detail_info["total_bytes"], 1000) + + async def test_gcs_to_gcs_poll_operation_success(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "metadata": {"status": "SUCCESS"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.SUCCESS) + + async def test_gcs_to_gcs_poll_operation_failed(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "metadata": {"status": "FAILED"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.FAILED) + self.assertIn("error", result.detail_info) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_gcs_to_lustre_trigger_copy(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 200 + mock_post_resp.json.return_value = {"name": "operations/import-123"} + self.mock_client.post.return_value = mock_post_resp + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path", + destination_path="/lustre/path", + ) + self.assertEqual(op_name, "operations/import-123") + self.mock_client.post.assert_called_once() + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_to_gcs_trigger_copy(self): + client = gcp_storage_client.LustreToGcsClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 200 + mock_post_resp.json.return_value = {"name": "operations/export-123"} + self.mock_client.post.return_value = mock_post_resp + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="/lustre/path", + destination_path="gs://dest-bucket/path", + ) + self.assertEqual(op_name, "operations/export-123") + self.mock_client.post.assert_called_once() + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_done_success(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "response": {"some_metadata": "val"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.SUCCESS) + self.assertEqual(result.detail_info, {"some_metadata": "val"}) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_done_fail(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "error": {"message": "import failed"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.FAILED) + self.assertEqual(result.detail_info["error"], {"message": "import failed"}) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_in_progress(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": False, + "metadata": {"percent_complete": 42}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual( + result.status, gcp_storage_client.OperationStatus.IN_PROGRESS + ) + self.assertEqual(result.detail_info, {"percent_complete": 42}) + + @mock.patch("google.auth.impersonated_credentials.Credentials") + async def test_service_account_token_impersonation( + self, mock_impersonated_creds_class + ): + mock_imp_creds = mock.MagicMock() + mock_imp_creds.valid = True + mock_imp_creds.token = "impersonated_token" + mock_impersonated_creds_class.return_value = mock_imp_creds + + client = gcp_storage_client.GcsToGcsClient( + project="test-project", + service_account="sa@test.iam.gserviceaccount.com", + ) + + token, project = await client._get_token_and_project() + self.assertEqual(token, "impersonated_token") + self.assertEqual(project, "test-project") + + # Verify that impersonated credentials were constructed correctly + mock_impersonated_creds_class.assert_called_once_with( + source_credentials=self.mock_creds, + target_principal="sa@test.iam.gserviceaccount.com", + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py b/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py index e882fc1732..c53e78f4de 100644 --- a/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py +++ b/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py @@ -56,7 +56,7 @@ class CheckpointVersion(enum.Enum): ORBAX_CHECKPOINT_INDICATOR_FILE = "orbax.checkpoint" CHECKPOINT_METADATA = "_CHECKPOINT_METADATA" -_OCDBT_MANIFEST_FILE = "ocdbt.manifest" +_OCDBT_MANIFEST_FILE = "manifest.ocdbt" _ZARRAY_FILE = ".zarray" diff --git a/checkpoint/pyproject.toml b/checkpoint/pyproject.toml index f4490abe08..49c3e116d3 100644 --- a/checkpoint/pyproject.toml +++ b/checkpoint/pyproject.toml @@ -81,6 +81,7 @@ tiering_service = [ 'fire', 'greenlet', 'grpcio-tools>=1.80.0', + 'httpx', 'pysqlite3', 'pytimeparse', 'sqlalchemy>=1.4.0', diff --git a/docs/guides/checkpoint/v1/async_checkpointing.ipynb b/docs/guides/checkpoint/v1/async_checkpointing.ipynb index da315d7c10..ce11493998 100644 --- a/docs/guides/checkpoint/v1/async_checkpointing.ipynb +++ b/docs/guides/checkpoint/v1/async_checkpointing.ipynb @@ -139,7 +139,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/sync_checkpoint" + "print(sorted([p.name for p in path.iterdir()]))" ], "outputs": [], "execution_count": 9 @@ -175,7 +175,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/async_checkpoint" + "print(sorted([p.name for p in path.iterdir()]))" ], "outputs": [], "execution_count": 11 diff --git a/docs/guides/checkpoint/v1/checkpoint_format.ipynb b/docs/guides/checkpoint/v1/checkpoint_format.ipynb index e5851d0ed4..949b760bcb 100644 --- a/docs/guides/checkpoint/v1/checkpoint_format.ipynb +++ b/docs/guides/checkpoint/v1/checkpoint_format.ipynb @@ -321,7 +321,7 @@ }, "cell_type": "code", "source": [ - "!ls {directory / 'ckpt-0'}" + "print(sorted([p.name for p in (directory / 'ckpt-0').iterdir()]))" ], "outputs": [], "execution_count": null diff --git a/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb b/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb index 165de5b4c2..f0553319cb 100644 --- a/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb +++ b/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb @@ -1,1131 +1,1223 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "GYCcRRZas1PS" - }, - "source": [ - "# Working with PyTree Checkpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iJwUDQVA7GIV" - }, - "source": [ - "A [PyTree](https://jax.readthedocs.io/en/latest/pytrees.html) is the most common way of representing a training state in JAX. While Orbax is designed to be as generic as possible, and provides customization options for all manner of checkpointable objects, PyTrees naturally have pride of place. Furthermore, the standard object used to represent large, sharded arrays is the {py:class}`jax.Array `. This, too, has extensive first-class support." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Vfj9mDHsb7QF" - }, - "source": [ - "## Exclusive APIs to checkpoint PyTrees" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "e3JtbxFhVq_6" - }, - "source": [ - "The following APIs can be used to checkpoint PyTrees exclusively.\n", - "\n", - "To save:\n", - "\n", - "* `ocp.save(...)`\n", - "* `ocp.save_async(...)`\n", - "* `training.Checkpointer.save(...)`\n", - "* `training.Checkpointer.save_async(...)`\n", - "\n", - "To load:\n", - "* `ocp.load(...)`\n", - "* `ocp.load_async(...)`\n", - "* `training.Checkpointer.load(...)`\n", - "* `training.Checkpointer.load_async(...)`\n", - "\n", - "Of course, the `save_checkpointables(...)` and `load_checkpointables(...)`\n", - "flavor APIs can be used to save a PyTree too." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ej99peApVq_7" - }, - "source": [ - "Let's setup a PyTree of jax.Array to play with these APIs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6NknwqBeVq_7" - }, - "outputs": [], - "source": [ - "from etils import epath\n", - "import jax\n", - "import numpy as np\n", - "import orbax.checkpoint.experimental.v1 as ocp" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "b0j19QxuVq_7" - }, - "outputs": [], - "source": [ - "sharding = jax.sharding.NamedSharding(\n", - " jax.sharding.Mesh(jax.devices(), ('model',)),\n", - " jax.sharding.PartitionSpec(\n", - " 'model',\n", - " ),\n", - ")\n", - "create_sharded_array = lambda x: jax.device_put(x, sharding)\n", - "pytree = {\n", - " 'a': np.arange(16, dtype=np.int32),\n", - " 'b': np.ones(16, dtype=np.int32),\n", - "}\n", - "pytree = jax.tree_util.tree_map(create_sharded_array, pytree)\n", - "pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Kf_OCzGbVq_8" - }, - "source": [ - "## Basic Checkpointing" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sU2YIn10Vq_8" - }, - "source": [ - "Let's use `ocp.save_*`/`ocp.load_*` to work with the pytree created earlier." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Szq9yBPhVq_8" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/basic/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "# Simple save using default options:\n", - "ocp.save(path, pytree)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Kr6nrYg3QR73" - }, - "source": [ - "We can easily restore using the following snippet.\n", - "\n", - "**Warning: do not use for production-sensitive cases.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "CtEMu9BBRHYW" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path)\n", - "loaded" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "F-CPPqeaRRks" - }, - "source": [ - "It is not recommended to load this way for production-sensitive cases because the user cannot make any guarantees about what they are loading. If the shapes of some arrays have changed in the model since the checkpoint was saved, errors can be seen when attempting to create the model. If the device topology has changed, we will see errors when attempting to place arrays on devices.\n", - "\n", - "It is therefore recommended that users always specify an **abstract pytree** when loading." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R1Sbx3DYQMpS" - }, - "source": [ - "### Understanding Abstract Trees and Leaves" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4f8Ze-q3R5XE" - }, - "source": [ - "An **abstract PyTree** is just a normal PyTree, but with abstract leaves. An **abstract leaf** is a cheap representation of a leaf type (such as an array) that contains only metadata, and does not represent the real values. (Contrast with a *concrete* PyTree, which contains real data in the form of large arrays, and other types.)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PVyCvC7VaisK" - }, - "source": [ - "Let's create an abstract PyTree matching the structure of the PyTree we originally saved." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "u5JJaHrYVq_7" - }, - "outputs": [], - "source": [ - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", - " 'b': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", - "}\n", - "abstract_state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "0GCij-iuVq_8" - }, - "outputs": [], - "source": [ - "# Load using abstract_state.\n", - "loaded = ocp.load(path, abstract_state)\n", - "loaded" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TjOlZAkTVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tQ6L_wtnVq_8" - }, - "source": [ - "The `metadata` method returns a `CheckpointMetadata` object with a number of properties, but the core `metadata` property is just an abstract PyTree. This can also be used for loading as shown below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_sJOi2I3Vq_8" - }, - "outputs": [], - "source": [ - "metadata = ocp.metadata(path).metadata\n", - "metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "1SQMyU44Vq_8" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path, metadata)\n", - "loaded" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jGTdk0tVVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "U7URSieWVq_8" - }, - "source": [ - "Note that it is also valid to provide a \"concrete\" PyTree for loading rather than an \"abstract\" target, since by definition, the concrete leaves contain all the same properties provided by the abstract leaves.\n", - "\n", - "However, this requires that you fully initialize the target train state\n", - "before loading from the checkpoint, which is inefficient or impractical for real use cases. It is better practice to only initialize metadata (either by manually creating `jax.ShapeDtypeStruct`s or using `jax.eval_shape`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3JfwbnA_Vq_8" - }, - "outputs": [], - "source": [ - "ocp.load(path, pytree)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "wjpnDhAqb6t0" - }, - "source": [ - "### Standard Leaf Types" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "p6yM2hBGdNXZ" - }, - "source": [ - "The following standard leaf types are supported by Orbax by default. Each concrete leaf type has a corresponding abstract leaf type. Most abstract types are implemented as `Protocol`'s, so that any object implementing the required properties can be accepted as a valid abstract type." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G0oCkcDZb8g_" - }, - "source": [ - "| `Leaf` Type | `AbstractLeaf` Type | Properties |\n", - ":------- | :-------- | :-------- |\n", - "|`jax.Array`|`AbstractShardedArray` (`jax.ShapeDtypeStruct`) |`shape`, `dtype`, `sharding`|\n", - "|`np.ndarray`|`AbstractArray` (`np.ndarray`) |`shape`, `dtype`|\n", - "|`int`|`int`| |\n", - "|`float`|`float`| |\n", - "|`bytes`|`bytes`| |\n", - "|`str`|`str`| |" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "01zrVpfcdq7m" - }, - "source": [ - "`None` is always a valid abstract leaf; it serves as an indication that the leaf should be restored using metadata stored in the checkpoint.\n", - "\n", - "`Type[AbstractLeaf]` is also always a valid abstract leaf; it again serves as an indication that the leaf should be restored using the metadata, but with the additional constraint to load as the indicated type. For example, instead of specifying `jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=...)`, it is sufficient to pass `jax.ShapeDtypeStruct`. Similarly, instead of passing `0` to restore as an `int`, the type itself may be passed." - ] - }, - { - "metadata": { - "id": "nRn_2IgV09xT" - }, - "cell_type": "markdown", - "source": [ - "To summarize, here are the ways you can load a PyTree using abstract leaves, with the way we most recommend at the top, and the way we least recommend at the bottom.\n", - "\n", - "**1. Fully-specified abstract values**\n", - "\n", - "This provides the most loading validations and requires the least amount of\n", - "unnecessary metadata reads.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=jax.sharding.NamedSharding(...))\n", - "}\n", - "```\n", - "\n", - "**2. Only types specified**\n", - "\n", - "This guarantees that each leaf will be loaded with the indicated type, but metadata\n", - "will be used to restore specific properties for each leaf.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct,\n", - " 'b': int,\n", - " 'c': np.ndarray,\n", - "}\n", - "```\n", - "\n", - "**3. `None` specified (per-leaf)**\n", - "\n", - "This is essentially the same as (2), but metadata will also be used to decide\n", - "which type each leaf should be loaded as.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': None,\n", - " 'b': None,\n", - "}\n", - "```\n", - "\n", - "**4. `None` specified**\n", - "\n", - "This loads the PyTree structure without any checks, and can lead to errors later\n", - "in your code if the checkpoint does not have the structure you expect.\n", - "\n", - "```\n", - "abstract_state = None\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UR34KTImVq_8" - }, - "source": [ - "### Customizing Loaded Properties for Arrays" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TErvR-zDVq_8" - }, - "source": [ - "#### Array dtype" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZDSJhefZVq_8" - }, - "outputs": [], - "source": [ - "def set_loading_dtype(x: jax.ShapeDtypeStruct) -> jax.ShapeDtypeStruct:\n", - " return x.update(dtype=np.int16)\n", - "\n", - "\n", - "cast_dtype_abstract_state = jax.tree_util.tree_map(\n", - " set_loading_dtype, abstract_state\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "scZbqSQ4Vq_8" - }, - "outputs": [], - "source": [ - "ocp.load(path, cast_dtype_abstract_state)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rhkTT9wMVq_8" - }, - "source": [ - "#### Change sharding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NLjuYubRVq_8" - }, - "source": [ - "**NOTE: This can often be a particularly sharp edge.**\n", - "\n", - "Sharding commonly needs to be changed when loading a checkpoint saved on one topology to a different topology.\n", - "\n", - "**If changing topologies, you MUST specify sharding when loading.**\n", - "\n", - "Unless you are loading on the exact same topology, Orbax does not make any decisions about shardings on your behalf. If you have the exact same topology,\n", - "however, it is possible to avoid specifying the sharding when loading. This is demonstrated below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "atPqix9IVq_8" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Pap_vpN9Vq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "T_T3QjRLVq_8" - }, - "source": [ - "In the example below, we alter the sharding while loading." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "J0tvS901Vq_8" - }, - "outputs": [], - "source": [ - "sharding = jax.sharding.NamedSharding(\n", - " jax.sharding.Mesh(jax.devices(), ('x',)),\n", - " jax.sharding.PartitionSpec(),\n", - ")\n", - "\n", - "\n", - "def set_sharding(x: jax.ShapeDtypeStruct) -> jax.ShapeDtypeStruct:\n", - " return x.update(sharding=sharding)\n", - "\n", - "\n", - "change_sharding_abstract_state = jax.tree_util.tree_map(\n", - " set_sharding, abstract_state\n", - ")\n", - "loaded = ocp.load(path, change_sharding_abstract_state)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "M84bx2smVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zA_Vdck2Vq_8" - }, - "source": [ - "We can use pytree metadata instead of the abstract pytree." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cAN_1Xj_Vq_8" - }, - "outputs": [], - "source": [ - "metadata = ocp.metadata(path).metadata\n", - "change_sharding_metadata = jax.tree_util.tree_map(\n", - " lambda x: jax.ShapeDtypeStruct(shape=x.shape, dtype=x.dtype, sharding=sharding), metadata\n", - ")\n", - "loaded = ocp.load(path, change_sharding_metadata)\n", - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sIj3AYAidwvh" - }, - "source": [ - "#### Change leaf type" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "01BAYxDGez1T" - }, - "source": [ - "The abstract leaf type dictates the loaded type for each leaf. If we save a value as a `jax.Array` but provide an abstract leaf without the required `sharding` property, Orbax will load as `np.ndarray`. Similarly, we can save as an `int` and load as a `float` if we specify `float` as the abstract leaf." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cDdy1bk0dzDk" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/change-type/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "pytree_with_scalars = {\n", - " 'a': np.asarray(12),\n", - " 'b': 13.5,\n", - " 'c': create_sharded_array(np.arange(8)),\n", - "}\n", - "ocp.save(path, pytree_with_scalars)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "UxWeE0E8eg52" - }, - "outputs": [], - "source": [ - "abstract_state_with_scalars = {\n", - " 'a': float,\n", - " 'b': int,\n", - " 'c': np.empty((8,)),\n", - "}\n", - "ocp.load(path, abstract_state_with_scalars)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LBSbal0hVq_8" - }, - "source": [ - "### Partial Loading\n", - "\n", - "You may wish to load part of a PyTree contained within a saved checkpoint. For example, consider the following item:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "96Zk1nrzVq_8" - }, - "outputs": [], - "source": [ - "original_item = {\n", - " 'params': {\n", - " 'layer1': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " 'layer2': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " },\n", - " 'opt_state': [np.arange(8), np.arange(8)],\n", - " 'step': 101,\n", - "}\n", - "\n", - "path = epath.Path('/tmp/checkpointing-pytrees/partial/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "ocp.save(path / '1', original_item)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "g9q0W8GoVq_8" - }, - "source": [ - "If we want to load only a subset of PyTree nodes (`params.layer2` and `step`, for example), we can use Placeholder values." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "s9WFY0vuVq_8" - }, - "source": [ - "#### Placeholder\n", - "\n", - "To load part of a PyTree item, we can specify which nodes to ignore during loading by using `...` (`ocp.PLACEHOLDER`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NHreKKn8Vq_9" - }, - "outputs": [], - "source": [ - "reference_item = {\n", - " 'params': {\n", - " 'layer1': {\n", - " 'kernel': ...,\n", - " 'bias': ...,\n", - " },\n", - " 'layer2': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " },\n", - " 'opt_state': [..., ...],\n", - " 'step': 101,\n", - "}\n", - "\n", - "ocp.load(path / '1', reference_item)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MH3hBSxPVq_9" - }, - "source": [ - "## Advanced Customizations" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nZcttkVHVq_9" - }, - "source": [ - "`ocp.Context` enables more customizations.\n", - "\n", - "For customized save/load behavior, these APIs should be invoked within a `ocp.Context`\n", - "instance, which in turn can be configured with a number of options like Saving, Loading,\n", - "FileOptions etc.\n", - "\n", - "The usage pattern is as follows:\n", - "```\n", - "with ocp.Context(\n", - " pytree_options=PyTreeOptions(...),\n", - " file_options=FileOptions(...),\n", - "):\n", - " ocp.save(path, pytree)\n", - "```\n", - "\n", - "Let's explore few examples. Please also take a look at API Reference for specific option details." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CY0cRK32Vq_9" - }, - "source": [ - "### Saving" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DpuH7FMHVq_9" - }, - "source": [ - "#### Customizing Array dtype" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bq3Hj3-aVq_9" - }, - "source": [ - "we can customize the on-disk type used to save individual arrays. First, let's save and load as normal." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "FZt44Li_Vq_9" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/advanced/')\n", - "path.rmtree(missing_ok=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "68BXPuRcVq_9" - }, - "outputs": [], - "source": [ - "ocp.save(path / '1', pytree)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LcXASaTWVq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '1')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "WnCRCzlOVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yhb3Py_9Vq_9" - }, - "source": [ - "Now, let's set the dtype of selective array when saving." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9VpoTnHmVq_9" - }, - "outputs": [], - "source": [ - "def scoped_storage_options_creator(keypath, value):\n", - " del value\n", - " last_key = keypath[-1]\n", - " # Override 'a' to int16\n", - " if isinstance(last_key, jax.tree_util.GetAttrKey) and last_key.name == 'a':\n", - " return ocp.options.ArrayOptions.Saving.StorageOptions(\n", - " dtype=np.dtype(np.int16)\n", - " )\n", - " # Return None to use global default storage_options for other leaves\n", - " return None\n", - "\n", - "ctx = ocp.Context()\n", - "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", - "\n", - "with ctx:\n", - " ocp.save(path / '2', pytree, overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "mHh_ucu_Vq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '2')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sDHCc2s4Vq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P58IxTVHVq_9" - }, - "source": [ - "Now, let's set the dtype of all arrays when saving." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "W-k7jRWFVq_9" - }, - "outputs": [], - "source": [ - "scoped_storage_options_creator = (\n", - " lambda k, v: ocp.options.ArrayOptions.Saving.StorageOptions(\n", - " dtype=np.dtype(np.int16)\n", - " )\n", - ")\n", - "\n", - "ctx = ocp.Context()\n", - "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", - "\n", - "with ctx:\n", - " ocp.save(path / '3', pytree, overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Cgs_vBOWVq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '3')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9acbLs7CVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XZZq8Nxelxwn" - }, - "source": [ - "#### High Throughput with `ocdbt` option" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GhMBAVqDn1s2" - }, - "source": [ - "For high throughput and avoid creating separate subdirectories for each leaf, enable `use_ocdbt`. Please note that it is enabled by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "W_FD_od_mb8z" - }, - "outputs": [], - "source": [ - "ctx = ocp.Context()\n", - "ctx.array.saving.use_ocdbt = True\n", - "with ctx:\n", - " ocp.save(path / '4', pytree, overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G6Cm0PYcoIF4" - }, - "source": [ - "A checkpoint created with this option enabled can be identified by presence of files `manifest.ocdbt` and subdirs like `ocdbt.process_*`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "IA4tTKDGoGlf" - }, - "outputs": [], - "source": [ - "!ls /tmp/checkpointing-pytrees/advanced/4/pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GqA8Kdk6orZc" - }, - "source": [ - "However, for use cases like large stacked models, disabling this option may be more efficient." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qJvm7QxCpUOT" - }, - "outputs": [], - "source": [ - "ctx = ocp.Context()\n", - "ctx.array.saving.use_ocdbt = False\n", - "with ctx:\n", - " ocp.save(path / '5', pytree, overwrite=True)\n", - "\n", - "!ls /tmp/checkpointing-pytrees/advanced/5/pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-OZF-uwBpb8y" - }, - "source": [ - "Please note how each leaf is written in its own subdir when `use_ocdbt=False`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_AnAJYT8Vq_9" - }, - "source": [ - "### Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5aTJ6eUJVq_9" - }, - "source": [ - "#### Pad / truncate shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2Qg0HbH3Vq_9" - }, - "source": [ - "Ordinarily, specifying a target array with a different shape than in the\n", - "checkpoint results in an error." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p1I0sfNEVq_9" - }, - "outputs": [], - "source": [ - "# Original shape.\n", - "loaded = ocp.load(path / '1')\n", - "\n", - "(loaded['a'].shape, loaded['b'].shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2X8mMPBkVq_9" - }, - "outputs": [], - "source": [ - "different_shape_abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(\n", - " shape=(8,),\n", - " dtype=abstract_state['a'].dtype,\n", - " sharding=abstract_state['a'].sharding,\n", - " ),\n", - " 'b': jax.ShapeDtypeStruct(\n", - " shape=(32,),\n", - " dtype=abstract_state['b'].dtype,\n", - " sharding=abstract_state['b'].sharding,\n", - " ),\n", - "}\n", - "\n", - "try:\n", - " ocp.load(path / '1', different_shape_abstract_state)\n", - "except BaseException as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HMPpVIiLVq_9" - }, - "source": [ - "We can pad or truncate arrays as they are loaded by specifying `enable_padding_and_truncation=True`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Rbu-q1upVq_9" - }, - "outputs": [], - "source": [ - "\n", - "ctx = ocp.Context()\n", - "ctx.array.loading.enable_padding_and_truncation = True\n", - "with ctx:\n", - " loaded = ocp.load(path / '1', different_shape_abstract_state)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HP6gm2HEVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].shape, loaded['b'].shape)" - ] - } - ], - "metadata": { - "colab": { - "last_runtime": { - "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", - "kind": "private" - }, - "private_outputs": true, - "provenance": [ - { - "file_id": "1QNxBBBN16Br9Xj-a7LvtJzJWjOBhjFps", - "timestamp": 1686159333109 - } - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" + "cells": [ + { + "id": "6914e6f6", + "cell_type": "markdown", + "source": [ + "# Working with PyTree Checkpoints" + ], + "metadata": { + "id": "GYCcRRZas1PS" + }, + "execution_count": null + }, + { + "id": "c510117b", + "cell_type": "markdown", + "source": [ + "A [PyTree](https://jax.readthedocs.io/en/latest/pytrees.html) is the most common way of representing a training state in JAX. While Orbax is designed to be as generic as possible, and provides customization options for all manner of checkpointable objects, PyTrees naturally have pride of place. Furthermore, the standard object used to represent large, sharded arrays is the {py:class}`jax.Array \u003cjax.Array\u003e`. This, too, has extensive first-class support." + ], + "metadata": { + "id": "iJwUDQVA7GIV" + }, + "execution_count": null + }, + { + "id": "60fd24e1", + "cell_type": "markdown", + "source": [ + "## Exclusive APIs to checkpoint PyTrees" + ], + "metadata": { + "id": "Vfj9mDHsb7QF" + }, + "execution_count": null + }, + { + "id": "af8bc7b5", + "cell_type": "markdown", + "source": [ + "The following APIs can be used to checkpoint PyTrees exclusively.\n", + "\n", + "To save:\n", + "\n", + "* `ocp.save(...)`\n", + "* `ocp.save_async(...)`\n", + "* `training.Checkpointer.save(...)`\n", + "* `training.Checkpointer.save_async(...)`\n", + "\n", + "To load:\n", + "* `ocp.load(...)`\n", + "* `ocp.load_async(...)`\n", + "* `training.Checkpointer.load(...)`\n", + "* `training.Checkpointer.load_async(...)`\n", + "\n", + "Of course, the `save_checkpointables(...)` and `load_checkpointables(...)`\n", + "flavor APIs can be used to save a PyTree too." + ], + "metadata": { + "id": "e3JtbxFhVq_6" + }, + "execution_count": null + }, + { + "id": "c2677229", + "cell_type": "markdown", + "source": [ + "Let's setup a PyTree of jax.Array to play with these APIs." + ], + "metadata": { + "id": "ej99peApVq_7" + }, + "execution_count": null + }, + { + "id": "5bb3f520", + "cell_type": "code", + "source": [ + "from etils import epath\n", + "import jax\n", + "import numpy as np\n", + "import orbax.checkpoint.experimental.v1 as ocp" + ], + "metadata": { + "id": "6NknwqBeVq_7" + }, + "execution_count": null + }, + { + "id": "87e1bb5b", + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('model',)),\n", + " jax.sharding.PartitionSpec(\n", + " 'model',\n", + " ),\n", + ")\n", + "create_sharded_array = lambda x: jax.device_put(x, sharding)\n", + "pytree = {\n", + " 'a': np.arange(16, dtype=np.int32),\n", + " 'b': np.ones(16, dtype=np.int32),\n", + "}\n", + "pytree = jax.tree_util.tree_map(create_sharded_array, pytree)\n", + "pytree" + ], + "metadata": { + "id": "b0j19QxuVq_7" + }, + "execution_count": null + }, + { + "id": "76928efc", + "cell_type": "markdown", + "source": [ + "## Basic Checkpointing" + ], + "metadata": { + "id": "Kf_OCzGbVq_8" + }, + "execution_count": null + }, + { + "id": "345282a3", + "cell_type": "markdown", + "source": [ + "Let's use `ocp.save_*`/`ocp.load_*` to work with the pytree created earlier." + ], + "metadata": { + "id": "sU2YIn10Vq_8" + }, + "execution_count": null + }, + { + "id": "35052c1d", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/basic/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "# Simple save using default options:\n", + "ocp.save(path, pytree)" + ], + "metadata": { + "id": "Szq9yBPhVq_8" + }, + "execution_count": null + }, + { + "id": "61ba3a26", + "cell_type": "markdown", + "source": [ + "We can easily restore using the following snippet.\n", + "\n", + "**Warning: do not use for production-sensitive cases.**" + ], + "metadata": { + "id": "Kr6nrYg3QR73" + }, + "execution_count": null + }, + { + "id": "7abc5fef", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path)\n", + "loaded" + ], + "metadata": { + "id": "CtEMu9BBRHYW" + }, + "execution_count": null + }, + { + "id": "0dc43b69", + "cell_type": "markdown", + "source": [ + "It is not recommended to load this way for production-sensitive cases because the user cannot make any guarantees about what they are loading. If the shapes of some arrays have changed in the model since the checkpoint was saved, errors can be seen when attempting to create the model. If the device topology has changed, we will see errors when attempting to place arrays on devices.\n", + "\n", + "It is therefore recommended that users always specify an **abstract pytree** when loading." + ], + "metadata": { + "id": "F-CPPqeaRRks" + }, + "execution_count": null + }, + { + "id": "99e12640", + "cell_type": "markdown", + "source": [ + "### Understanding Abstract Trees and Leaves" + ], + "metadata": { + "id": "R1Sbx3DYQMpS" + }, + "execution_count": null + }, + { + "id": "5a6792e2", + "cell_type": "markdown", + "source": [ + "An **abstract PyTree** is just a normal PyTree, but with abstract leaves. An **abstract leaf** is a cheap representation of a leaf type (such as an array) that contains only metadata, and does not represent the real values. (Contrast with a *concrete* PyTree, which contains real data in the form of large arrays, and other types.)" + ], + "metadata": { + "id": "4f8Ze-q3R5XE" + }, + "execution_count": null + }, + { + "id": "0353949d", + "cell_type": "markdown", + "source": [ + "Let's create an abstract PyTree matching the structure of the PyTree we originally saved." + ], + "metadata": { + "id": "PVyCvC7VaisK" + }, + "execution_count": null + }, + { + "id": "95dc146f", + "cell_type": "code", + "source": [ + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", + " 'b': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", + "}\n", + "abstract_state" + ], + "metadata": { + "id": "u5JJaHrYVq_7" + }, + "execution_count": null + }, + { + "id": "d6332cfd", + "cell_type": "code", + "source": [ + "# Load using abstract_state.\n", + "loaded = ocp.load(path, abstract_state)\n", + "loaded" + ], + "metadata": { + "id": "0GCij-iuVq_8" + }, + "execution_count": null + }, + { + "id": "299c34c5", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "TjOlZAkTVq_8" + }, + "execution_count": null + }, + { + "id": "8917eb60", + "cell_type": "markdown", + "source": [ + "The `metadata` method returns a `CheckpointMetadata` object with a number of properties, but the core `metadata` property is just an abstract PyTree. This can also be used for loading as shown below." + ], + "metadata": { + "id": "tQ6L_wtnVq_8" + }, + "execution_count": null + }, + { + "id": "0664d045", + "cell_type": "code", + "source": [ + "metadata = ocp.metadata(path).metadata\n", + "metadata" + ], + "metadata": { + "id": "_sJOi2I3Vq_8" + }, + "execution_count": null + }, + { + "id": "205df646", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path, metadata)\n", + "loaded" + ], + "metadata": { + "id": "1SQMyU44Vq_8" + }, + "execution_count": null + }, + { + "id": "37966ae1", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "jGTdk0tVVq_8" + }, + "execution_count": null + }, + { + "id": "b2e2b86a", + "cell_type": "markdown", + "source": [ + "Note that it is also valid to provide a \"concrete\" PyTree for loading rather than an \"abstract\" target, since by definition, the concrete leaves contain all the same properties provided by the abstract leaves.\n", + "\n", + "However, this requires that you fully initialize the target train state\n", + "before loading from the checkpoint, which is inefficient or impractical for real use cases. It is better practice to only initialize metadata (either by manually creating `jax.ShapeDtypeStruct`s or using `jax.eval_shape`)." + ], + "metadata": { + "id": "U7URSieWVq_8" + }, + "execution_count": null + }, + { + "id": "4fb8d9da", + "cell_type": "code", + "source": [ + "ocp.load(path, pytree)" + ], + "metadata": { + "id": "3JfwbnA_Vq_8" + }, + "execution_count": null + }, + { + "id": "bd7690cf", + "cell_type": "markdown", + "source": [ + "### Standard Leaf Types" + ], + "metadata": { + "id": "wjpnDhAqb6t0" + }, + "execution_count": null + }, + { + "id": "5968ec28", + "cell_type": "markdown", + "source": [ + "The following standard leaf types are supported by Orbax by default. Each concrete leaf type has a corresponding abstract leaf type. Most abstract types are implemented as `Protocol`'s, so that any object implementing the required properties can be accepted as a valid abstract type." + ], + "metadata": { + "id": "p6yM2hBGdNXZ" + }, + "execution_count": null + }, + { + "id": "4573a1ae", + "cell_type": "markdown", + "source": [ + "| `Leaf` Type | `AbstractLeaf` Type | Properties |\n", + ":------- | :-------- | :-------- |\n", + "|`jax.Array`|`AbstractShardedArray` (`jax.ShapeDtypeStruct`) |`shape`, `dtype`, `sharding`|\n", + "|`np.ndarray`|`AbstractArray` (`np.ndarray`) |`shape`, `dtype`|\n", + "|`int`|`int`| |\n", + "|`float`|`float`| |\n", + "|`bytes`|`bytes`| |\n", + "|`str`|`str`| |" + ], + "metadata": { + "id": "G0oCkcDZb8g_" + }, + "execution_count": null + }, + { + "id": "da09a3b7", + "cell_type": "markdown", + "source": [ + "`None` is always a valid abstract leaf; it serves as an indication that the leaf should be restored using metadata stored in the checkpoint.\n", + "\n", + "`Type[AbstractLeaf]` is also always a valid abstract leaf; it again serves as an indication that the leaf should be restored using the metadata, but with the additional constraint to load as the indicated type. For example, instead of specifying `jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=...)`, it is sufficient to pass `jax.ShapeDtypeStruct`. Similarly, instead of passing `0` to restore as an `int`, the type itself may be passed." + ], + "metadata": { + "id": "01zrVpfcdq7m" + }, + "execution_count": null + }, + { + "id": "41d64797", + "cell_type": "markdown", + "source": [ + "To summarize, here are the ways you can load a PyTree using abstract leaves, with the way we most recommend at the top, and the way we least recommend at the bottom.\n", + "\n", + "**1. Fully-specified abstract values**\n", + "\n", + "This provides the most loading validations and requires the least amount of\n", + "unnecessary metadata reads.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=jax.sharding.NamedSharding(...))\n", + "}\n", + "```\n", + "\n", + "**2. Only types specified**\n", + "\n", + "This guarantees that each leaf will be loaded with the indicated type, but metadata\n", + "will be used to restore specific properties for each leaf.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct,\n", + " 'b': int,\n", + " 'c': np.ndarray,\n", + "}\n", + "```\n", + "\n", + "**3. `None` specified (per-leaf)**\n", + "\n", + "This is essentially the same as (2), but metadata will also be used to decide\n", + "which type each leaf should be loaded as.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': None,\n", + " 'b': None,\n", + "}\n", + "```\n", + "\n", + "**4. `None` specified**\n", + "\n", + "This loads the PyTree structure without any checks, and can lead to errors later\n", + "in your code if the checkpoint does not have the structure you expect.\n", + "\n", + "```\n", + "abstract_state = None\n", + "```" + ], + "metadata": { + "id": "nRn_2IgV09xT" + }, + "execution_count": null + }, + { + "id": "b6a7681c", + "cell_type": "markdown", + "source": [ + "### Customizing Loaded Properties for Arrays" + ], + "metadata": { + "id": "UR34KTImVq_8" + }, + "execution_count": null + }, + { + "id": "9369d5dd", + "cell_type": "markdown", + "source": [ + "#### Array dtype" + ], + "metadata": { + "id": "TErvR-zDVq_8" + }, + "execution_count": null + }, + { + "id": "971b5f81", + "cell_type": "code", + "source": [ + "def set_loading_dtype(x: jax.ShapeDtypeStruct) -\u003e jax.ShapeDtypeStruct:\n", + " return x.update(dtype=np.int16)\n", + "\n", + "\n", + "cast_dtype_abstract_state = jax.tree_util.tree_map(\n", + " set_loading_dtype, abstract_state\n", + ")" + ], + "metadata": { + "id": "ZDSJhefZVq_8" + }, + "execution_count": null + }, + { + "id": "0b59e9a7", + "cell_type": "code", + "source": [ + "ocp.load(path, cast_dtype_abstract_state)" + ], + "metadata": { + "id": "scZbqSQ4Vq_8" + }, + "execution_count": null + }, + { + "id": "803e5cc5", + "cell_type": "markdown", + "source": [ + "#### Change sharding" + ], + "metadata": { + "id": "rhkTT9wMVq_8" + }, + "execution_count": null + }, + { + "id": "238e155f", + "cell_type": "markdown", + "source": [ + "**NOTE: This can often be a particularly sharp edge.**\n", + "\n", + "Sharding commonly needs to be changed when loading a checkpoint saved on one topology to a different topology.\n", + "\n", + "**If changing topologies, you MUST specify sharding when loading.**\n", + "\n", + "Unless you are loading on the exact same topology, Orbax does not make any decisions about shardings on your behalf. If you have the exact same topology,\n", + "however, it is possible to avoid specifying the sharding when loading. This is demonstrated below:" + ], + "metadata": { + "id": "NLjuYubRVq_8" + }, + "execution_count": null + }, + { + "id": "196eef08", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path)" + ], + "metadata": { + "id": "atPqix9IVq_8" + }, + "execution_count": null + }, + { + "id": "ea0ac0bf", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "Pap_vpN9Vq_8" + }, + "execution_count": null + }, + { + "id": "31011e33", + "cell_type": "markdown", + "source": [ + "In the example below, we alter the sharding while loading." + ], + "metadata": { + "id": "T_T3QjRLVq_8" + }, + "execution_count": null + }, + { + "id": "ca7af6eb", + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('x',)),\n", + " jax.sharding.PartitionSpec(),\n", + ")\n", + "\n", + "\n", + "def set_sharding(x: jax.ShapeDtypeStruct) -\u003e jax.ShapeDtypeStruct:\n", + " return x.update(sharding=sharding)\n", + "\n", + "\n", + "change_sharding_abstract_state = jax.tree_util.tree_map(\n", + " set_sharding, abstract_state\n", + ")\n", + "loaded = ocp.load(path, change_sharding_abstract_state)" + ], + "metadata": { + "id": "J0tvS901Vq_8" + }, + "execution_count": null + }, + { + "id": "23a43ee2", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "M84bx2smVq_8" + }, + "execution_count": null + }, + { + "id": "b73ad6a0", + "cell_type": "markdown", + "source": [ + "We can use pytree metadata instead of the abstract pytree." + ], + "metadata": { + "id": "zA_Vdck2Vq_8" + }, + "execution_count": null + }, + { + "id": "28cc7757", + "cell_type": "code", + "source": [ + "metadata = ocp.metadata(path).metadata\n", + "change_sharding_metadata = jax.tree_util.tree_map(\n", + " lambda x: jax.ShapeDtypeStruct(shape=x.shape, dtype=x.dtype, sharding=sharding), metadata\n", + ")\n", + "loaded = ocp.load(path, change_sharding_metadata)\n", + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "cAN_1Xj_Vq_8" + }, + "execution_count": null + }, + { + "id": "5cc090f6", + "cell_type": "markdown", + "source": [ + "#### Change leaf type" + ], + "metadata": { + "id": "sIj3AYAidwvh" + }, + "execution_count": null + }, + { + "id": "758157c2", + "cell_type": "markdown", + "source": [ + "The abstract leaf type dictates the loaded type for each leaf. If we save a value as a `jax.Array` but provide an abstract leaf without the required `sharding` property, Orbax will load as `np.ndarray`. Similarly, we can save as an `int` and load as a `float` if we specify `float` as the abstract leaf." + ], + "metadata": { + "id": "01BAYxDGez1T" + }, + "execution_count": null + }, + { + "id": "dd9a8ce2", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/change-type/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "pytree_with_scalars = {\n", + " 'a': np.asarray(12),\n", + " 'b': 13.5,\n", + " 'c': create_sharded_array(np.arange(8)),\n", + "}\n", + "ocp.save(path, pytree_with_scalars)" + ], + "metadata": { + "id": "cDdy1bk0dzDk" + }, + "execution_count": null + }, + { + "id": "c91174ff", + "cell_type": "code", + "source": [ + "abstract_state_with_scalars = {\n", + " 'a': float,\n", + " 'b': int,\n", + " 'c': np.empty((8,)),\n", + "}\n", + "ocp.load(path, abstract_state_with_scalars)" + ], + "metadata": { + "id": "UxWeE0E8eg52" + }, + "execution_count": null + }, + { + "id": "153334fc", + "cell_type": "markdown", + "source": [ + "### Partial Loading\n", + "\n", + "You may wish to load part of a PyTree contained within a saved checkpoint. For example, consider the following item:" + ], + "metadata": { + "id": "LBSbal0hVq_8" + }, + "execution_count": null + }, + { + "id": "90272421", + "cell_type": "code", + "source": [ + "original_item = {\n", + " 'params': {\n", + " 'layer1': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " 'layer2': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " },\n", + " 'opt_state': [np.arange(8), np.arange(8)],\n", + " 'step': 101,\n", + "}\n", + "\n", + "path = epath.Path('/tmp/checkpointing-pytrees/partial/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "ocp.save(path / '1', original_item)" + ], + "metadata": { + "id": "96Zk1nrzVq_8" + }, + "execution_count": null + }, + { + "id": "2a00857d", + "cell_type": "markdown", + "source": [ + "If we want to load only a subset of PyTree nodes (`params.layer2` and `step`, for example), we can use Placeholder values." + ], + "metadata": { + "id": "g9q0W8GoVq_8" + }, + "execution_count": null + }, + { + "id": "b84eaaeb", + "cell_type": "markdown", + "source": [ + "#### Placeholder\n", + "\n", + "To load part of a PyTree item, we can specify which nodes to ignore during loading by using `...` (`ocp.PLACEHOLDER`)." + ], + "metadata": { + "id": "s9WFY0vuVq_8" + }, + "execution_count": null + }, + { + "id": "d1c17716", + "cell_type": "code", + "source": [ + "reference_item = {\n", + " 'params': {\n", + " 'layer1': {\n", + " 'kernel': ...,\n", + " 'bias': ...,\n", + " },\n", + " 'layer2': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " },\n", + " 'opt_state': [..., ...],\n", + " 'step': 101,\n", + "}\n", + "\n", + "ocp.load(path / '1', reference_item)" + ], + "metadata": { + "id": "NHreKKn8Vq_9" + }, + "execution_count": null + }, + { + "id": "6bfc7009", + "cell_type": "markdown", + "source": [ + "## Advanced Customizations" + ], + "metadata": { + "id": "MH3hBSxPVq_9" + }, + "execution_count": null + }, + { + "id": "e4a66e76", + "cell_type": "markdown", + "source": [ + "`ocp.Context` enables more customizations.\n", + "\n", + "For customized save/load behavior, these APIs should be invoked within a `ocp.Context`\n", + "instance, which in turn can be configured with a number of options like Saving, Loading,\n", + "FileOptions etc.\n", + "\n", + "The usage pattern is as follows:\n", + "```\n", + "with ocp.Context(\n", + " pytree_options=PyTreeOptions(...),\n", + " file_options=FileOptions(...),\n", + "):\n", + " ocp.save(path, pytree)\n", + "```\n", + "\n", + "Let's explore few examples. Please also take a look at API Reference for specific option details." + ], + "metadata": { + "id": "nZcttkVHVq_9" + }, + "execution_count": null + }, + { + "id": "da3e1ed1", + "cell_type": "markdown", + "source": [ + "### Saving" + ], + "metadata": { + "id": "CY0cRK32Vq_9" + }, + "execution_count": null + }, + { + "id": "79a09a4a", + "cell_type": "markdown", + "source": [ + "#### Customizing Array dtype" + ], + "metadata": { + "id": "DpuH7FMHVq_9" + }, + "execution_count": null + }, + { + "id": "a621ff4a", + "cell_type": "markdown", + "source": [ + "we can customize the on-disk type used to save individual arrays. First, let's save and load as normal." + ], + "metadata": { + "id": "bq3Hj3-aVq_9" + }, + "execution_count": null + }, + { + "id": "a2264fa6", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/advanced/')\n", + "path.rmtree(missing_ok=True)" + ], + "metadata": { + "id": "FZt44Li_Vq_9" + }, + "execution_count": null + }, + { + "id": "7af654a9", + "cell_type": "code", + "source": [ + "ocp.save(path / '1', pytree)" + ], + "metadata": { + "id": "68BXPuRcVq_9" + }, + "execution_count": null + }, + { + "id": "25b249e1", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '1')" + ], + "metadata": { + "id": "LcXASaTWVq_9" + }, + "execution_count": null + }, + { + "id": "038f2721", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "WnCRCzlOVq_9" + }, + "execution_count": null + }, + { + "id": "a5a088ef", + "cell_type": "markdown", + "source": [ + "Now, let's set the dtype of selective array when saving." + ], + "metadata": { + "id": "yhb3Py_9Vq_9" + }, + "execution_count": null + }, + { + "id": "08d2a51e", + "cell_type": "code", + "source": [ + "def scoped_storage_options_creator(keypath, value):\n", + " del value\n", + " last_key = keypath[-1]\n", + " # Override 'a' to int16\n", + " if isinstance(last_key, jax.tree_util.GetAttrKey) and last_key.name == 'a':\n", + " return ocp.options.ArrayOptions.Saving.StorageOptions(\n", + " dtype=np.dtype(np.int16)\n", + " )\n", + " # Return None to use global default storage_options for other leaves\n", + " return None\n", + "\n", + "ctx = ocp.Context()\n", + "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", + "\n", + "with ctx:\n", + " ocp.save(path / '2', pytree, overwrite=True)" + ], + "metadata": { + "id": "9VpoTnHmVq_9" + }, + "execution_count": null + }, + { + "id": "7c4c8c9f", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '2')" + ], + "metadata": { + "id": "mHh_ucu_Vq_9" + }, + "execution_count": null + }, + { + "id": "e1cf3f3b", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "sDHCc2s4Vq_9" + }, + "execution_count": null + }, + { + "id": "70eb8746", + "cell_type": "markdown", + "source": [ + "Now, let's set the dtype of all arrays when saving." + ], + "metadata": { + "id": "P58IxTVHVq_9" + }, + "execution_count": null + }, + { + "id": "48917323", + "cell_type": "code", + "source": [ + "scoped_storage_options_creator = (\n", + " lambda k, v: ocp.options.ArrayOptions.Saving.StorageOptions(\n", + " dtype=np.dtype(np.int16)\n", + " )\n", + ")\n", + "\n", + "ctx = ocp.Context()\n", + "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", + "\n", + "with ctx:\n", + " ocp.save(path / '3', pytree, overwrite=True)" + ], + "metadata": { + "id": "W-k7jRWFVq_9" + }, + "execution_count": null + }, + { + "id": "506d2a46", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '3')" + ], + "metadata": { + "id": "Cgs_vBOWVq_9" + }, + "execution_count": null + }, + { + "id": "3e415dcb", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "9acbLs7CVq_9" + }, + "execution_count": null + }, + { + "id": "03152a87", + "cell_type": "markdown", + "source": [ + "#### High Throughput with `ocdbt` option" + ], + "metadata": { + "id": "XZZq8Nxelxwn" + }, + "execution_count": null + }, + { + "id": "653e3c07", + "cell_type": "markdown", + "source": [ + "For high throughput and avoid creating separate subdirectories for each leaf, enable `use_ocdbt`. Please note that it is enabled by default." + ], + "metadata": { + "id": "GhMBAVqDn1s2" + }, + "execution_count": null + }, + { + "id": "f6daa343", + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = True\n", + "with ctx:\n", + " ocp.save(path / '4', pytree, overwrite=True)" + ], + "metadata": { + "id": "W_FD_od_mb8z" + }, + "execution_count": null + }, + { + "id": "38c16f01", + "cell_type": "markdown", + "source": [ + "A checkpoint created with this option enabled can be identified by presence of files `manifest.ocdbt` and subdirs like `ocdbt.process_*`." + ], + "metadata": { + "id": "G6Cm0PYcoIF4" + }, + "execution_count": null + }, + { + "id": "7c9a8da3", + "cell_type": "code", + "source": [ + "print(sorted([p.name for p in (path / '4' / 'state').iterdir()]))" + ], + "metadata": { + "id": "IA4tTKDGoGlf" + }, + "execution_count": null + }, + { + "id": "bad9abc2", + "cell_type": "markdown", + "source": [ + "However, for use cases like large stacked models, disabling this option may be more efficient." + ], + "metadata": { + "id": "GqA8Kdk6orZc" + }, + "execution_count": null + }, + { + "id": "603bf145", + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = False\n", + "with ctx:\n", + " ocp.save(path / '5', pytree, overwrite=True)\n", + "\n", + "print(sorted([p.name for p in (path / '5' / 'state').iterdir()]))" + ], + "metadata": { + "id": "qJvm7QxCpUOT" + }, + "execution_count": null + }, + { + "id": "ec1db3e0", + "cell_type": "markdown", + "source": [ + "Please note how each leaf is written in its own subdir when `use_ocdbt=False`." + ], + "metadata": { + "id": "-OZF-uwBpb8y" + }, + "execution_count": null + }, + { + "id": "6a9d4943", + "cell_type": "markdown", + "source": [ + "### Loading" + ], + "metadata": { + "id": "_AnAJYT8Vq_9" + }, + "execution_count": null + }, + { + "id": "ef2e649f", + "cell_type": "markdown", + "source": [ + "#### Pad / truncate shape" + ], + "metadata": { + "id": "5aTJ6eUJVq_9" + }, + "execution_count": null + }, + { + "id": "f813d121", + "cell_type": "markdown", + "source": [ + "Ordinarily, specifying a target array with a different shape than in the\n", + "checkpoint results in an error." + ], + "metadata": { + "id": "2Qg0HbH3Vq_9" + }, + "execution_count": null + }, + { + "id": "bcf99098", + "cell_type": "code", + "source": [ + "# Original shape.\n", + "loaded = ocp.load(path / '1')\n", + "\n", + "(loaded['a'].shape, loaded['b'].shape)" + ], + "metadata": { + "id": "p1I0sfNEVq_9" + }, + "execution_count": null + }, + { + "id": "2a8a944e", + "cell_type": "code", + "source": [ + "different_shape_abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(\n", + " shape=(8,),\n", + " dtype=abstract_state['a'].dtype,\n", + " sharding=abstract_state['a'].sharding,\n", + " ),\n", + " 'b': jax.ShapeDtypeStruct(\n", + " shape=(32,),\n", + " dtype=abstract_state['b'].dtype,\n", + " sharding=abstract_state['b'].sharding,\n", + " ),\n", + "}\n", + "\n", + "try:\n", + " ocp.load(path / '1', different_shape_abstract_state)\n", + "except BaseException as e:\n", + " print(e)" + ], + "metadata": { + "id": "2X8mMPBkVq_9" + }, + "execution_count": null + }, + { + "id": "ded90dfd", + "cell_type": "markdown", + "source": [ + "We can pad or truncate arrays as they are loaded by specifying `enable_padding_and_truncation=True`." + ], + "metadata": { + "id": "HMPpVIiLVq_9" + }, + "execution_count": null + }, + { + "id": "bd96c69c", + "cell_type": "code", + "source": [ + "\n", + "ctx = ocp.Context()\n", + "ctx.array.loading.enable_padding_and_truncation = True\n", + "with ctx:\n", + " loaded = ocp.load(path / '1', different_shape_abstract_state)" + ], + "metadata": { + "id": "Rbu-q1upVq_9" + }, + "execution_count": null + }, + { + "id": "322ae4c3", + "cell_type": "code", + "source": [ + "(loaded['a'].shape, loaded['b'].shape)" + ], + "metadata": { + "id": "HP6gm2HEVq_9" + }, + "execution_count": null + } + ], + "metadata": { + "colab": { + "last_runtime": { + "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", + "kind": "private" + }, + "private_outputs": true, + "provenance": [ + { + "file_id": "1QNxBBBN16Br9Xj-a7LvtJzJWjOBhjFps", + "timestamp": 1686159333109 } + ], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 0 + "language_info": { + "name": "python" + } + }, + "nbformat_minor": 0, + "nbformat": 4 } diff --git a/docs/guides/checkpoint/v1/maximizing_performance.ipynb b/docs/guides/checkpoint/v1/maximizing_performance.ipynb new file mode 100644 index 0000000000..bfedf8de40 --- /dev/null +++ b/docs/guides/checkpoint/v1/maximizing_performance.ipynb @@ -0,0 +1,848 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "colab": { + "provenance": [ + { + "file_id": "1GZkiWMcvYFs_tW-Y-p07R4M5YU8N9K1r", + "timestamp": 1781122549898 + } + ], + "last_runtime": { + "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", + "kind": "private" + }, + "private_outputs": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Maximizing Performance" + ], + "metadata": { + "id": "Z7zpSM8GbuVA" + }, + "id": "Z7zpSM8GbuVA" + }, + { + "cell_type": "markdown", + "source": [ + "Checkpointing performance is a core focus of the Orbax team and we strive to make our library as performant out-of-the-box as possible. However, many performance features are tailored for more specific use cases and are usable on\n", + "an opt-in basis.\n", + "\n", + "When we say \"performance\" with regards to checkpointing, we generally have a few primary metrics:\n", + "\n", + "\n", + "* **Save blocking time:** The time spent blocking the main thread. In an async save, this is largely attributable to the device/host transfer bandwidth (e.g. 32 GB/s with TPU PCIe Gen5) and is minimally attributable to Orbax.\n", + "* **Save total time:** The total time required to save a checkpoint completely. Since the majority of the total time is comprised of disk I/O occuring in a background thread, its performance impact is less obvious. However, we still want to ensure reliably fast checkpoint completions, as frequent checkpointing can [increase overall training goodput](https://developers.googleblog.com/boost-training-goodput-how-continuous-checkpointing-optimizes-reliability-in-orbax-and-maxtext/).\n", + "* **Load total time:** The total time required to load a checkpoint. This is critical for warmstart training, evaluations, and inference. It impacts accelerator utilization, experiment recovery times, and developer velocity. While loads are performed less frequently than saves, their blocking nature makes them more critical.\n", + "* **Memory consumption:** The additional RAM consumed by checkpointing. RAM is a scarce resource just as accelerators are and excess memory consumed by checkpointing inhibits other useful applications of available memory.\n", + "\n" + ], + "metadata": { + "id": "IfDQ4BvjdlUL" + }, + "id": "IfDQ4BvjdlUL" + }, + { + "cell_type": "markdown", + "source": [ + "Below, we will reference various performance benchmarks. These are performed with single- or multi-region GCS buckets using Llama 3.1 models (8B, 70B, and 405B) adapted for JAX\n", + "using [MaxText](https://maxtext.readthedocs.io/). Typically, we use TPU topologies v5p-8, v5p-32, and v5p-128 for each model size, respectively, unless otherwise noted." + ], + "metadata": { + "id": "Ru3PCuGWeS4d" + }, + "id": "Ru3PCuGWeS4d" + }, + { + "cell_type": "code", + "source": [ + "import jax\n", + "from jax import numpy as jnp\n", + "from orbax.checkpoint import v1 as ocp\n", + "import os\n", + "import numpy as np\n", + "from etils import epath\n", + "\n", + "# Use 16 fake devices.\n", + "os.environ[\"XLA_FLAGS\"] = \"--xla_force_host_platform_device_count=16\"\n", + "root_directory = epath.Path('/tmp/maximizing_performance')\n", + "root_directory.rmtree(missing_ok=True)" + ], + "metadata": { + "id": "m4vRKQH9uGpt" + }, + "execution_count": null, + "outputs": [], + "id": "m4vRKQH9uGpt" + }, + { + "cell_type": "markdown", + "source": [ + "## Key Performance Optimizations" + ], + "metadata": { + "id": "wGLp4RbEdFS5" + }, + "id": "wGLp4RbEdFS5" + }, + { + "cell_type": "markdown", + "source": [ + "### Asynchronous Checkpointing" + ], + "metadata": { + "id": "X9q5tTpleOpY" + }, + "id": "X9q5tTpleOpY" + }, + { + "cell_type": "markdown", + "source": [ + "Async checkpointing is THE key checkpointing performance improvement, as it can hide almost all of the cost of slow disk I/O behind training computations. It is strongly recommended that all users should enable this feature. See our [guide](https://orbax.readthedocs.io/en/latest/guides/checkpoint/v1/async_checkpointing.html) for more details." + ], + "metadata": { + "id": "Vvsjf1PWeT_2" + }, + "id": "Vvsjf1PWeT_2" + }, + { + "cell_type": "markdown", + "source": [ + "### Storage Format" + ], + "metadata": { + "id": "sGEcdDp1byZK" + }, + "id": "sGEcdDp1byZK" + }, + { + "cell_type": "markdown", + "source": [ + "Orbax provides two [storage formats](https://orbax.readthedocs.io/en/latest/guides/checkpoint/v1/checkpoint_format.html#pytree-checkpointables) (leveraging [TensorStore](https://google.github.io/tensorstore/)) referred to as `OCDBT` and `non-OCDBT`.\n", + "\n", + "At a high level, the `non-OCDBT` format stores each model weight as a separate directory while the `OCDBT` format packs model weights into a series of uniform files.\n" + ], + "metadata": { + "id": "sIr79qECjwMb" + }, + "id": "sIr79qECjwMb" + }, + { + "cell_type": "markdown", + "source": [ + "#### `OCDBT` Format\n", + "This format packs model weights into a series of uniform files.\n", + "\n", + "**+++**\n", + "\n", + "* Improved performance for large models.\n", + "* More tunable settings for extracting performance on different filesystems.\n", + "\n", + "**---**\n", + "\n", + "* Higher overhead, poorer performance for small models." + ], + "metadata": { + "id": "dABkdAeE0oWQ" + }, + "id": "dABkdAeE0oWQ" + }, + { + "cell_type": "markdown", + "source": [ + "#### `non-OCDBT` Format\n", + "This format stores each model weight as a separate sub-directory.\n", + "\n", + "**+++**\n", + "\n", + "* More human-readable.\n", + "* Low-overhead on small models.\n", + "\n", + "**---**\n", + "\n", + "* Performs poorly for models with many distinct arrays (especially small arrays).\n", + "* Poorer performance for large models." + ], + "metadata": { + "id": "zxj_93sz0qg8" + }, + "id": "zxj_93sz0qg8" + }, + { + "cell_type": "markdown", + "source": [ + "| Operation | Model | Topology | OCDBT | No OCDBT | Speedup |\n", + "| :--- | :--- | :--- | :--- | :--- | :--- |\n", + "| Load | 8B | v5p-8 | 22.85 ± 4.68 | 17.97 ± 4.22 | 0.79×* |\n", + "| Load | 70B | v5p-32 | 41.19 ± 4.17 | 40.04 ± 4.61 | 0.97× |\n", + "| Load | 405B | v5p-128| 59.14 ± 4.06 | 65.18 ± 12.01 | **1.10×** |\n", + "| Save (Background) | v5p-8 | 8B | 59.37 ± 9.04 | 33.71 ± 3.20 | 0.57×* |\n", + "| Save (Background) | v5p-32 | 70B | 62.47 ± 9.09 | 80.84 ± 8.82 | **1.29×** |\n", + "| Save (Background) | v5p-64 | 405B | 91.76 ± 14.91 | 123.33 ± 19.27 | **1.34×** |" + ], + "metadata": { + "id": "Yxmp4zh5i8MR" + }, + "id": "Yxmp4zh5i8MR" + }, + { + "cell_type": "markdown", + "source": [ + "You can configure the storage format following this example:" + ], + "metadata": { + "id": "IfB0aOAIupoi" + }, + "id": "IfB0aOAIupoi" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = True # default\n", + "with ctx:\n", + " path = root_directory / 'ckpt_ocdbt'\n", + " ocp.save(path, {'a': jnp.arange(8)})\n", + " assert os.path.exists(path / 'state' / 'manifest.ocdbt')\n", + "\n", + "ctx.array.saving.use_ocdbt = False\n", + "with ctx:\n", + " path = root_directory / 'ckpt_non_ocdbt'\n", + " ocp.save(path, {'a': jnp.arange(8)})\n", + " assert not os.path.exists(path / 'state' / 'manifest.ocdbt')" + ], + "metadata": { + "id": "fMZ4DPnHuoBV" + }, + "execution_count": null, + "outputs": [], + "id": "fMZ4DPnHuoBV" + }, + { + "cell_type": "markdown", + "source": [ + "### Loading with Resharding" + ], + "metadata": { + "id": "qNpl940nd7Ai" + }, + "id": "qNpl940nd7Ai" + }, + { + "cell_type": "markdown", + "source": [ + "When warm-starting or recovering a training experiment, or when performing evaluations or inference on a different topology than the training topology, *resharding* is needed.\n", + "\n", + "To facilitate efficient (fast and low-memory-overhead) checkpoint loading with resharding, we need to first understand a bit about how Orbax uses TensorStore to store large distributed arrays.\n", + "\n", + "Using TensorStore's zarr driver, distributed arrays are represented in a chunked format, where `jax.Array` shards correspond one-to-one with TensorStore shards. If an array is resharded and its new boundaries don't align with the originally saved chunks, the system is forced to load and discard excess data, which increases I/O and memory (RAM) overhead.\n", + "\n", + "However, we can subdivide larger write chunks by configuring granular \"subchunks\" at save time, which reduces overhead." + ], + "metadata": { + "id": "NP9etjit0sM2" + }, + "id": "NP9etjit0sM2" + }, + { + "cell_type": "markdown", + "source": [ + "This is illustrated in the figure below. For example, if our array is shape `(4, 4)`, it may be sharded across 4 devices using shard shapes of `(2, 2)`. The checkpoint will be saved with chunks of the same shape.\n", + "\n", + "If we attempt a reshard operation where we need to load shards of shape `(1, 2)`, each read will have 2x overhead, since the `(2, 2)` chunk is the smallest readable unit.\n", + "\n", + "This can be resolved if we configure `(1, 1)` chunk shapes though, as a `(1, 2)` will read two chunks with no overhead." + ], + "metadata": { + "id": "8RAwtLBhr5h8" + }, + "id": "8RAwtLBhr5h8" + }, + { + "cell_type": "markdown", + "source": [ + "![8K5xDXQpmtFovE9.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAxUAAAG3CAYAAADCVnjGAAAQAElEQVR4AeydBWAUR9/Gn02ABHd3dy9UoMXa0lJ3L9C3+lXevrW3rm/dnbq7Gy1t0UKhuLu7EyAQCJFvn0nmuNxdkktySU4eyOzszvx35Ld7M/Mf27hM/RMBERABERABERABERABERCBIhCIg/6JgAhEAAElUQREQAREQAREQATCl4CUivB9NkqZCIiACIhApBFQekVABEQgRglIqYjRB69si4AIiIAIiIAIiECsElC+Q09ASkXomSpEERABERABERABERABEYgpAlIqYupxl1RmFY8IiIAIiIAIiIAIiEAsEZBSEUtPW3kVAREQAW8COhcBERABERCBEBGQUhEikApGBERABERABERABIqDgMIUgUggIKUiEp6S0igCIiACIiACIiACIiACYUxASgXC+OkoaSIgAiIgAiIgAiIgAiIQAQSkVETAQ1ISRUAEAAiCCIiACIiACIhA2BKQUhG2j0YJEwEREAEREIHII6AUi4AIxCYBKRWx+dyVaxEQAREQAREQAREQgdglEPKcS6kIOVIFKAIiIAIiIAIiIAIiIAKxRUBKRWw9b+W2pAgoHhEQAREQAREQARGIIQJSKmLoYSurIiACIiACOQnoSgREQAREIDQEpFSEhqNCEQEREAEREAEREAERKB4CCjUCCEipiICHpCSKgAiIgAiIgAiIgAiIQDgTkFIRzk+npNKmeERABERABERABERABESgCASkVBQBnm4VAREQgZIkoLhEQAREQAREIFwJSKkI1yejdImACIiACIiACEQiAaVZBGKSgJSKmHzsyrQIiIAIiIAIiIAIiIAIhI5A5CkVocu7QhIBERABERABERABERABEQgBASkVIYCoIERABPwJyEUEREAEREAERCB2CEipiJ1nrZyKgAiIgAiIgC8BXYuACIhASAhIqQgJRgUiAiIgAiIgAiIgAiIgAsVFIPzDlVIR/s9IKRQBERABERABERABERCBsCYgpSKsH48SV1IEFI8IiIAIiIAIiIAIiEDhCUipKDw73SkCIiACIlCyBBSbCIiACIhAmBKQUhGmD0bJEgEREAEREAEREIHIJKBUxyIBKRWx+NSVZxEQAREQAREQAREQAREIIQEpFSGEWVJBKR4REAEREAEREAEREAERCCcCUirC6WkoLSIgAtFEQHkRAREQAREQgZghIKUiZh61MioCIiACIiACIuBPQC4iIAKhICClIhQUFYYIiIAIiIAIiIAIiIAIxDCBYlcqYpitsi4CIiACIiACIiACIiACMUFASkVMPGZlUgTyJSABERABERABERABESg0ASkVhUYX+ht37dqFjRs35giYbhkZGTncdCECIiACIhCrBJTvSCeQkpKCtLS0ImcjNTUVa9asCUlYRU5MKQSwb9++UohVUeZFQEpFXnRKwI8Fy9NPP40GDRqgbt26aNu2LSpVqoQLLrgAv/zyC1q2bIn58+eXQEoiM4pvvvkGjRo1QtOmTdG8eXOP4TWZTpkyJTIzplSLgAiIgAjkSoBlP+vM3Azrg/79++OKK67A6NGjcw2npDzmzZuHqlWrokyZMqhQoQJY7xcm7uXLl+Oyyy5Dx44dUaVKFRxzzDGoWLGiuf6///s/bNiwoTDBRtw9119/vck/7bBNfAwmTEpFKT/0f//737jjjjtwyy23gFr33r17MXv2bMTFxeHUU08FRyoyMzP9UsmCZOTIkX7useYwZMgQ/Prrr7j77ruxevVqY7p27Yqff/4Zo0aNQqdOnaIOyddff43jjz8+6vKlDImACIhAsARY9v/55594+OGHsXXrVmNOP/100O2nn37Cyy+/DCoVn3/+uSkv+/Xrh+Tk5GCDD7kclYA//vjDKAAMPD09nVaBzDPPPIMuXbrgs88+AxvT27dvN0rEihUrcP7552PEiBFo3bo1Pv744wKFG67CubVzyO6DDz4AZ3G888474HW45iHW0iWlohSf+Gq3Efz666+jRYsWuPXWW1G2bFmTmlatWuHTTz/FmWeeaa59D7xv8uTJ4BCqr1+MXHuyWb58eXTu3Bnt27f3uNWoUcO40Z2jPh6PKDmhwrRly5YoyY2yIQIiIAIFJxCo7K9Tp44p+3v37m065R588EF89NFH4L8JEybguuuu42mpGHYUMl3NmjUrVPxsRN9+++04cOAAvv/+e3BUwtZvHK1/4IEH8Pzzz5t2wfDhw81Mh0JFFCY35dXOiY+Px2233WYUtJtuugm8DpNkx3wypFSU4iswa9YscBSiWrVqcBwnR0ocxzG97zkcsy9+++237DNZsUaAPTLs7Yq1fCu/kUpA6RaB0iXgParLEQzWuaWbooLHzhEJdjzyTvbecxYDz30NFY169eqZNRacBcGefF+ZSLnOr51DhZEzO5566qlIyVJMpFNKRSk+Zs6rZPQzZ840PSibNm3ipcf06tULffr0QeXKlY0bG5Sc8vTf//7XXO/Zswfbtm0zhmszjKPX4eDBg5g7dy6mTZtmplZ5eZnTQ4cOgT3eHDq16zZ4z8KFC/2GExk33ceNGwfODQ0Unwk0+8Cwly5disWLF2P//v3GdfPmzWaqku9idBbyXGz2119/YdWqVWZI09xQAocdO3aAoz6Mmyy9o2Sed+7caYbMOWxO3pyiZmV4vnv3buPPaWrW3drM78SJE82ULOtmbYbNimK1O1q1YMECMB3Wj3GuW7cOixYtAp+NdeezuvTSS81ift7P9NIwDVZGtgiIgAiIwGEC7Nm3V1yD4Dg5O/CsXzD1EMvdgtSDDJt1B+tM2rwujPnggw88dQTXW+YWBmc7cBoU/Vl3cFSb56wj1q9fb+pj1rV0Y73MdLG+4TXyOLCuYztixowZYD3oLcq6nnUT4wtFO4KM82vnMO1sL3F9yZIlS7yTk+M8mDqYbQ5ySEpKMvdSEWOdzPCNQ4ADGY5z20Jcs8n3a86cOUaRCyAac05SKkrxkR955JFITEw0KeBcSA5hcj3Atddeiy+//BLc2YGNUk6PohALk1NOOcXzo+YCNA730lB5oAwNG+3nnHOOWcR09dVXm6lVtWvXBueU0o8yNG+99RbYq8HpVpwq9N1336F+/fpmwRfD5A+NcixMOGTL+akcXh08eLBZTD5gwACsXbuWIh7DAueqq64yC9EuuugiM8+zZs2aOOGEE9CjRw9wHiwNb+CP95VXXgH9jzrqKDz00ENmDQTXQXComjLFaa655hrUqlULd955J+677z7DomHDhnjhhRdMtKvdBj/TRqWOhovsyN94uocH3aF1jjLRj/e5TuaPw+1cJN6tWzc89thj6N+/v1mgx2dsBNwD5/3ymXAxIfPL+b+us6k4GGeTJk3QoUMHsLeJ7ny+fDacH8xrKhx8RjRctEc3GREQAREQgZwEOMXYutgOOXtNO9h6qCD1IMPliDLrc5bnrIdZ/3GUgY1Y+hfE2MY672F9QTs307hxY4/X1KlTzTnXWdCd04T79u1rpoaxvUEFhPVQmzZtTOejEfY6UFFg3c167uabbzYLxJmf//znP57OwlC3I4Jp57DOYx3LdHNKmVeSzWlB6mC2r7hOhYv5ORrUs2dPnHbaaWbznKOPPtrUySZQ90Blgh29J598spmi/r///c9ssMO6nh2BrkjM/0mpKP5XINcY+EN97rnnzKJsCrFwY+PxjTfeMLs/sWE5e/ZsehnDBbpckGYu3AOveQ8NCyzXCex14DkVBC7WoibNBjoLRCoJRxxxBDgyQlk2/m2hw2su/OKOU1QyGM67775LZ7BQZi8HFZAffvgB/PGcd955oKbOAoo9PEbQPdx44414++23TfrZq8H8DBw40CyeO+uss8AfIQtaVxR33XUXKM/4li1bZmRYeLKH4IwzzjCL1ClXHIbK0JtvvmmCZsOdebG9EywwmQfuvMW8XX755UaOvVzMv7lwD0888QSYdi4St70cVCIoT2WBIzXscWElQgWL83lp3FvNwkEyTcxWKulGwwKbox4s5HhtDa/5nDm8TTdWLLym+fHHH+kkIwIiIAIxTYB1GztvXnzxRbMBCju+2FnVrFkzsD5jHecLKNh6qCD14FdffYUTTzzRbPfKDUM4Gs76gPUAe/1905DfNetFK8POLXseyKaSYN1XrlxpTmmfffbZ5pz1DutgjnKzc4rrM9jjf+yxx4I7ahkh98B6mO0FdoCxLTFp0iRwlIZ1EDvennzySVcKCHU7gu2a/No5HLm55557TPy+h8LWwY8//rjZmWv69Olge4R1MdtP7Ei1cVBmzZo1ZrYG2w8cCbLM2FawcrFsS6ko5afPRiYbtGyc+y424pAiCyYWCDaZjnN46NZxHLMWw3Ec62163HkfG6Fs+FsP7jzBhjoLCjaG6c6hUrrznIbaNzVz/nA4Z3P48OF0Bkcm2CvAHgQ6MJ2chsNzKhgsyHlOwwKKNsOhTWPP2cBmQcACgT0gVKjozwVXdsEZG+MXX3wxkpKS8Nprr9G7WAx7/VmIsnFuezratWsHjh4xQm/lgYvjHMcxI0RU+OhPY6dMsferXLlyRtniTiT04z1UQnjuOI5RoHjOCo8FMxlydKN69ep0zmGobHJL3ByO2ReOk/WsHcfxe/bZIrIKTUA3ioAIRDIB1hvsEOOIOUflWUYzP1xXYesvXltTkHoo2HqQU4g524Bx0O7fvz9PjWEnGjcSMRcFOHgrIpzBkNetjN/6O45jTlm/spOMF+yZf/TRRz2Lm9krz12zeB/dKUNjO8s4uuHNjnW44zhg/c2pP8XRjnCcrHQzHY7j+NV1zA/ra/p7G7ZHClsHcxoX2z6sm2nYhmLYVLxo01DJoDJGpZWKB934brEjMT9lj7KxYKRUhMFTZuOWhR8LRPYKsGeFU1+YNL7AHMrjeTBm7NixRizQD47fwKAnh/nYw81zb2MLv3PPPRdc0GZ/VCxQuPc1G88c8aDiwFEPey+VGHvOqUA855xH2jT23PrRjT9Ouy6Du1j9888/sIa7elDGOw5eF9VQqWGPC8NhQcgRHMbBfcN///13s00fd+igv3eeqHhQ4aI7e2hsoc6eGk6hssoDe6NYMFPOlz+VhISEBHqBcZkTHURABERABApOIJc7OCrOxjCns7Ksp6EoR565bTvPvU1B6qFg60GOeFOxYTyc1ku7qIYjLTYMG7a99rW9e/mpEPj6B7o+7rjjjDNnRrDNwQtOvabNHntbN9PmyDvrcq4R4RRhynibULQjvMMryHlR6mC2w2z9zzipuNC2dTrPKcP6n9OkOPWKnZOXXHKJ2VSHnCgT60ZKRSm+AWzcswC0SeBLPGjQINx///3gwh+OUtCPIxm0gzH8wVOODWXa3oYNaV6zULJyvLaG05vsubfNQmbo0KFm7QPTxN56DgF6y9jze++915xyTQh7DdgTxKlYdOT3OGjTsBeJNg2HEN9//328n23YK8MeHtuQp0woDJUhcrVhMY1UnDhiwEKCChkVutaEfAAAEABJREFUHOvvbdu0c6SHSh6ndVGe80ytnDfTvPhzWpi9R7YIiIAIiEDxELjwwgthR4O9R59tbAWph4KtB70b2raDzMZXWJv1lL2Xu0ba80A2pzJZd66VtOd52VxbSH9O4aFiwXPLhtOlbN1sbc4m4DSoQKMuoWhHMP7CmKLUwZZBXvFypynOLrEdhFwIzu3/uaaCbYK87o0VP1+lIlbyHRb55Lx+ztPksJtvgtgo5f7LdHcch1auhj90hkMB7hhFm70ItL2NdaM2zulM3n48pzttb8NRBhZMH374oZkjyoKGH5uzjWxvWZ6zkOF8zbp164K9H5xyxSlWY8aMgZ0yRTkuWKNNw6lCTL+voSJD/1AY7ijB0RcqEAyPQ7fseWIhxNEhjlhwjmT37t3p7We4yN1Ok+IHiDiFjPlhT4UVtux5bVnznIajMhwq5jkXyNHOy1Cxysvf24+9c3xO3m46FwEREAERgNkkgxzYAGR5z3Nrgq2HWL4GWw/aKbSMoyDlOOVzM+xk45Qc+rMuph3IcD0e13PQj/UM6wae52eoOFgZO6PBsuFoh2/dzOtXX30V3CjE3mftULQjbFjetnc7x9vd+zyUdbB3uPaciukjjzxi1q5ypsNLL71k1lWybueCdSsXy7aUilJ++pzyxAVkgZJBP7pzsRRtGs7dd5wsJcMOy7FnYfXq1fQ2DX+esDecvQ48t4ZTl3jO9RvB9qDwR2SnAnGuor0vt5EKLkrjMCBHAViAc4EZF14NGDCAUXsMC147bYiNeo9H9gkXR3HBdPZlkS0u3mKlwoKWgXG0hTZ3ovJOW275oqxVpLhNLr9oyrUgdLeGu0LYbYIta+vHZ0TFgtcc7aFNYxdqWz+6cfs/O/TMa29je0gOHjxonPmOsHALNJ3NCOggAiIgAjFMgB1dNvtcz8ZzTuflCHqw9VBB6kF2XNnpy6yHGZ81LOdp7HWwNqc/cUMRynPBN9POc1/Dhj4VILpz/R6nKfHc27CDzfua53ZKLjce4QYxdOOuT7T//vtvz05PvKbhFCtO7WIdz+v8TEH4Maz82jmUCWQKUwcHCic3N06r43oK1vOcCsU1MlzA7jiOH6Pcwoh2dykVYfCEOZzGBiunCtnkcCiN06C4aJc9+dadDXGOBPCa80FZgHDrOjaO6caw2KvOngfuakF/FmKcW8o1BZxiZRvUnOrDwoH30fC7Cd7XdPPeSYI7Q9GNDVkuaGJYvOYOUhwa5jlHQDgqwYKJPR4cpWDvPxczca2ILYQ4EsOGOcOg5s+wOWLD+YocMWCeuTCOYeZl2EPANLOHxsrZfHH6FXdm+Ne//mWmlNGfBT5tmy/OwbTfiOA0LDs9ivdSeWB6KE/D3au4NR/POQLju26CDX6mnYoCnx0VKspSmWHhw3OmxVuJIRe628qO5xxitfFykT6fpVUaOPpDGSqRDJdTuli4kSPdZURABMKMgJJTLAQClf0cGWB9wHrPRsrGrz3n2kWec20ce76DrYdsfcF7WVfRzqseZBnOMplbhds6haPXrDd4H+9nme9db9EtL8MRcjt6z1F2rhOxHYuc0sxF1qx3WP9Q1nv7c+9w2XHGbdStcsHe9vHjx4PTozm118pypywu4mYnF7dw5SwF+nGGBacmkx3XCtr6ln40oWhH5NfOIUOmi/GxruQz53lB6mB7n+Vgw2RnLMOzbPku8ZrujIPvDjtC7TXX0PDcKmGUiWUjpaIUn77jOODe0dTi2XPCYUbO/2RPP+focWoNF0Z5F2hMLnsgKMvCgPMAuasDG5b0471UMrhGgA1OXrO3gr3qLIhYkNnG8CeffAI2kjn6QDNs2DCwR4ThWMNGL3s/uCiJDX2miUoLt6fjQmWm7aGHHjJDgLzHKgJsqLNHhfGxl57rRx588EFweJLulKUixMKM93Afb+ad4XEYkV/TPOmkkyiWp2EeOdXqzDPP9MjxWw50o2LDRefeI0F2pIKFJ3fHYmHBPLGH5p133gFHCDjNiYU9GTPvNmAO6958883mkkqgOfE5cNHWuHHjQIWKz4Q7QpApK7v33nsPzJv3LVT8qKyw54nPhe8DFQnuJkE57jxBN/LkNQtzpoFpYbh8zt7rcigjIwIiIALRToCdQOwk4roJ1l/sPWanDstLNoht/p9++mmzxTkbwc8++6zZ4Y91gi03g6mHCloPskHPHn6uheDuhyyrWcfQMK1MGzvVOFLC82CM4zigIsRRBc42YH3M+oV1DetNfvOJdQmn8rJeyC1Mjtzwg3Gs98iK6yCZXq7V8FbAKlasCNZLDzzwgPkeVfPmzc13nZhmTo1iJyXroeJoRzDtebVzWM8yj2TJDjd2ZrJxz/voNy6IOphrIvn+8KvcDIe7TXI7fipoZMMZAHRnxyPbDVQ6mN/+/fvDfiOMm7hwfQk7WfkeMv5YN1IqSvENYK8F90RmbwB/mNTw2ajlFCK+6DxnY943ifxRs7FOGS4c5ncK+LJbOfY4sKeCPe1sjLKHgb3aLMRYiFi5K6+8EhwdoCZOwx4H2tbf2lyQxUKIPS0ssNjzwh53urNRTi2dvQZUgDiHkyMpLPD5Y6Rhzw4LPM5JZXq4sMmGzR8xFQMqVRzFIANOHWIhbmXysvlDZvzBGpt/FpCcosX8clSCPFmAsoBmPtgLxjBZOVEhsGngVDAW6CxUrZuvzefDQo2KCRUo3s+Rp2HDhpmt8bzlmY5vv/0WjI8jPByt4nNibwjjIlf2qLDS4H0cFubUMKaboxUcIVIPCcnIiIAIxBIBdhix/rL1FstZlovsYWYHjmVBZYMdTSyPOarOuoh1IstpKxNMPcT6Lph60IbJ+o51OOtp1imsgzl1hj39LLdZn7LTyMoHa7O8Z13B/LADih1krIfZFuDIPBWXvMLi7AfWdaxz2f7g9rusrzmrwPc+ruN40O0MZJ3MfDD/jIcjMBxNoHyo2xEMk4bPh/Uy42Wc3u0ctilYZ9rnzdkY3unnvfnVwezM5PvDMGhYz7L+ZUcsRy/se0Wbfuz05OwQpoPPkG0hPgdy5MgP0ywDSKkoxbeAL6n3Qif+SPnD4BoK9hLklzQOr7KnIi85FqiMJy+ZYP2YJo6M5CbPoWX+SDkiwp4P9nrQDBkyBGycc3iW97IgpO1r2HCmQuTrXpzXVMY4qsHGum88rKQ42sFCmgUbKwU26G0Pl698oGuGzYI5kJ+3G4dt2dti08FnxneD70QgJo7jgLtsMP3e4ehcBERABETAnwDrS07F5Wg5y1t/iSyX/Oqh/OrBrFAOH1n+sx6wLqxDaRgO/ax7YWzWGRxZZ0O4oPezXvFOV373s24qTDy+4TLfzL+ve27XfG75tXNyu5fuzGNROTMcGtbLtGlY/7KOVh1MGoeNlIrDLHRWRAKcgsQfHYeYOfrA3gMGyR5/LmbiVB8WTHZqD/3C2bB3gooFe4GoCDHd559/PjgFKZzTrbSJQKEJ6EYREIGoI8AF4xydYcY4MsEed/bO81pGBEJJQEpFKGnGeFjs0ec0Hw6HclSCU3s4Z5EaPecgcsoQ55kGO7WptHFSQeIwL6dMcd4tR5F4XdrpUvwiIAIiIAKxTaAguec6vJkzZ4J1GHvX2UHG6dEFCUOyIhAMASkVwVCSTNAEuKCJ33DgHEyuVeB8Q64N4FxMLqLr2bNn0GGFgyDXQXDtAvPBqU9cuBUO6VIaREAEREAERCAYApyazPUJ3JGQC5q51o/rSIK5VzIiUBACUir8aMlBBERABERABERABERABESgIASkVBSElmRFQATCh4BSIgIiIAIiIAIiEDYEwlap4HZrXCAbNqSUEBEQgVIjwKl0XPBfaglQxCIQxQS4lWZxZk9hRxYBlrWc9htZqVZqC0qA2/KGup0dlkoFv1XArd/4cZX8IHGHIX4hkntTe8vymrsceLuV1Dl/kHxQnMPID6mUVLylFQ/3ceZzKK34wzle7h4VzunLL218rtwFKz+54va/8cYbwW+ScP/w4o5L4YtALBHgWjFuRBEOv/No5h5JdQG/4cRvZLEdFanPhHVXuO5wlVubqaTfEW6s06FDB3ADneznXGQr7JQKfufgnHPOAb9vwEWygXLID9hwa0/C4HcYuMtQ5cqVwa8bfvDBB+A3Bbjt5yOPPBLo9mJzoyJx9dVXg3tHd+nSBf/+97/BPaT5ERwujiq2iEshYH74hXtHlylTBnwGTz/9dCmkIu8ouQsVvzPB96NFixag3bRpU5x44onmxhEjRoA7O9HdGvrT2AqWH+vj82zWrJm5v5lr9+/f39yf34Hf5uB3JmjnJ1sc/meffTb4oaSCmA8//NCTFD5j8uOe4txS1+NRCif8mBU/IMWPLlJpL4UkKEoRiDoCv/76Kx544AF8/fXX4Hbf/Hga9/XPzbCcZPnH3fxGjx4ddTyKK0OsA0qzLihovl577TX06dMHeX3Pw4Z59NFHI7f3hTs/duvWDdxu/rHHHkNJdLKedNJJSExMBL/DQcXIprO0bdanebWZSuMdYfuU391ip8KsWbNCgijslIobbrgB69evxxtvvOGXQTbwWJhR4WBheN5554H7L1O745eH+dLyGwn8xPrYsWNRkr2aHF3hl6Tfeust3H777eAXIP/44w/wE+7cceG6667zy08kO3BrOuaPH7JhPoqbNbejDWbkimmx5n//+x/45U1e82vdHM7lzlT8oirdqLwyTCqn9KMZPnw4+HVRFkiU4cd+fvvtN/BrqnymL730EqiM0C8vQx5UcDMyMvDOO+8gPT3dT7wwefILJBcHxv/DDz/gzz//BD/mxEKDSjqHO+lGw/gvvfRSHHnkkeCH/ehGhd0GSXbMM5X0zz77zDqXis3tfb/44gtQ6QlHBbZUoChSESgCAW4petFFF+Gpp54CO74YFMsJlgMPP/ww2ElGc/rpp5ty5KeffgK31KZS8fnnn4Nbg/fr1w+cqsx7i9uwvGJ5XdzxFDb83NLHsji/uqCwcRbHfZxhMWHCBFx77bVBBc+6ge8M2118X2hYZ9KNfv/5z3+wa9cu3HPPPWAnFd2CCriQQtxl8sILLzR3k705CYNDXm0mprO03pFbbrnFKJD87Ydidk9YKRWEyhfi1ltvNb3f3u8BeyfZQGePJRsY/JjaQw89ZApDbvNJw8KPP4Zmbm+y970lcc4eHjbMqBnzITmOY6K1H5wpqYLXRFoCB35FklpuSbBmY3/y5MngkGFBssZeElaWbEzb+6iE8hnxmo1t+nsXnhwGpJvjZD0/ynHUiY1r9rbwHWvXrh2d8zT8gudtt90GKl033XQTeO19Q2Hz5B1GXudUwKnQUMn+9NNPwYL9kksuMaMt9j6O2AwdOhRUvvgxJOabFYL1px9Hbfj9EY56WPfSsqn8DB482KSX2xSXVjoUb3QTiIXccWoIy7JKlSqBo382z6xHWQ60b9/eOoHfNaAby/tTTz0VDz74ID766CPjz/q2JPfp7uYAABAASURBVDrMiru8NJkpwiGv9LHsz6suKEK0xXIr22B85meddVZQ4bMNwPejRo0aHnnOGqHbgAEDwHqEnbzsdGUHMK8XLVrkkQ31CRUXKnihDreo4eXVZirtd+T+++8H26/8vliR81nUAEJ5P1/matWqwbsRaMNnD8nvv/9uLjkSwIaeufA58H72DPs4F/ulXejmmy4+JPaOf/zxx8WehmiNgL0eRckbCzHHyVIS+BxYoXqH17p1a88lh/S5KNjj4J5wSt4nn3yC4cOHu1fB/7HyZS8/ewJ97ypqnnzD873etm2bmc5w1VVX+XoFvKbyw98d77MC7HliZbls2TJ4NzKsf2nYXFtBplSESiN+xSkC0UCAjTz+rjnlwo7KFiRfHKWw8hzBYKefvS4Ou7jLy6KmOb/05VUXFDXuUN7PjjuOBrOu43S4UIXNBvUAV8FgeKxP8+NFuSBMVImU5jvCDrtevXrhxRdfhO0ILyzcsBmp4MfF2BvN4VTO0ffNEKc20Y1+nCLF89wMAXEOfW7+1MgmTpwINph8ZTgMxalMnC7DOXBJSUlGhL2+XBfBHmvj4HXgPEE7bMS5+GyY0bDQZr6OPfZYcHqW1y2eU/7A+CEaDkXbRUVMH6d38eHa9DCtjJ890PZmxssGMLV+q9TQj3FZd9p045QyhstzX8MKYc2aNeAHcphv5tVXxl5zxIVcaFu3wtjMF4dZx40bB4bn29BnmJThcPd///tfXoKMyZUmkLwRCnBo1qwZbIHGHm6yhdc/Kgz2knnniJm9ps1pQJxbyrUJvA6GL58l35Xly5eDH/7jfTQFzRN7dvghQY6wMP8MIxjDEQeOMnCtUTDylGHvEu/jOTnwXeN7wffO/g6Y/kC/D74PXPRFed5vDd9vvp98r3ivdfe1g80nnwHny3711Vfge+sbjq5FQATyJ8A1FJTi2kPaBTWcRmnv4VoBx8nqtLFu1uZvlGVCXnULw+K0S9YFXLvF8pXlJsNgmVGQOoALi1kOscxkmcIwvA3LI9YBrC85LZl+vId1EePitTW8pjvTVZQ6Kre6wMZjbdbPoWyX2HALYnOKKct6rg0tyH3ByPI5WzmuLbDnvjafG59fXnUe3yk+F46wM9w5c+YgrzaBfQ9ZF1Fx8o2T18E874K8PwyThnUj3x/avA5kcntHeA/bkGxDMO32XraBeJ1bXijHtObVtqSMtznttNPA38L333/v7Vzg87BRKr799lvTSODiIN9csHHNgoDubChxyJbneRnOe+eaC28ZDtly0W23bt1AJYVzQ/lyU9bKcR4gp8VQKeGoA3uuOR2LX4ImdN7PhUlscPEevoiUt2tAfvzxRzNczOFDTrPhfcwTFQvKW8MGInuRqSRxXisXnnP+PhtN/NIl57fR2PRwgRwbfRyxYRiMn/Ic6uOaAC4KpzsNe96t++OPP44LLrjALEhmzzPjpAwNG45cX8BwuE6F08kYR6dOncBhbcpYw/UTHL6kLAscppFDjGwsWplgbRYYzdyGPvlz5xFOaeEzZcN/7dq1nmCY7lNOOcUoE3TkehpypWHFQbdgDXterOz7779vT0EG7Jlh3q0j/VkIeV9ffvnlYG8L3YLhe9lll5kF+3wHOG2A99EEmydWenwXOPJ28803g+GRPacxsQBiWHkZKtackpeXjK8fnyd/I3SnIsEF2nxOfCc47ZDu9n20vw++F1xvwt8Kh8sp37dvX7NZAt8tTp268MILQXkaNh4YjjUFzSd7VfnusTzgwm0bjmwREIHgCLDO+u6778AOB05RCe6unFKvv/66x8F2+ngc3BOWq/z9s8zKq25hfcZpqlwnxXKXdR4X/dpNVoItL9lG4Bo5Kjisn1hns15mJyX93CSZP655ZHwsl5h3cmC9yPnurFfY+KNgKOsolt1sN/jWBYyHhmUu/UPZLmG4hTFsx7DeYVldmPtzu4cdbLax2rZtW5xxxhnw/RdMXUBlgu0pKsOc1ssRa74vZGc7UH3DpaLG53vuueeC9RTrNb4H3nLBPm/eF8z7w7BZNwbbZsrtHeGzaNSoETjlmr8FKk9sf3IaYm55CbZtyTR6G/5Oec02LO3CmrBRKuwLR1C+mfFeld6wYUNf74DXfBiEbz2pRLBhyMY5tTf2frBBzAY954TSUJbDuuzVZ28or9ko5w5H06dPB0ceWEhSO2ZjmP6cC8dCmnMmec0GFRuk1nBKF919DadxvP3226bBT42cjeSBAweaBXFsnPHHwhfSNz02HKaDi58C8WJvj50DzwV11FjtlDDavI/h3HXXXWA6WMAyb2wwsveGBQB/9FaOvcKcf88f9KhRo8ARJTIkO/YqMKyCGFZIZFyvXj1wMTELAyqA49xRC8uP4bE3zfac22tWVjRsWNItWEMerHAoz+F69rbzfMyYMWatBllQAaQbCzf2rPGc8fNdGTZsGC+NCYYvRzu4MM3c4HUIJk/kf8QRR5h3gdO1uH6IPWZsvL/wwgt48sknvUIMfEpllZVmYN/ArlTsLFcWwnwulpm9w/d9pDLGxgnfHxpWnEwvC32OCPG9ZkHIBeFUGDl10YZV2HyyAmEYUioAcpARgYIQ4Lox9nTyN876K797Z86caTan4NSIO+64w4z68nfPDoR3330XnELlGwbL0/zqFv7+2RnG8ox1DJUKNupZb7P+ZJjBlJccsWe5RQWB5SXrZ3aKsaHI8FiWMg8Mj51q3h0bTDtHolkHMhzmh3KhrKNyqwsYT3G1Sxh2Qc3s2bNBdt5rDAsahpWncsLOWk7/5podtlNY57IeHj9+PNh+sbK0+S7wObENwmfIOiRQncf2GNshfK5sW3FTFdt5Zt8ZhmcN4+TMFsrwfaDyyk45rnPkKICVC/Z5B/v+8H0uSJspt3fEdmoznRzJYhuA7yjrvtzywt9dMG1LhultQlWvho1SsXr1apM/78U+xsE98CVwLfNnG/vmIsgDG0fczYLibNTYhpLjOGDhR3f+APgSs5Cl4sLF4HRng5wvMt1p+OOgO4eeaFvjOIGHf6kVWxlv2ypRLECtuz1nI5YNUr5ojNM7PVaWNnuxOXLDc2/DxmHLli2NExUDNkS5yJ2a+r333gvmjQ3n5557zshQIeI9vKDSRVkOgXJbOQ6H2UKGdv/+/SlmDF/eQM/LeOZx4MgEe2aoeVOMeWSjk+d8VrYC4LXjHObqOA4cJ8vQryCGjWwbH5+pnfLEiowLmNkD7j2awR8uw6ccRxo4OsFrGrLKjy9l2LtAeV/jOI7HyXEcvzxxizfy5zoGm2bewHfCcRzwuXHIl27FZTgqwx4S9vZ5x8Fn5f0+8pprhmjzHvZCUZ4FOEeAOBLIa46m0eZvjDZNYfNpf1OsiBiOjAiIQPAE8qprA4XCsogNbvaAstffdriwg8G7fLL3Blu3sGHEhiB7nNmryw4qlsMc+WCZa8NzHMeeespKxznsdt9994Ejl6yb2Tllhak0sXOM5QTLKLozfLrznIadH6x3Wcez99fWAaGso7LrAkaXw7CuK+52SY4I87mgIsB6mTMy8hHN15vvCw2nsrETko17thWuueYas/2sbwDB1gVUeqgQU8FlJxbD4XvIDmNb19DNGk6JYpuHdSndqHzSZv3JTmWe0wT7vIN5fwrTZsrtHWG7hR11TCPzzZkK+eUl2LYlw/Q2fD6O44C/d/Lx9ivIedgoFZzOw4SzwUvb27Cha6/5otrzvGw+WBaClOFLzWue+zb02CjnfHn62YXgPLeG05bYWLLXfPg8t+HxvDCGvbu8z1thsufWj/5FNRzG5FQoKmPUnm0hxh8nf3AMn/PyWMBbwx1A6M7eAE6Dsczt8Bj9imJYEXGeICsPNj75I2BcNkxWEPY8lLatMBgmlQm+H5x2Z0chqEzZd4E9ZOzJ4LQf7/t4r7fJja+3TEHPOVzLe9ibY58JbRaCfDeYLtswoFxpGv4+HOdwBc9Cl+nhlCmmn+c01t37d1PYfFKZZpgsZGnLiIAIBE8gr7o2UCgcQWejj4139iLTUI69odzpkOfeJti6hQtDWd5yVIG9uvxdU5mgUsIOLO8w8zrnonP6+9btdLOdHOzx5Qg33byN7SRjhxtHsKmY0L8k6qhwapewTmEHGkcVODODDIpi2FnLd4ablLCTkLMuOOpPJS9Q3RVsXcD6hutWOb2NjW1OXWOnIOPyrm+8026fMd1s+43n3nVRYZ+3Ddv7/SmONhPTS2Pj43lueWEbgf62Pel9bv3o5mvYMWj9i1K3ho1SYXcaCPTDZy8nCxxCYOOT0414bkwuB2rEzzzzjPFlY8ycuIdAPxjb4OE0JFckx5/tFc3hGIILas8M5ssvvwR7LFiQcviWbhxiph0KwylGgcJhj5N15xAiG9nWsMeIhTp7cbwLAKts2PsKa/OF5Y5MLARYmbCHhEOahQ0v2PvYI2UrGQ71suBjRcQCj2FQoWWhx3MyuPPOO8GpTlzvQrdAJje+gWSDdbPPhlPE7DOxNhUfDoGyVyHY8IpTLrffR27u7Jm06SlsPq0yHKissGHLFgERCEwgr7o28B05XblGimUlXTl9lba3sb9ruuVVt3DEkw15dkBQlr9rdjJxugp7nukWjLH1e151OzvGrJx3mLmV3yVRR3mnJ6+0l0S7hKNFbIRyeo83n1Ccs0PTdsyxBzzQzk/2ncmvzqOSwqnqVEaZNk4JYto5dYdTbenma3Kri7zlCvu8A70/xdFmsmkNJi9FaVvyN8i4ilK3ho1SQY2TmeHQC21vwyEgfh2ZbtxOktMqeJ6bYcOFc/85R48y7BGhTUONnLY1hMgXndec7kG7JAwbhWzMcpHRcccdZxYucViWc/ztVKBg0sHGb15y3qMs3nJclGuvOSWMcwp9DRv+XPBr5fKLy8rlZbPg4lx/PkMqFCxMOPe+IIoUG9hMa17x5OZnCzf6c3qX9zXduBicNs2rr74KLv6z2jvdfE1ufH3l8rv2zpN9NhzmZD59DdPFRYX5hRlu/r7pKWw+OaWPYQVTwFJORgRE4DCBvOraw1J5n9mpJmzUeTeOeZf9XfM8r7qFjS/2OrOjkLs9cSSd9bzjOOAH0jhlhmEEMt7lpa3ffet23mfdWE5zag/dvA3dva95XlJ1lE0347Tp5DlNSbdLOP2bi+QbN27M6ENubKcwA+YIDW1vY9+Z/Oo8KrFcxE8lkSNcL730Ergehm04LqL2DjPY86I870DvT6jbTMHmw8oVtm3J3yJZMJyi1K1ho1TYHzwLKWbK1/DjHGyM0p3TZjjExPNA5qGHHjI7WwwaNMh4c7cAKia8YE8IbWvYY80fMK/ZyKVdEoZKD3vqOVLBQpkLpDnlhjsgBYqf2j7dbVp5zvmKdtiQ1wUxfPHt2hIujvK9lwvROX+PvUkcKaK/b48J00JDv2ANCwU7vYlTsezoR24jFexVc5ys6TV2uJLPjBVSsHF6y3GXBQ7z0Y1hs+ef59Z4uVmNAAAQAElEQVRwgb+3cumrdFi5otiM13FyzxPTwPC5oNL+yHlNwyFkTkPjO8PrSDaFzadtbHCUKZLzr7SLQGkQyK+uBZBvsthwsUJ2nRSntHK0Pdi6hYtxudkHOxK5Ro1TSNgw5Egsw+a0XNr5lZe23mb9xA5F3mONre+5AYita6xfbnZJ1VHh0i7hwnVuhsPZHbkxKap7oPeFYVKRZHsg2LqAU/C4noLtOU6F4rpOvkeO48C3rmT4wZiCPu/8wgx1mym/+Hz9C9q2tPfbepXtH363yroX1A4bpcLOFeMuS4Eywd5i7tJAjZZDVd27dwdfMA6XWXlOIeJ2ctymjqv9LRgOlXGnADbMqZywAc97qMDwpeQ55xKyQU9tjQ03O8WKBR4b7yys6G4btnyBeU13Pgzbi093po9h8j7ez3OGR3nbCGfBzlEJrnfglByOUjBPXHREpci30Uh3hmMLcJ5zKJDp5Tmn6ZAFh62808OCmfHa9FGWhsOt7A3ivDxq/mTLBcwMj6zYw8TFS5RlPJQjV+7kQzf2rHCqkM0f02V7kOmfm/Fe+Ms4KccwuFCOcfCahZxlSMWHIzp051xd8uWiPvaq0K2ghtxtvrggjYqddxjsebBD782bNwe3JPT253kwfJknPn/KkymfAc9p8ssTdyRh2ng/lSCO5vA+7p7EKWl8dlwLRLdgDNeOMH4a9ujYe8iYbjR89tadNq/pzveW13y2fIdsXqy7zSd/B5S34fN3wmu+j+xV4ggjw+H7T3f6FzafLCOoGPL9Y5gyIiACwRNgPcMyiPUgf4fed/L3y98nf+/Wnb97urHstW7s2LDnduE2NwThCALLp2DrFpYLbMyyrGN4TA93FmSdaHvNmda86gBOh2E5zfqPU1qZTpYzXPvBTU9Yr3CKLcO39SHPaViWM288tybUdZQtIxm+LT95XtztEsYRjCEb1iesW4KR95XhGh0yZN6sH69pbD3BDmHbHmNbgffQsK5l/VaQuoDvGTtCWecwPnYw89wqJnx3WefRj4ZKC98HvmuMk240TB/dCvK8g3l/GHZB20y5vSP554WxATYvvGIbpyBtS95Dw3qVNmdn0C6sCRul4vTTTzd54IJUcxLgwEKGBRgXiHF+PLdjoxsVDmrCXOTFRiELJb7E3kFwMQ+3LGVhRQ2Xw7fcEo8FJhfjsoeE8lz0RU2TLxt7NjhFhjsGsGHEYWMOudGdQ3jU6Pgy0J+7BdGdBTXlqBRwfiLXJtCdvS0MlyMTjMc2bPmSM738obEHngvKHnzwQXBolO6UpWFhya1m2RPEHlrmm4oEf5T0525UdGN4TA+3jmW8DJfuVAgo523YMOf2bkwLlTHOk+UPjCw47/Gkk04y4vxWBHvNuYiN3MmNQ440jINCrETYQ8XzvAyVI07f4SIrKi5kxQqBW81xa0HGT6WKYdtwODTLYVEOdXJYjr1afIbWv6C2neJkbd/7OTrB94j+jpM1ouAtEwxfvm9cTEY+bFjzh87Cz4aTV55Y+PI5P/DAA6AiQeWG+SZfKtWsKJk+G1Z+Nitdxs/3gHOc2cvDdHFxGt9hzgvlYjrvcNhzxXtYIFOWSh+55Pb7YEXO8FmYUZ6/Y77vfB/Zk8hnS3fHcUB3KoaFySeVWc6dZScEmXinWeciIAL5E2BdxLKdHQe+v3uWD/x9ct0Ef68sK9jJxN82G342dG5nyfKDCsSzzz4Llk2sA7lgljLB1C0sw1j+s6HG6Zyss1keUbGwHU4Miyav8pIdQyxPWN5y0w9es03AXQ2ZRtaBrDMZDhcjcyc/5o2Gm3SwPqOfNaGuo/KqC+hXXO0Sm5+8bLZfuO082yp8HnnJ5ubHspido3yX+L6QK+tvvkfceYv3kTHZsy5jXcGp6axH2TFEJTfYuoBpZHysm/nOcD0OZxuwfuI7y7jYluC6AqaD7zrfKa674O6J3OGL7jRsT7GtVZDnzTzk9/4wDQVtM/E94PvLdHm3FwqaF8bN9hxt1r1sQ/L9z6ttSVkatvFoc8SQdmFN2CgVbDTyReMPjA2Z3DLEl4qjCmwMUxvltmIsUNiYJ0QWPixUAt3Pgo/hU/sjbCoUbKCwYHGcrMYjHwgLW774NNS+OQLCMKl1swCkO236sSHOhh+v6c4CkVoxfzxUIHht3RkufwBscHFHDfa+sBCngkLDgpQfDGLhyrzwh2DzwfC4UxF7kthwY5rYkKfWTl7s6WF62ChneijHeGmYBi46tmF52/xhsyDm0DUVIfbccMiYPzRvOaaJU62obDH9HOXhMCTjYo83G3tUbLzvye2cw9v8QiTv4Xxajn7wmdKdGjd7HZgfez+fG38YjJuFFD/OwvfA+hfU5g+V7w0rvkD3skDkHN/c1nkwz/nx5bO0MuTPd4KjUTa+/PLEnngWeHwWzDd5Me9UDtlzZ8MJxmZByPiZDr4PfO9p85rvDAsxpsc7LCrovIdyNHx3+T7n9vtgpcDw7O+ANu/h+8geEMvChsXCnfEVNJ989vwdFrXgY9wyIhCrBGxn1BdffJEDAX9X/N3y98vfqndZwc43K8zGIxujrEs5ssx6iz3O3uVIfnUL42I5zN80w+HaOk5rZd3OjiMbF22Gm1cdwM1WuDHL4sWLwbqTaWEdxTqSChHDoOGaDeaPeaOx+aSft2FdxDI3FHVUMHVBcbRLvPOT2zk/vEcerH9zk8nPnXW4Ld/t+0KuDJcNens/FQi2W9jOYJ3EDlq+Q9Y/mLqAsxX4vrAOZrxsC7Ht4t2+YceXTQ/rJKaD7zs7JXnN507DcyrEjD/Y5x3s+8MwC9Jmyu0dKWhe2DYraNuSaWWbi+1AKvnsuKRbYU3YKBXMAPffZyOHPSO8zs+wIcMGIIdi2auan7y3PxdI8yX2diupc4628EVnLwo1Wqafho1c9gbxgzFMC0ciaHsbDpkyr9TA6U6lhho7G5osWOlWWMMGYH5hkBnZ2TjYW0zDZ0E/6x6MzXt4bzCylOEwNkeYeF4U4ziOWdyVVxhU4izjvOSK6hdMnpgOKrVFjSvc7w8mn6wE2CvG306450fpKxECiqQQBFjvcM0hFQU2nAsRhLmF5Rd7mjmyzrrJOAY4BKpbWNewDqA4lRQ2QO013QIZxpdfHcCwWC8Gur+gbkxPSdZRrFsLWo8WNE/e8pz6xFkiHK32di+uc8dxwFELNlw55Sq3eHKrC7yfK9PMtk9ROhh94y/o8/a9P9A1nyefq/Xj+0TDuOhn3UNhF7Ztyc5tKtEcLSkqz7BSKjilhcNA7AGn9hkKyOEYxplnnmk+QMcGErVD9ggzndQWueiIU534o6J2TXcZERABgOuk2BPJhXXelYvYiIAIFJwAN+Ng55btxCp4CLojkglw5gGn5HKKdsnkQ7EUN4HCtC3Zkc9ZEWx7U6koahrDSqlgZrgWgHsOc36cbWzTPZoMR1c47YpDaSzQ2SvO+YfUvDlXkFo857f5TkGKJgbKiwgUhACno/H3wq2IWT4U5F7JioAI+BPg9Awq6lQuuIbOX0Iu0UyAoxRsi3DEKprzGUt54/MsaNuSu3xyxIRrlUPBKuyUCg6hsieSNtdHhCKT4RgGh3qfeOIJcM48P37HkRmujeAQFKd/9ezZM+hkS1AEop0Ad2njItCi7kwR7ZyUPxEoCAHu+MO1e5xTzrVKBblXspFNgOtX2KB0nKz1pJGdG6XeEihI25JrebiTFmfMcHdUG0ZR7LBTKpgZzjVjRvllTV7LiIAIxDYBLuLkbmERTkHJF4GwI8CNK7h+j72VYZc4JajYCHD+PZXJYotAAYc9Ae6kxY5srtMNVWLDUqkIVeYUjgiIgAiIgAiIgAgUjICkRUAECkNASkVhqOkeERABERABERABERABERABD4ESVyo8MetEBERABERABERABERABEQgKghIqYiKx6hMiEDICShAERABERABERABEQiagJSKoFFJUAREQAREQATCjYDSIwIiIALhQUBKRXg8B6VCBERABERABERABEQgWgnEQL6kVMTAQ1YWRUAERCDWCGSmpyMzMzPWsq38ioAIiECpEZBSUWroFXEICSgoEYhaAnOuvBIzL7oYsy69FLMuuwy0Z158MWacfx6SZswo1XyveOYZTB44EH/17o3ZQ4eGJC1FDTN56VL8M+Rk/FajOkY3aYrdM2eGJF0KRAREQAREIG8CUiry5iNfERABEShVArUHD0btk07C5h++x4aPP8aWX35C7cEnoc4ppyGhdu1STVvVHj3glC2D3dOmIXnJ4iDSkr9IUcI8tHs3/jnxROyePgMDFi9Fh2eeBpWy/GOVhAiIgAiIQFEJSKkoKkHdLwIiIALFSKDBeeeh8dDLEVc2wcRSrkYtc0238k2aGLfSOtRyRykanH9BSKMvSpjb//gDKWvWoHqfPkisXw+VOnZE8sJFIU2fAhOBmCCgTIpAIQhIqSgENN0iAiIgAiIQfgT2uwoFU5XmjljQXv/Bh6h14gk8lREBERABEShmAlIqihlwgODlJAIiIAIlQiB1105sG/0nVr/2KjZ+9RWSlyzxizdt715s/vZbrHzxRax95x0kTZuG9JQUPznrwIb7ll9+wTq3wZ7fmo6DW7dh66+/YtXLL5v4U9autcHksAsSZo4bfS4S69c3LnvmzsW69z8w08U6v/KKcePhwOYthseOCROwfexYk1e6y4iACIiACBSdgJSKojNUCCIgAlFJILIztWbECIxp1gILb7kNB7dsNY37cR06Yvbw4Ug/eNBk7tCePRjbpi2mn3MOMg4cwI7x4zF54AD80aAeto4aZWTsIXXXLnCh+JgWzbHi6aexZ85szP3XlVh4221WJIdNmdHNm2Lh7bchdds2rHYVC8ZF5cUKFjRMe19udtVevYzXoR07sOA/N+OoP/+A9xSxnX9NwMzzLsDkfv0w65JLsPL55428DiIgAiIgAkUnIKWi6AwVggiIgAiEFYGtP/+Medddh3J1aqPP35PQ9qGH0O3dd9H+icex/v33MfWUU0x69y9fjoObN5nzhhdfjO4ffoi2/3sUaUl7MOuii5GZkWH8eJhzxRWm57+h636028vf8bnncOzMGag1aBC9c5gNn3+ORXfcgbJVq7vxT0bbhx8G76nSrYvrfhv2rVhh5AsSprkhn0Ol1q2RmL3OpGLLlqjUvn2OO7g+pf5556LmgAEYtGYNenz6aQ5/XUQoASVbBEQgLAhIqQiLx6BEiIAIiEDRCHBqE3c/YihLH32Mlmnwl6lY0ZzzUPeMM2hhx+jRSNmwEdxpqcMLL6DLO++ifOPGxq/2CScY+9CunZ7pUnsXLsSW77837o2vvBKO45hzJy4OVTp3NufeB45S8LrOkJNRtkoVnsKJj0f9885HZmoaNn72GQoapgkkj0NGaipmXX65qxDtNlK7Z8zAqhdfNOf2wKlYG7/8HN3efx9xZctaZ9kiIAIiIAIhIJCfUhGCKBSECIiAAbtfRAAAEABJREFUCIhAcRP456STsHvWLPCDb3vnzzXRla1R3dj2ULb64esdY0Yb53quorF3wQJM6tsXfzRoYL5/YTzcQ2ZamnuECdecuIdEV8a1cv3j6EbygoXGf/OPP2Li0cd4zOrs9Q2pu5IKFKYJLI9DWnIyppx4IjZ88jGOnTUTNfr1M9KL773HE8++lSsxe9gwdHrltRxTooygDiIgAiIgAkUmIKWiyAgVgAiEAwGlIZYJcDpRyurVZrTBcRyUrVrN4Mg8lG5se8hMO3zNc65vGNuqNda9/TYaDR2KgW7D+8hRf1hxj51Qt47n3NVaDp8HOOPoRWLDBsan4QUX4OixYzym/6JFODklxXw/oiBhmsDyOKx7913sHD/eVRaaomKLFujq5qdszZrI2J9ipnpR2Zp68smof845aHTJJXmEJC8REAEREIHCEpBSUVhyuk8EREAEwoAARyaWPvSQSUlio0bGrpU9hSnZ54N0nus4B1V798LSB+9HZnoaml5/HZpedRXiExNxaOcOE4b3oUafvohLSDRO+5cuNbY9MH57bu1a2fHvnj3bhBnvhmsN13ts+PRTFDRMG3Ygmzs50T1t9x5X58lExVatcNTo0ShbuzYObtqEv3r0QIWWLdDxpZcoZkzy4sXYNXmyOS/RgyITAREQgSglIKUiSh+ssiUCIhAdBLhOgrskZWZmLZrOzMgAr82OTj/9hFkXX4INH32EcnXqID4h6wN5LW+7DYmNGmPbqF89uzgxnKUPPmigNLv+BlTp2BFlKlY21/uXr3CVi3QwbO91CJwWxe1l48uXR+v77jWyy5580rPl7M6JE7Hm9RHG/YDbeE9Zv96ct7rzTlRs0wa7XP9VbkOeu01lHDoEbms769LLwGlYBQ3TBJzLofaJJxofrgNZ++ab5jyhdh3UHnS8Oedh94yZ2DZyJDLS0pDqKk6zr7jC7IhFPxkREAER8CWg64ITkFJRcGa6QwREQARKjMD4Lp3xe40aSEtKMnFymhOv/6hXF1NPPx0bP//MuNuF1ryo7CoMfadORd3TzsCc4Vfgt6rV8EfdekjduhVd3noLHbMXMPf86ivU7N8fm775Gn/Ur4+JRx6JxIYN0enVV8DpQ7MuuggzXcMwW99zD7p9/DEOJSXhdzfuP91RkblXX+0qD63pjQNr12F048ZI27cPFZo1Q59Jk9Bo+HCsfu11jKpRHb/Xqo1ljzyCnm566w4ZYu4pSJjmhlwOza67Du0ef9xVVmpg3rXXuvHVxJgWLVzlYTuOHDUK7Z95BmVdhtPOOhujqlfD2FZtUKF5c7S8445cQpSzCIiACIhAQQlIqSgoMcmHgICCEAERCJbA8WvW4tTMzHzNsdOn5wgysX49HPH11zhh4wYMXL0SJ+3dg/4LF6KJ1+5N1Y8+GkePHYshBw6i3/wFOHbaNLP9bLP/ux6Dt283cfb6/ntPuFyP0H/+fJy4eQv6L15swjvS7f0/wVVWTtqzB0Pc0Qi721S5WrXAbWwHLF6EEzZtxglbt6Df3Lmoe+aZnvB4UpAwKZ+b4ejIidu2unldDW6jy/weNep3cBSj5a23YgDXcyTvRd9/puKEzZvQ45NPUKZSpdyCk7sIiIAIiEABCcQVUF7iIiACIiACEUagXPUaeW6hyu1VE+rU9s9VLi6cumQb5GUqV0ZC7dqgHVemTMA7uK2snZoVUMB1LGiY7i1+f9y2tkLTpqjUrl3A/DKOyh06IK5cOb975SACIiACIlA0AlIqisZPd4uACIiACIiACIhAqRJQ5CIQDgSkVITDU1AaREAEREAEREAEREAERCCCCUipyPfhSUAEREAERCBWCWz55RfM+7//A7/psXXUKOxfvRqZmZmxikP5FgEREIFcCUipyBWNPERABCKKgBIrAsVAIH3fPuydPx/LH30UU086CWOaN8evlSpifNdumHH++Vhy//1Y/8knSJo+HWnJycWQAgUpAiIgApFBQEpFZDwnpVIEREAERKAUCDRwFYdjJkzAiVu34sSdO9Fn8mR0fvV11DnlZGSmpWHT119jzhVXYGKvXvitcmX80aAhJg8ceHh047ffskY3MjKgf1kEdBQBEYhOAlIqovO5KlciIAIiIAIhJlCuenVUP+ooNB42FO0fexxHfPstuE3vkP37MWDZMvT+6Se0uPUWVGzVKmt047HHMPXkk7NGNypX8oxuLL7vvsOjG3v3hjiVCk4EREAEQkKgwIFIqSgwMt0gAiIgAiIgAocJcCtbKhJ1Tj0V/CZGlzffhBnd2LLl8OjGayM8oxubv/v28OhGlSqe0Y25112XtXaDoxurViFToxuHIetMBEQg7AlIqQj7R6QERiUBZUoERCAmCHhGN4Zefnh0Y/4C2NGNXj//7BndSF60EMvt6EaLFvi1QkWM79IFM84/D2Z04+OPs9ZuaHQjJt4dZVIEIo2AlIpIe2JKrwiIQMwQ2Pb775h2xhm5mulnn41F//0vNnz6KVI2bAzIZc/8+WaO/6zLL0ek7Vq04plnTNr/6t0bs4cODZi/gjoWNMyChh+svB3dqHvKKYdHN8aNx4nu6MbgXbvQZ8oUdH7jDdQ97VRkpqXDjG78619ZazfM6EYDw8aMbrzwArZqdCNY9JITAREoJgJSKooJrIIVAREQgaISqNCqFeqdfQ7i3R7rLT/+CJoKLVsat3pnnoXKnTph96zZmHXJJRjrui996CE/xWHViy9ix9ix2PDRR8YuappK8v6qPXrAKVsGu6dNQ/KSxSGJujjCDEnCvAIpW60aqh95JBq7oxvtHn0sa+2Gz+hGy9tuM2s3zOjG409krd3IbXTD5Zem0Q0vwjqNQgLKUhgQkFIRBg9BSRABERCBQAQquo1ENiyrH32Ux7vuaaeZxmbj4cPQ9uGHcdTvo9D1vfeBeAdLH3wQC2+73SPLk/qnn04LZavXcJWQzuY8Ug61Bg5Eg/MvCGlyiyPMkCYwj8Cc+HijSHB0o8Utt8Cs3TCjG5vhGd14M3t0Iz3DHd34DnOvvAoT3ZGe3+zoxoAB8Ixu/Por9mvtRh7E5SUCIlAQAlIqgqT16quvombNmjlMrVq1ULt27RyGblEnl51v5q2w+fVlwutA4dHd10iudo73zvIpbi7nnXdekL8OiZU2gcbDhqLVHf81yVj13LNIcnumzYV7qOMqIYPWrcPxGzcgoU5t10V/0UjAM7px+eUwoxvffIP+8+fj5H3JGLB8Obh2w4xutGkDz+jGkCEY4yquZu1G585ZazfuvRfruXbDfYcO7dlTYqgeeOABv3IuUBlHN1sGWptuha2beK8Nx9p08w3P+nnb0SBXo0YN5Gaqc7czLxNOcr5p43Wg9NHd14SL3OXubzU9Pb3EfmMlEZGUiiAoP/XUU7jhhhuwc+fOHGbHjh3Yvn17DkM3yflz8WXCa7Ly5Ud3XyO57TneO8unuLl8/fXXuOqqq5CamhrEr0QipU2g4aWXwqZh9SuvmFNOednq9kYnTZmCjZ9/gUO7dxt3HlLWr8f6jz7Cyuefx7oPP8S20X9i+9ix9Mph9i5ahI1ffYWVL7xg5HzXbuyePRtr3nrL+G/9+WekuApMjgDci+1jxmDzt9+a+w9s3gKma+OXX2LvwoWub86//WvWYMsvv2DdBx8iacaMnJ4+Vwe3bgPzt+rll00aU9au9ZHIuixImFl3RM/RjG60bAnP6MYbb+AYO7qRlJS1doOjG+6IViZHN77/3jO6MapqVfzRoAEmc3Tj2muznrH7Pu1buRKh3JnqXleRefjhh/3KuUBlHN1sGWhtuvnWJXSz/tamW2HlbBjedqDwvP3teTjL7dq1C7mZJPf98DbhJOedLnseKH3Wz9sOF7mP3PJ32LBh0VPYuDmRUuFCyO/vxx9/9Ij069cP3maAW9h6m/79++fwpyzdvGV4Tjf6eRu60c/b0M1bhud085bhOd3o523oRj9v4+1vz8NVjr0LHvDuSaTmI9w5B0qfi9v8vf322zh06JA51yG8CVRo3hyIc0wid8+caWwqDnOvvgYz3FGnOcOHIWXNGuO++bvvMKFbNyQvWYIylStj54QJ+OfEk7D+/feNPw8Ht2zFVLehOb5DB3BdxsGtW7D4v3didJMmpiFPmeVuh8tf3btjwycfG0Vh0d13G//p55yD9P37KWIMF5NPv+B8zBk6FKtffcXEPfvyoRjfsSP48TgKpbqNm1mXXeb2nDfHiqefxp45szH3X1di4W230dvPUGZ086ZYePttSN22DatdxWJsm7ZY+eKLHtmChum5MUZOyrpKg1m74faYtnv0URzhO7rhKnctb78dFTm6sXgxlnPthju6wfU7fqMbbgOJI2SHCjG68cUXX3iI2/KIdrjWTUwbTRHS59dOKI3wPNCzTwZ07gJr+nXuBF/T38s/FuRsHr1tXya89va353T3NdaPdjZyfOP+5ux5NNhx0ZCJkspDA7fHZty4cRjnZca4PXDeZuzYsTn8KUs3bxme041+3oZu9PM2dPOW4TndvGV4Tjf6eRu60c/bePvb83CVY7rss33UrfAiNR/hzjlQ+oYPH27Ry44QAk5cHMpUqgL+2782S3mo3L49BgRY4LzcbbQnuuUZ12Q0ufJKdHWVx0aXXgI4Duy/WZdfiq0//YQGF16EPhMnmu1Qy9WuBWSke6ZX7XTdKV+xZSu0ue8+HDdrFsrWrm1GJda8+Sa9jDl22jTUHnS8OV/+v//hCLc3vP6555jrXf9MNfacK67Aho8/RsOLLsbRbjna8bnncOzMGag1aJDx9z5s+PxzLLrjDpStWh19/p5s1pbwnirdurjut2HfihVGvCBhmht0MAQ8oxuuAtHiP/9BFzO6MQ4nbtmMwW7vdZ9//kFnO7qRkYnN37ujG1ddbdZumNGN+l6jG+5IGEeTghnd6O4qqLY8os06YIxPHUt3XyO5MX7tDjLKj8uoUaPM8+bh7RtvxJjHHvWYcY89Dl8z1svfyvrK8Dpa5GwevW3mz9d4+9tzXxleWz/aQ3r2JPaoM9GnVETdI1KGREAERCA4AukH9hnBhDp1jc1DfIUKQPYIBq9p0pOTsXfePIzv0hkLbr0NGz77DK1dpaDtI4/QG7td5WD773+Y8yZXXWlsHrgwmHJNr72Ol+jgjlS0uvsutLrnHnPNxmgtd7SWF7smT6blMfEVK5rzqm5lWqVzZ3R49jl0c3u3W993r5kGtcVtmFKgsavkOI7DUziuokRZc+F14CgFL+sMORllq1ThKRh3/fPOR2ZqGja6+eHUqoKEaQLRIV8CZnSjd280tqMbX3+dtXZj/z4McJW5XhzduMMd3WjbFsnu6MaKp57GVFc58R7dmH7uuVh8771m+t2uqVORoI/85ctdAiIQCQSkVETCU1IaRSAKCShLoSXAaSdsUDPUKl270srVtH3oIcRXqojkBQux6rlnMeviizGhZw/snj7d3LN3/nxj8/Egq0gAABAASURBVFC+SRNaxlQ/6igzKmAXfFds3RqVO3ZyRwfuwBj3fEyrVtj+52gjm5mWZmzfQ9UjjjBOCXXroNGllxqlgEqMcXQPHEFxrVz/OJef6abA5h9/xMSjj/EYu5YkdVcSChImw5IpGgEqgNytrK6rQJjRjREjcIw7qn/Cpo2e0Y0ub72Juqdn7Ua25YcfMNcd3Zh05JF4YuVKfOjqkcOXLcdcrt1wRze2jByJYEY3ipZq3S0CIhBKAlIqgqCZkd2LYu0gbgkrkb///htLly4NqzTFQmK4GOy3337DgQMHYiG7ymMpE9jxR9bIApNRa9BAWrmaWiecgAHLVqDjSy+h4SWXoJw7spG+Zy9mDRuKjEOHkNCgvude77URHkf35OC2bRjftQv4jQyOfPT++WcMXL4c9c452/XN/S+hXj0/TyoYHsfMTM9poBPHHb1IbNjAeDW84AIcPXaMx/RftAgnp6SgwzNPoyBhmsB0yI1Akd3t6Eajyy6DWbvhjm70c0fKTs4e3RhRrz6+cR/7toSy2LdkCTi6Me2UU8y3V7h2Y1ynTjCjG+6I2LoPPwRHN7w3HShyAosYAKcarcler1TEoHR7AQhsdWV/c4376rjHyPqzac7Mp7yLrFwBUioi7YkVIr1pbo9hpCpEhchu2NzCwoLsaYdNopSQqCRwYOMmLHk4a+pS9T590PSaa/PM59g2bbBz/Dg0v/FGdP/4Y7A3udHQoaBiwR2VavTpi7gK5U0Yu9xOCXOSfdgzdy5WPvec+ZgeRwziEhLR44svUKltWyNxaOdOYxfkYOJzw+E9+306QAL9fqgUUZY7T8UnJsLbcAeqDZ9+ioKGyfBkSpaA4yqIHN1YWD4RP7hR/9CwkasgjjXv4+Ddu9F36lRwdKPemWeC//jxx3lXXwOOboyqVg1/1G+Av/v3x9xrrgF3MTOjGytWhHRnKsabn+FmFtG2NWh+eQ4Hf27GmuYmxDbQ3dOI+SueNJd+9qVUlP4zUApEQAREICCBjNRUcAejdLf33Qqk791r3FJ37TRTfLggmmsj9s6dg2pHHmmUBDbWKJ/hdiik7toFZGRVYWl79piRCE5N4px2z/avjgOwgdemNcq7owBspLd96GHXzcGS++8HG+8ML2XtWkw74wwcSkpCfOXKdDLh7V+1ypzvcXuf7fSnAxvWgztI0SMtORkZ2SN2tJkmb2Uhvnx5cG0FZZc9+SRsfrkQfM3rI+iMA5s2gbtZ8aLVnXeaHYl2TZyIVe5oS/rBgyYd3Pp21qWXoWz16ihomAxXJnwIcK1MtV69YEY3uLjfZ3Sj98iRaPnfO1CpXTvscxVRrrMxoxutWsEzunHOOVgcpqMb4UNaKRGB0BGQUhE6lgopigkoayJQGgTWf/wJfq9RA4vdRrSNn416uv1eoyb+OqIXVjz1FKofcwy6vvse+kyejArNmllRJLk9vX/UrgOOJnANxeT+A7DNbYw58WVQsWVLTO7fH+O7dgM/gLbXHYHo8eVXnnv5obQj3GuuqZh01NEY5cY36di+aDxsGFq7ikbj4cPdRt1/Ua5WTfzVswcmdOuOBTfdhN6/jgQXY++eNRN/NKgHbm/L8LeN+h1MA7eU/b1ObaRu3+6Jiyet3cZfN3fUhArL7/Xq4s9GjTD36qtd5aE1vXFg7TqMbtwYafv2mTz2mTQJjdw0rH7tdTdt1fF7rdpY9sgj6Pn5Z+C8ft5UkDApLxP+BKgwc3Sjzskno8XNN6PLiBFZoxsbN8IzuvH2W+DohhMfhy0//QTv0Y3f69Y7PLrhjrh5Rjei7CNk4f8klcJoJCClIhqfqvIkAiIQFQSaXDEcp2Zm5m7S08w6ht4//ojGw4fBcZwc+a7hKhunpB3CkAMpOHlvMnhe94wz0H/JYhz522/gl5ZpHzdnDo6dPh1VfRZ413N7eul+0t49GLhyBY5fsxZtHngAcWXKGNP+iSdw4pYtGLxjJ46ZNNE07hgn7znl4CGcmp6Bqj164MStW03cTAPNqYfSkFC7do608qLRJZeYnYRO3LwF/RcvRv+FC3GkqwSd4N5/kjvKMuTQIZTJ3kWqXK1a6PbuuxiweBFO2LQZJ2zdgn5z56LumWcyKI8pSJiem3QSkQQ8oxuXXop27uhGT1cp7ucqy1y7MXDlSnB0o9Vdd6JS+/ZZoxvPPAPP6EbFSjBrN9x33jO68c8/CKe1GxH5UJTomCIgpSKIx+04WRW142TZQdwSViJ9+vRBmzZtwipNsZAYfrzvpJNOQmJiYixkV3mMIAJsfDG5juMgsX492Gu6BTJxZcuibLVqgbyMG/1sY984FPHAqUtlKlUyofDjfFRAaFOZMY4+B6Y/PiHBxzXnZUHDzHm3roqTQFxcVlPE2qGOi6MbFZo3Rx07uvH660YBPsF3dOOss5BjdOOoozCqWjVkjW70g1m7wdGNX34x30LJzB7dGDBgAJo1axbqZEd5eEXPXl03iJNck/X2uCcR9GfT7DiR2a7MDbXNV27+cncJOE7WQ3ecLNt1iqi/+Ph4FFdhHVEgSjixjuOgjNuj6ziR+d6UMC5FJwIiEKMEHCerjHScLLskMVAhNWs3OLrB6XO+oxu//gp+i6VS+w7Yt2wZVnB049RTMZZrN8zoRkfMvuACLL3vXqzjzlQc3UhKKsksxGxccW7Oy7hGf+FDgM8kfFKjlISEgAIRAREQAREQAREoPAHP6IY72tzi3/9GF45ujBkDjm5wKl7fadPQhWs3zjobjttxt+Wnn2HWbnB0o3r1w6MbV18N7pa2haMby5fDjm4UPmW6UwTCl4CUivB9NkqZCIhAdBNQ7kRABCKQAKfiVTviCPDjje3M6MaXZj2PWbuxahV6Z49uVO7QEftcRWLFM89iGkc3WrfGyAoVMK5TR0w3azfuxroPPsSuYhjdSD9wAJ7d3SKQsZIcmQSkVETmc1OqRUAEREAEREAESoRAcJGY0Y1mzVAne3Sj82uv4WgzurEBdnSj67vvop4d3fh5JOZdcw0meUY36uLv/v3AXc9WPPssijK6wW2hlz/5JLh1NLd0Di4HkhKBohGQUhEEP/vhOGsHcUtYiUycOBFLliwJqzTFQmJ27dqFkSNH4oDbYxQL+VUeRUAERKAwBGzdGqkfkBvjKg6rV6/OM+ue0Y1LLoFndGPObHhGN377Da3uvhtmdGPFCqx89jn/0Y2zz8aiu+/KGt2YMgWHkpLyjLOtO4qy/NFH8Vvlylj79tvIzMjIUz7SPPlF7ZFuojNdE2l/Ns3e3+uJtDwESm+pKxWBEiW30BJggR1tL25oCRVPaGQu9sXDVqGKgAhEDwGWlZGcGypDLOsLkwfP6MbgweDaDTO6MXo0TtiYPboxfTo4ulH/7HPglCmDrb/8inn/dy0mHX00Rpm1G+7oxnHHeUY3tv78s5lyxbUb5Vz/Tu5oCdM196qrwI9kbncVIF5Hg0l3M0E1yTbQ3cuI+YvENAcDV0pFMJQkIwIiIAIiIAIiIAIlSMCMbvTsCX5rpe3DD6Pnl1+iH0c3kvdhINduuKMb/MBj5U6dsM8d3Vj1/POYetppGGvXbnTogO1//IHyLVqYVCcvWoIpgwZh6slDkLx0qXHTQQRCSUBKRShpKiwREAEREAERKFUCijzaCTiOgwpcu+GObjS/6SbY0Y3j16/PWrvhjm50e+891D/3XCAuDqlbt2QhyWDfPrD1t18xrm1bpL3yCipm+egoAiEhIKUiJBgViAiIgAiIgAiIgAiULgE7utHw4ovR9Lr/w+ZvvkG6O7LBVDnlytHymMyU/ejpudJJiROIwgilVATxUO2H4xzHCUI6/ET69u2Ltm6vRPilLLpTVL16dQwZMgTly5eP7owqdyIgAiJQBAK2juWHWosQTKndOmjQIDRv3rzU4g8U8d6FCzGx1xEerzqnnoo2DzyAI//8A4OTknBqZibK3n4HJngkIu+knpvkIa6JxIasTbPjRGa70sUe8M/mK6CnHHMScJzIfPgssB0nMtOe8wkEfRUWgo7juCPP+olB/0RABEQgDwKOE9n1U7jVsQe3bsPm775DpxdfxKC1a40C0funn9D67rtRe9DxKFu1ah5PI3K8+Naohg2v56XnEV7PQ6kRAREQgRgioKyKgAiEmkBCndrgAu5655yD8o0bhzp4hScCuRKQUpErGnmIgAiIgAiIgAiIgAigkAiu3l0Zw5KqRKx5bb+mLxfk0UupKAgtyYqACIiACIiACIiACARF4O2UCvjwQHn8UqZWxBmme3xqzsXtQWU6hoWkVATx8O1HbawdxC0FESl22QkTJmDx4sXFHo8iyElg165d+Pnnn3HgwIGcHroSAREQARHwEODH43hhbZ5Hkvnzzz+xatWqSEpyiaW1rJOJtg0boFOzpiE3Z9WvgHcrrUTnpk1CHjbTW6NK5WLjZD9+F+kffvQFJKXCl4iuRSBEBGxhYe0QBatgSo2AIhYBERABfwIs42n8feRS3AS4WBuwTfTiji104UdeioPLu5SK4DhJSgREQAREQAREIBIIKI0iIAKlQkBKRalgV6QiIAIiIAIiIAIiIAIiED0ECqpURE/OlRMRKGYCjpM1MOs4WXYxR6fgRUAEREAESoGA4zhwHKcUYlaUWdOIIo995KU4uHdNSkUQnPhhG4pZm+eRZI477ji0a9cukpIcFWnlF7VPPfVUJCYmlkJ+FKUIiIAIRAaB+Ph4k1Brm4sIOhx//PFo3rx5BKU4OpK6LKE2nqs1AJkRqNBZpcJx7Fl0PBMpFdHxHJULERABERCBSCSgNIuACIhAlBCQUhElD1LZEAEREAEREAEREAERKB4CCjV/AlIq8mckCREQAREQAREQAREQAREQgTwISKnIA46vV6TuQ82P9oV32n1JR8c1mZN9dORGuRABERCB4iHAsrJ4Qi6ZUFnOR3oeSoZUiGPJzER8ZkaIA1VwRSEgpSIIeiwwKBaphcbEiROxZMkSZkGmBAnwi9ojR45ESkpKCcaqqESgGAkoaBEoBgK2jk1PTy+G0Is/yNGjR2PVqlXFH5FiyEGgdep23LxjPJwIVCysKhSp7cocD8LrQkqFFwydioAIiIAIiIAIiECkE1D6RaA0CEipKA3qilMEREAEREAEREAEREAEooiAlIoCP0zdIAIiIAIiIAIiIAIiIAIi4E1ASoU3jSg950f7HCe6PrASCY/KcRyIPUrvn2IWARGICAKOE9n1Ez/ax7I+ImBHUSIz4CANbMZG3vsTeSkO7sXh0whOMoalbGFh7UhD0bdvX7Rt2zbSkh3x6a1evTqGDBmCxMTEiM+LMiACIiACxUXA1q1snBdXHMUZ7sCBA9GsWbPijEJhByCwIqEWXqzVD5kRqJRapcJx7FmADEagk5SKCHxoSrIIiIAIiIAIiIAIiIAIlCCBfKOSUpEvIgmIgAiIgAiIgAiIgAiIgAjkRUBKRV505CcCJUVA8Yj5S2okAAAQAElEQVSACIiACIiACIhABBOQUhHEw7MfJ7F2ELeElQg/KGQ/LhRWCYvyxPB9SUtLA+0oz6qyJwIxQ0AZDT0BW0ZaO/QxFG+ILOdVxxYv40Ch86N3ZTPSAnnJrZQISKkIArwt6KwdxC1hJTJp0iQsXbo0rNIUC4nZtWsXfvvtNxw4cCAWsqs8ioAIiEChCNgGubULFUgp3jR27FisXr26FFMQm1G3St2Bm3b+ldsXtcMaSkZ26iK1XZmdfD9LSoUfEjmIgAiIgAiIgAiIgAiIgAgUhICUigLQSk1Nxc8//+wxCxcu9Lt7xYoVHn8ru3jxYj+5ZcuW+ckFGk1YsmSJn9zy5cv9wlu0aJGf3MqVK/3kFixY4Ce3Zs0aP7m5c+f6ya1bty5Lzus4e/ZsP7kNGzZ4SWSdzpw5009u48aNWZ5exxkzZnjkJkyY4OWT83Tq1KkeOct527ZtOYXcqylTpvjJ7dixw/XJ+ff333/7yXGkIacUMHHiRD+53bt3+4qBaWeY9Bg9erS5Z+/evbzMYcaNG2f8bB5o79u3L4cML8aMGeMnl5KSQq8c5s8///STO3jwYA4ZXowaNcpP7tChQ/SSEQEREIFSIZCcnJyjXGJ96psQ1rssJ73NqlWrfMUwf/78HGFRvrTquk2bNvmlb/r06X7p27Jli59ccdZ1rB/9Isx2mAgHP7vn3sa/pgPG+8hQPtl18/0bvmsKbtk+Noepku4/in/lzsk5ZHhPxXT/OuyaHZNw+t75Jppbdow39yRm+Ndh1+/4y/gxHGvKZvpPmbppe1YYVoZ2GWSa8L0Pv7gXzKO3cZ1y/GW4V97+PB/pusXCn5SKAjxl7qHdo0cPWNOwYUO/u+vWrevxt3INGjTwk6tXr56fXP369f3k6GbDsTbv9RVkHNbf2nXq1PEVQ6NGjfzirVWrlp9ckyZN/ORq1qzpJ9e0aVM/OX6fwVewefPmBZZr166dbzCe65YtW/qFV7lyZY+/PWndurWfXKVKlay3x27Tpo2fXMWKFT3+9oTf+7B8rV2hQgXr7bGZdrtveYcOHUzYgb5XYf1sWLQTEhI84diTjh07mjDob025cuWst8fu1KmTn1yZMmU8/vaka9eufnJ8v62/7OggoFyIQCQRYNlnyzfarE990896l37epnbt2r5iaNy4sV8ZF+q6rkaNGn7xBqrrqlWr5ifXokULv/RVrVrVTy5QXVelShU/uVatWvmFl19dx/rRL6Bsh3Zug7qHe+5t/Gs6oIOPDOUDfZlpXMXW+KVyhxxmf1xZ9+6cf2Mq+csdjCuTU8i9+rNSGywul9XGGVmpvQn3kBPv+uT8+71SO+PnHXfWB/Nyyv1aOSsMbzl/lQLo7t7GPHob1ynHn+NeefvzvJvrFgt/UioK8JTZ6GLj3ZpABQB/xNbf2oEKADaArb+16eabHN5r/a3NOHzlmBbrb20rx0al/bhQILlAjedq1arBhmPtQI1nKhDW39rBypUvX943G2AhbcMJpBTZG6jgWDlrJwb4yFwgOVZcNhxrs7Kx4Vg7UKOdlZf1t3bZsv4FI9POyo/s85Oz/tbmPTZd1mblav2tzffR+lubCqf1t3awcvYdsWHJFgEREIGSIOA4bIYBLHNtuUXb1mHeaQhUhwUrF+q6ztZhrANsORuoTrRy3vnwrusaNGhg6ttQ12H51XWsH73T5H3OrkZ2h3ob/5oOqO3e5C3Dc38VAFhVriaWJNTNYdICKAEry9XKIcN7AsmtSKiNhYl1ccApg8XZ4aY7/k3aZa4cw/A2mQHl6vjFm46s99LNoueP+fM1Hs/sE97lK8PrbG9jUcacRNnB/wlEWQZDkR3b2LJ2KMIsyTCOOeYYsCe+JONUXAArl5NOOgmBKgrxEQEREAERyCJg61ZrZ7lGzrF///7gqH3kpDg6UrrKVUBerXks9EXt8Hme0a9UhA9rpUQEREAEREAEREAEREAEopKAlIqofKzKlAhEHgGlWAREQAREQAREIHIJSKmI3GenlIuACIiACIhASRNQfCIgAiIQkICUioBYcjraj5NYO6dv+F9xq9D09PTwT2iUpZAfcuJ2rpH63kTZ41B2REAEwpSALSOtHabJzDVZ3G4+lurYmYfKgCY90PZIuVIKvUd8ZgbKZ6SGPuCoCbHkMyKlIgjmtqCzdhC3hJXI5MmTwe9ihFWiYiAxSUlJ+OOPP3DgwIEYyK2yKAIiIAKFI8AOGN5pbZ5Hkhk/fjwCfQMjkvJQkLR2LJOOp/dVRNktdRG3uS7u31sRvx0sh90ZJbunUYvUHfi/nZPguMpFQdIfDrL8lgXTEantSqY9kJFSEYiK3EQgHwLyFgEREAEREIFYJJDgZOLTqrsxqFzWR+n+t68Shuyqjhpba6PNtpq4ZndljNhfHsvT4mMRT0znWUpFTD9+ZV4EREAEopqAMicCIhBiApz2tCo9HndU3I+LE1NM6A4y3f8OlqeXwVspFfB/e6rg8t1VkFHKU6RM4nQoMQJSKkoMtSISAREQAREQAREQgfAnkOIqA/MPxeOHAwl4bl8FXO+OPpy4sxpauyMRiVvqoNX2Whjsjk784vpTnWCOaFO5aBqXjq+q7cbfNXchLugZUQxBJtIJxEV6BpT+/AmUK1cOZcqUyV9QEiElwA858YumjqNSNaRgFZgIiEBUEXCcrDLScbLsSMscy/lIrGP3pLu8m7cHep2AX+p0wnB3ZOG4HdXRaGttVHIVhy47auHspKquUlER89LKoFF8BoaVP4CPqu7G1Jo7sLPOVlxe4QAc93+ma2o4mXilyl4sq70D5yQeKPbHeMiJwz6nnBuP45rI+ou8FAfHV0pFEJzYOKSYtXkeSeaoo45Cq1atIinJUZHWatWq4YQTTkBiYmJU5EeZEAEREIHiIGDrVmsXRxzFGeZxxx2HJk2aFGcUhQqbi4A37EvFhE178e6Sbbhn2npcOGYFen2/ADU+nInzN1SDc/cHcP71ECZWb4lN6XHoXCYNt1bch+/ckYZ5Nbcjue5WrK+zDRPcUYd3q+7BPZX24cLyB3FE2TQsS4/Hy/srmLQ9UCkZa2pvx3UVUlDGVS6MYzEfVperiRE1+yAzApVRq1Q4jj0rZlglFLyUihICrWhEQAREIB8C8hYBERCBAhE4lJGB5bsPYNS63Xh14RbcMmUtzvh9GTp+PQ8V3p+Bxp/NQf9fFuPGv9fix7VJOJiegf4NquCxXo3wWO29yLzrLGRe1xePL/kBv9VIwqtV9+I/Fffj9MSD6Fg2HeVzafOmZjq42h3ZuDAxBRtrb8MDrrJRKS6zQGmXcPQRkFIRfc9UORIBERABERABESg2AiUb8L5D6Zi7Yz++W70Lz8zdhGsnrsYJI5egxedzUeG9GWjz1TycPGopHpixAX9vSUalsnE4p3l1jOjTDBNObYeNF3fDvuE9Me+cTvjuhNZ4undjXNu+DnqUTwO2bwAKsSXr7EPx+MIdzfi02h7Ui88oWSCKLWwJSKkI20ejhImACIiACIiACMQCgR0HDuGfrcn4dPkOPDxzA4aOW4m+Py1C/U9mo/IHM9HtuwU458/leHnBVix2RyaaVSqHq9rVwqcDWmL6WR2w6/Lu2H5ZD0w5owM+cd0e7tkIQ9vUQt96lVGvQtmQI+xdLg1tyuijuiEHG+EBhp1SEY48OS+R6bI2zyPJHDx4EIcOHYqkJEdFWvkhp5SUFETqexMVD0GZEAERCHsCtoy0dtgn2CeB/MBpWlqaj2vOS+ZtXfJBjN24B28v3oa7pq3D+aOX4whXWajmKg21P56No39chGETVuGT5Tux/UAautesgLu61sePJ7bGgnM6Y/+wnlhzUVeMO6Ud3jquOe7q1gDntaiBHjUromq5MjkjjIGr+Mx0VE4v/gXhxYEyWieKSakI4m1hYUAxa/M8ksw///yDFStWRFKSoyKtSUlJGD16NFjhREWGcmZCVyIgAiIQEgLsgGFA1uZ5JJm//voLa9euRWp6BpYkpWDk2iQzonDz5LU4ddRSdPhqPjhNqennczFo5BKz7uHXdbvNNxyOb1QFTx3ZCH+c3BarLuiCA8N7Ysn5nfHLSW3w8jFNcVOnuji1STW0r56IxDJqsnm/Fy1Sd+LqXZPhFGL6lnc4pXFulYpIbVfmxkxvaG5k5C4CIiACIiACEU9AGQglgb2p6Zi9cz++WbUTT83ZhKv/WoXbt1TEsX8nGcWh/dfzcervy/DwrA2Yui0Z1RPK4PwW1fHmsc0w6bT22HxJN+xxRxxmn90JXx/fCk/0aoyr29XBoIZV0LRyAuKibDegULJXWOFPQEpF+D8jpVAEREAEREAERKCECGxNOYTJW5Lx0bLteHDGBlw2biWO+XEhan88C1U/nIke3y7ABaNX4LVF27Biz0E0KJOBSxsm4ItBLTHz7I7YfXkPbLu0O/4+vQM+6t8CD/ZsiMta18LRdSuhTvnQr28oISyKprgJREH4Uiqi4CEqCyIgAtFF4PMVO3DUDwtzNdwyklMrvly5E+kZdiA9uhgUd25unbIWHb6eh6afzcHgX5cUd3QKP4wIZGRmYs3egxi9YQ/eXLwVd05bh3P/XI5u385HlQ9moN4ns9Hnp0XuKMRqfLlyF3YdTEPv2pVwf/eG+PnE1lh0bifsH94Tqy/sgtGntMN/aqbgmqblcU7zGuhWowIql4sPo9wqKSJQcgSkVJQc61KLiR9fK1s2pntHSoV9fHw8ypcvj0j9oFOpQFOkhkCXGhVxXfs6aFKxHKZu22dM5xrljdulrWqiWeVyeH/pNvMhq7Zfz8f8nSnmvmg93DBpDZ6csymk2TuxUVXUSSyLdftSsd41IQ1cgRWIgOM4Rt5xsmxzUcQDv8ewaNcB/Lw2CS/N34Ib/16DU35birZfzkPiezPQ/Iu5OMFVJu/4Zz3+XL8HcW7UJzeuiueObIIxQ9pijaswcBvWhed1ws+D2+CFo5vgxo51MKRJNbStVh7l4g83n1jOq44t4gMrxO2pTjz2xCW4d7oPzz1G0l/kpTg4uod/FcHJx6RUXFwWJmtHGoTevXujZcuWkZbsiE9v1apVMWjQICQksNCL+OwoAyVIoEP1RLMdZPdaFTyxnu42ZrhF5A0d6+LFo5ti/Gntjd/KPQdw+u/LomSXMZOlHAf2Kn+xcidWuj3LOTyKeDHYVSrObla9iKHo9lAQsHWrtYMNc3dqGmbu2Iev3Pfj8dkbcdWEVeCH3jj6VNFVHDp+M8/8Nh53FdJZO/ajVmIZXNKqBt4/rjkmn94e2y7thqShPTD9rI74clArPN6rMa5sV9t8HK5xpQRX0Qiu6de3b180btw42GTHlNyhTAdbd+/B1qTdITfT9sfj0biO2FJM4SftTS62Z2XfLMexZ8UWVYkGnNVaLtEoFZkIiIAIiEBRCXStUQE13UYSw1mdfBBLd0fm1opMf15mwqa92HEwLS8R+UUxgc37Ire+uQAAEABJREFUD2Hi5r34YOl23D9jPS4ZuwJH/rAQtT6aieofzsIR3y3Exa7bW4u3Y3VyKtpXTTQjCl8d3wqzXWVhr6s0bLqkGya6SvgH/Vvg/h4NcbE72ndknUru76dsFJMr5qwFGXwFJxO7kpOxdP36iDPs0CiDzCBzKjESkFJBCjIiIAIiEGEEDqRlIPlQ1senEuIdNKpYzi8HC3cdwDtLtuHuaevx0vytmL5tn58MHbgo9a3F28xuNtO2ZfXOzdm5HyMWbaU3uL/+e244ryzYYqYhbcyeLjTL7SVm+Az72bmbcSgjw8jbQzDxMwzuosN57Zzi9L7bePzS7XlmL/Qbbvznjl5ugpuxfR+YBppN+1ONmz0EEw9lp2xNNt8IYDw/rtmFtEw1GMilNA3XBB2qWgdo3xvbOx6H26euw1l/LEPnb+aj4vsz0ODT2Tju58W4dtJqfLNql/vOZ6BP3Up4qGdD/Dq4DZae19msb1h5YRf8MaQtXu/bDLd1qY+z3FGoLjUroGLZ+NLMXszHnVx3KzLqbcGhelsjzjDdn1TbE/PPsCAApFQUhFbJyCoWERABEciTwI4Dh8wc8YPpWY3iJ3s1ztF4YkON6xA6fTPPVQI2m7A+X7Edvd0eXi5QNg7uYfXegzj2p0VmUerjszdh3q4UnP77crPbzXl/rsDLrhLhimFR0gHcPX0Dbpq8Fne5CsrMHfvpjH+27sOtU9bh5ilrTGNw18EsJSfY+F9buAXH/LgIO1PT0KJyghtPCq6YsApfrNxpRl4+WLYdqdl5XOf2Qr/lKjY0G93eayYg2Hi4KLe/2zBlXO+5SstGVym5x83Pbf+sYzAyxUwgxVWAF7gKLhW55+ZtxvWT1uCkX5eg1ZfzUP69GVh77QtwbnkFG488C+M27kFCfBw43e/lo5uAH3pbd1FX8MNvC87tjB9ObI3njmqC6zvUxeDGVdHKHZkomz1FuZizoeBFQATyISClIh9A9LYfJ7E23SLJ8KvOqak5e/YiKf2Rmtb09HTs27cvaue6R+pzCV26SzakoeNXocXnc820j9ofz8Y7buO4TJyDP93eWX4gyzs1L8zfgtfcXv4aCfH45/T2eKxXI4w7tR06Vi+P512/f9we+7SMTFwwZgUmbUlG6yqJmHN2R7P95YwzO+AHtxd/+Z7D06m4qHmKG46Nw5aF17avg7eObWadPXYw8VP4pQXbUDOhDB7o3tDs1f9+vxamh5l9y71qVzJbcnaqUZ6iONPtef779A7GrWetisYtmHhsPids3oszmlbDX6e1M2tSZp3VESc3qmrC0aHoBJIOppmRsM9X7MCjszYa5bDfz4vQyB1p4IhDZ1fBPfOP5aBSMc8dBatfoSyGt66JD/s3R4P370Hmv49H53duxrQzO+LzgS3xqPvOXtG2No6rXxkN3VE4xwnfuef79+/HoUOHoH8lS4CTIgOPvZZsOgoTW1Z3UGHuDO97pFQE8XxsBWrtIG4JK5Fp06Zh5cqVYZWmWEjM7t27MXbsWBw4cLhxFgv5Vh6Lh8AzRzbGlDPaY+oZHXBNu9omEjaYf1qTZM69D68uzJq2dHzDqqieve6inNv7e17zrIXJX6/e5Y4yJGNa9nSoWzrX9WyD2cBtwF3SsqZ3cOa8ktc0Esc53MCrXNa/Ggkmfgaamp6Bje6oQ5PP5uAiV8Fhg/NWNy1PuXmlf34mmHimbks2u2cxrH+5jdS47LRTIeuQrbDQTyZvAqz/NuxLBde4cAravdPXm2fW6/sFqPHhTNT4aJYZCbts3Eq8t3QHKNupegXc0rkevjuhFeae3Qn7hvXE+ou7YcJp7fGeq0De070BLnTftXIblwP796DYvqidd9aK7Dtp0iSsW6dRryKDLGAA21z5sa7JOenSdYiAP6tU8HcVAckNOon+tUHQt0pQBERABESgpAjUdpWDOuXLooU7qvBan6ZoXy3RRP3Kwi1mz31z4R72p6WDC7fdU3y5cqcZ3eAIB80zc7OmQu1JTQfXTFCGpn21rNEAntP4XtMtWBNs/Azvf0c0REK8YxZif7FyJzgdqd/Pi/HT2t30ztMEG8/s7KlaDKxBRS3MJYfcDJXU5bsPYNT63eDUtFumrAW/idLp63mo8P4MNHaVP+6udMPfa93RrCQccJXCfg0q49EjGmHUSW2w/PzOSBl+BJZf0BmjTm6LV933lErFGU2rgyNO5cuoyZEbe7mLQDQQKOovPBoYKA8iIAIiEFEEHMdBS1e5YKK55GD2zqw1DryuUCYedV3lg+eXtaqJhed28pitl3Y3c9NHuI29BhUOL+w+6DYOKV8Yk5qRmeO2YOPnTX3rVcKS8zrjhaOa4IIWNcxUKObnlilrQMWHMr6GPeXfuyMtwcZT3yufPuvIfYOOiWsqY/N2poAMn5m7CddNXI0Tf12Cll/MQ/n3pqPNV/Nw8m9Lcf+MDWZqXEVXEeCi5xF9mmHCqe2w4eKu4Pcb5p3TyR2BaI1nejfBdR3q4IRGVY3CWybu8ChWTABVJkVABDwEpFR4UOhEBKKZgPIWbQS4577NEz/yxXMuvP5zw24McnuPeT1t+z4zEpBYJg7WjFi8zayrGODKuIMEFMOUrTlnJq/b778Gy8ryBm61SJvmzw17aOUwwcTPGzp9PR/frEoC14R8NrAlNl/SDWc3qw4qFhuz0xDnKlCUPZStvHy8fLvpSadbMPEM9MrnwqQU3uYxaYzIcxU9JzsOpJnpbZ8u34FHZm7EsPEr0fenRaj/yWxUen8mun47H2f/uRwvzt8KLsJvWrEcrmxbE58MaIlpZ3bArsu7Y/tlPfDPGR3wqftcHnFHIoa2qYW+9SrDW0mLHmLKiQiIQCgISKkIBcUwD6NChQooV65cmKcy+pLHL2pXrFgRcdqZJPoebjHniIteF7sN4G1u49BGtX7fIdCNPc10a10lgZYxI9dlrat4eNZGPD9vi9mLv1mlBFf+AG79Zx241//WlEN42u2Z5pSWxm4jsmq5MritSz1z/4sLNoO915zfy21jn8+eJmU8sw/VEsqgVfboyIJdKWYDAk6TGbGIM5uzhP5wFQxuBXt/j4bIL37ewTb9E3M2mgW+vKbikOKOmjSoUBbtsqdk9cz+ACC3lGUeRm9MRv/6lSkedD5v71LfyD86a5P5enZKWgbY4H5zSVbaN+0/hGnbksHdpIxgaR6CiJvPidv8cqckbul797T1OH/0chzx3QJU+2Aman88C0f/uMhVJlbhI1ex2JaShu41K+C/XevjhxNaYcE5nc2I1Tp31IEL+N86rjnu6tYA57ujRT1rVQTfjSCSETIRx3FMWI6TZZuLCDqwnFcdW/IPjJMZK7rRRuJbY9PsOPbMzUgU/EmpCOIh2kahtYO4JaxEjjjiCLRo0SKs0hQLialatSoGDBiAhITDjb9YyLfyWHQCX63ahS7fLjDfVKhUNg40t01dhy7fLMDfW5JNBDd2rIvLW9dEQpyDia5bXbch+fWqnbizW320qZqISae3w6WtauLHtbvNXv8NP52N95Zux6cDWuC8FjVMGPyC8Bt9m5ne555ug7Sq2yDlrlG967CqNiI5Dq8c0wQtXMXivukbUPOjWWah7vUd6nhkuEj313W7g46fox/tq5fHab8vMw3hmm4e1u9LxbfHt/KEeVvneji2XiUsdBUZ5uGoOhVwWtNqxj/YfHL3K+5SFe/WeM0/nwMuLH7I7cFvVqmcCScpNR1H/rDIrO0wDmFw4CJ2ftDwt3VJeHnBVtw8eS1OG7UUHbLXNzT9fC4GjlyC/7juVCqpoA1sWAVPHdkIf5zcFisv6IKU4T2x9PzO+OWkNnj5mKa4uVNdl111tK+eaEauwiCbJglx2R0v1jaOEXQ45phj0KhRowhKcXQktZabjQGuKa1muRt1of8iMc3BZNYtYoMRk4wIiIAIiEBJEbiqXW2kXnEE9gzriT1Ds8w+9zz1X0eAOzoxHZXKxoNbsCYN7WEakH8OaYftl3Z3G+BZvficpvJh/xbg4tmdl3XHXjeched2xoUtc+7sxLg4Pz5paHezMw9l+tTLCoPxeBtuLcvwkt20LD6vM3Ze3sM0Yjde3M1MmWEj1oYfTPxrL+oKfoeA8/TnnNURG92e8zlnd0LvOpU80TZ2R1zGn9oezP+Oy3qYKTpcT2EFgomHstz5iXnb7fLa4nJa4ja2J5/eAVsu6Ybdbj4OXtETXAhP2ZIy/HghF8x/4yqD/ADgNRNX4/iRi9HMVRgqvDcD7b6ahyGjluHhWRswdWuyO4IQj/Ob18CbriI48bR22ORy5zsy22X2jauIPdW7sdmad5CrXDSrnIB4V+EsqbwoHhEQARGQUqF3IAwJKEkiIALBEuCHwtiA7FyjPMrFBy7SOXWJayryCpMN9Srl+IWIvKSy/BiWbYCXdXuZ61Uo6zZ4yyChgPEzXQwxznHArWzzmnbDvOWXPobHtDHM3Ix3Phle7fJlzXa6zEdu9xTFnVO2JrsjSR8v246HZm4wHxY85seF4MhSFXdkqLs7InXB6BV4deFWcOclfgTwuva18fmglphxVgej8GxzlaC/z+iAjwe0xIM9G+Ky1rVwTN3KqOtyL0radK8IiIAIhJJAXCgDU1giIAIiIAKRTeDBGRvw69qsNRqcfnP1X6twIC2XneAjO6shST0Xra/ZexBjNu7BW4u34c5p63Dun8vR/dv5qPLBDNT7ZDb6/LQIV/21Gl+s2IVdB9PQq3ZF3NOtIX4+sTUWndsJ3E1pjTtqM/qUdnjz2OZm7cO57ohE95oVjcITkoQqEBEQAREoZgJSKoIAzEVxFLM2zyPJ7Nu3DwcPHoykJEdFWtPT07Fnz56I/aBTVDwEZaLABJpXTjBrLh5xe8Tv694AtRPLIi0z57axBQ40Cm7gYvefXWXrpflbcNPfa3DKqKVo99V8JL43A82/mIvjRy7B7f+sxe/r94AV6+DGVfHckU0wekhbrL6wi1EcFp7XCT8PbmO+6H1TpzoY0qQa2lYrn+sITxRgCyoLtm61dlA3hZFQcnIyUlP9d0wLoyQiGtOS5mbKf+851zEC/qK1RGXZFwH4SzeJtqCzdummpuCxz5gxA6tWrSr4jbqjSAR2796NCRMmSKErEkXdXNIEuHXoAz0a4h5XoeAuTo/2agSu3yjpdIRbfN+u3onTf1+Gx+dswozt+1EroQwublkd7x3XHH+f3h7bLu2GpKE9MfOsjvjy+FZ4oldjXNmuNgY0qIImlRLAKV7hlqdwSU9G9gdErB0u6Qo2HZMnT8b69euDFZdciAhsc8OZ4JpIHEe1SkWktitd7AH/pFQExFIQR8mKgAiIgAhEO4GhrWthz9Ae2HRJN0xylYgP+rcwW9pe0qomjqpTCTXdEZ1oZ6D8iYAIiEBeBKRU5EVHfiIgAtFDQDkRgSIQqFg2XiM2ReCnW0VABKKfgJSK6H/GyqEIiIAIiIAIRAwBJYzhCDkAABAASURBVFQERCAyCUipiMznVqBU82uf+gBbgZCFRDg+Ph6VK1dGXJx+ZiEBqkBEQASikoDjOCZfjpNlm4sIOrCcVx1b8g+snBtlZddE4ltj0+w49szNSOT9+aVYrR0/JP4OtlFobX+J8Hbp2bMnmjdvHt6JjMLUVa1aFf369YMqmyh8uMqSCIhAyAjYutXaIQu4hAI66qij0LBhwxKKTdFYAvyMZz/3IhKb5ZGYZhd1vn9SKvJFJAERKAUCilIEipnA5yt24KgfFuZq+IG2y8etxLNzN2P2zv3FnBoFLwIiIAIiEOkEpFRE+hNU+kVABESgEASOqFUR17Wvg7rly2Lqtn3G8KvcdLu6XW0MbFAF6/el4vap69Dj2wW4YsIqHEwv2uaNN0xagyfnbCpEasP3FqVMBERABEQgi4CUiiwOQR03btyIDz74QKYEGdgHI+4l/95Z9rKjk0CrqongNymOqlPRk8HTm1Qzble0rY3/HdEIY05ph1ePaYqEOAfvL92OS8auRGH3VeeXp79YuRMr9+pDnB7gOslBYNasWapfS7B+Zb1qH8AHo8fgg9Gjo9mEVd5Gzphh0UeVLaUiiMfpXYkOGzYMw2RKhMHzzz/veTpLly4tkTj1bIcZzpH6ESrPC6OTkBG4rkMd3NCprgnv29W7MHLdbnNe0MOETXux42BaQW+TfAwQUB2bVe4OK+G2xaeffup5u/5auADDXnhBpoQYeMBH2YmUiiAeqHeBF4R47IkoxyIgAlFN4LKWXBKZlcXXF23NOnGPLBt/WZuEh2duwJ3T1uHNxVsxdWsyDmUcnia1OzUNb7j3nDt6uXsHMGP7Pry3ZJsxm/anGjce1iUfxKsLt+AuN5xHZm7Ej2t2YcO+w/6UkYlOAurEiM7nqlzFHgEpFQV85iNHjkSkmSeffBJvvPFGxKX78ccf9zydvn37Rlz6v/jiCzzwwAP47rvvIi7t9erV87DXSfQQKGxOWlRJ8Nw6b2eK5/zfk9fitN+X4Vt3BCMxLg4vzN+Co35chGN/WoytKYeM3NLdB/DBsu1ITc801+uSU/GWq1TQbNyfJUPloc1X83Dj32uR4IazMCkFF4xZgeafz8U7rqy5UYeYIBBp9SvT+7///Q/vvPNOxJXzN998s+edOueccyIu/R9//LGpY3/++eeIS7sF7zjRtQ+UlAr7ZIOw2dA6+eSTEWmG290dd9xxEZducraPheeRZo4//ngcccQRGDx4cESzt89AduwSqFQ2HnHZ2acCwPURvJy7K0vBaFM1EQ/2bIhZZ3VEpTJx4MLvV9xRB8r0ql0Jf5/eAZ1qlOclzmxW3VzTrWetrPUcy/ccwMFspeP8FjXx2cCWuK97A6RlZoKLu/enpZt7dYhuAl26dInIsrJ3797o379/KNNeYmHZNyrS6lemd+DAgaaO5XmkmZNOOsmijyrb1hNRlSllRgREQAREIHQEDqRlwE5oql2+LOKye9deOqoJ7uhaD4+4CgWnQqW4cr1qZykKM7YFvw1tv/pV8GTvRmZReIfqia6CkYGeNbPCOZiRifleoyOhy5VCEgEREAERCCWB2FMqQkkvQsKqXLkyEhMTIyS10ZPMMmXKoFq1aojUDzpFz5NQTopKYNuBrGlKDKd7zfK0jGleOQHVy5XB8AmrUPWDmej0zXxM2bbP+HGUwZwEeehbtzLGb96Lll/MQzU3rP/7e63nzuxBDM+1TqKLgONkTQFxnCw70nLHD52qji35p1auXDlTxzpO5L03jhN5aQ7mCUupCIKSbRRaO4hbwkqke/fuaNasWVilKRYSU6VKFXAtSELC4fnosZDvUOVR4YQPgV+8dnwa0KCKSdiKPQfQ4su5uGvaepSLjzNTmtZf3A3cltYI5HPgblDfr95lpG6ZshZ9flqEn9Yk4T+d6mDH5d0xcnAr46dD9BOIi8tqilg70nLM6U8NGjSItGRHfHpr1qxp6ljHibwGuuNkpdlxsuyIfxjZGcj6JWdfyBIBERABERABbwJzd+zH47M3GaceNSvg5uztZT9ctgM7DmRtEfvlwJaeNRNbshdomxu8DnbK1KGMTOP68fLtGLV+t5nq9MqCrB2lhrWphRs61kWFMvHYkpIVthHWIZwJKG0iIAIiYAhIqTAYdBABERCB2CKQdDANi5NSsC1bMWDu1+87ZNwW7jqAkWuTcP+M9Tjyh4VYty8V3VyFgguoy2b3KldPKMNbjPl7S7Kxf1uXhHGb9prz1XtTMWvHPs/2sj1rVTDu3FJ2q6t4jN6YjP71K6NsnIMq5bKqogW7UsAtaPempuPJuVmKDG+auGUvNrpp4LmMCIiACIhAYQgU/z1ZJXnxx6MYREAEREAEwojAJyt2oMPX88FtYG2yrv97jXHr9M08nPr7Mny8bCcGNayC1/s0xfQzO6B11UQriuva1zajFjUS4nH2n8tR66OZeHjWRvx6Uht0qVEeK/ceQM/vFmLS5mRzz22d6+HYepWw0FUcGn46G0fVqYDTmlYzi76/HtQKfepWwoTNe1H749no+u0CdHbDeKJXI1QrF4//Tl2PS8etNOHoIAIiIAIiEJ4EpFQE8Vy4qwnFrM3zSDK7d+9GSkrW1o+RlO5ISmugtKalpWHnzp3Qh50C0ZFbaRO4vkNdZFzZK0+z8sIu+HlwG1zTvo5p/HunOSE+Ds8d1QTbL+uBnZd1x+oLu5p1FYMbVcXsszvh0L+ywu6fvQajcaUEjD+1PfYN64kd7j2fDGhppjkxTMr8dVp7HBjeE+vccBjvE70a446u9bHz8h4mjWNOaUdRmSgkYOtWa0daFpOSknDgwIFIS3bEpzc1NdXUsZGYEfuuWzsS8xAozVIqAlHxcbMP3do+3mF/OWfOHKxZsybs0xltCdyzZw/+/vtvHDx4MNqypvyIQA4C1RLKgN+yyOGYy0U5VxmpUi4+oC/96lYoG9AvRI4KJgwJ2I4Xa4dhEvNM0rRp07Bx48Y8ZeQZegI7duwwdWwkvjeR2p7M7ynG5ScgfxEQAREQAREQAREQAREoOQKKKRIJSKmIxKemNIuACIiACIiACIiACIhAGBGQUhFGD6OkkqJ4REAEREAEREAEREAERCCUBKRUhJJmmIbFr32WL18+TFMXvckqW7YsatSogfj4wPPHozfnylmICCgYEYgJAo6T9QEwx8myIy3T1atXh+rYkn9q5cqVM3Ws40Tee+M4kZfmYJ6wlIogKMVl78tu7SBuCSuRrl27omnTpmGVplhITOXKlXHMMceABV8s5Fd5FAEREIHCELB1q7ULE0Zp3nPEEUegfv36pZmEmIy7Zs2apo51nMhroDtOVpodJ8uOlgcopSJanqTyIQIiIAIiIAIiIAIiIAKlRCDslYpS4qJoRUAEREAEREAEREAEREAEgiQgpSJIUBITARHIk4A8RUAEREAEREAEYpiAlIogHr79SIm1g7glrET4tc99+/aFVZpiITGHDh3C9u3b9UXtWHjYyqMIRAyB8EuorVutHX4pzDtFO3fuREpKSt5C8g05gdTUVFPHRuJ7Y9Ns7ZDDKaUApVQEAd4+dGsHcUtYicydOxfr1q0LqzTFQmL27t2LKVOm4ODBg7GQXeVRBERABApFwH4R2dqFCqQUb5oxYwY2bdpUiimIzaj5RW3WsZHYNgsqzRH4WKVUROBDU5JFQAREQAREQAREQAREIJwISKkIp6ehtJQUAcUjAiIgAiIgAiIgAiIQQgJSKkIIU0GJgAiIgAiEkoDCEgEREAERiBQCUioi5UkVIZ382mfFihWLEIJuLQyBsmXLolatWoiPjy/M7bpHBERABGKCgONkfQDMcbLsSMs0P8JWoUKFSEt2aNNbCqElJCSYOtZxIu+9cZzIS3Mwj1hKRRCU4uKyMFk7iFvCSqRz585o3LhxWKUpFhJTuXJlHHXUUShXrlwsZFd5FAEREIFCEbB1q7ULFUgp3tSjRw/Uq1evFFMQm1HXqFHD1LGOE3kNdMfJSrPjZNnR8gSzWsvRkpvozIdyJQIiIAIiIAIiIAIiIAJhTUBKRVg/HiVOBEQgcggopSIgAiIgAiIQuwSkVMTus1fORUAEREAERCD2CCjHIiACxUJASkUQWO1HSqwdxC1hJcKvfSYnJ4dVmmIhMYcOHcKWLVuQnp4eC9lVHkVABESgUARs3WrtQgVSijdt374d+/fvL8UUxGbU/LAs69hIfG9smq0dLU8w1EpFtHDJkQ/70K2dwzMCLubPn4/169dHQEqjK4l79+7FtGnTkJqaGl0ZU25EQAREIIQEMjIyTGjWNhcRdJg1axY2b94cQSmOjqSyw5R1bCS2zSIxzcG8NVIqgqEkGRGIOgLKkAiIgAiIgAiIgAiEjoCUitCxVEgiIAIiIAIiEFoCCk0EREAEIoSAlIoIeVBKpgiIgAiIgAiIgAiIQHgSUKoAKRUx8BbUrFkTlSpVioGchlcW+dG7OnXqID4+PrwSptSIgAiIQBgRcBzHpMZxsmxzEUGH2rVro2LFihGU4uhIamJiIljHOk7kvTeOE3lpDuatkVIRBCX7lU9rB3FLWIl07NgRjRo1Cqs0FSwxkSlNRa53796gchGZOVCqRUAERKD4Cdi61drFH2NoY+jWrRvq1q0b2kAVWr4EqlevDtaxjhN5DXTHyUqz42TZ+WY2QgSkVETIg1IyRUAERCDsCSiBIiACIiACMUtASkXMPnplXAREQAREQAREIBYJKM8iUBwEpFQUB1WFKQIiIAIiIAIiIAIiIAIxREBKRRAP236kxNp53xJ+vtu3bwc/xBZ+KYvuFPGjd5s2bUJ6enp0Z1S5EwEREIEiELB1q7WLEFSp3Lp161bs27evVOKO5UgPHDgA1rGR+N7YNFs7Wp6jlIognqR96NYO4pawElm4cCE2bNgQVmmKhcQkJydjxowZ+qJ2uD5spUsERCAsCGRkZJh0WNtcRNBhzpw52LJlSwSlODqSumvXLlPHRmLbLBLTHMxbI6UiGEqSEQEREAEREAERKBUCilQERCAyCEipiIznpFSKgAiIgAiIgAiIgAiIQLgS0MfvwvbJKGEiIAIiIAIiIAIiIAIiECEENFIRIQ+qKMmsVasWKleuXJQgdG8hCPCjd/Xr1w/NF7ULEb9uEQEREIFIIOA4WR8Ac5wsOxLS7J1GfviuYsWK3k46LwECiYmJYB3rOJH33jhO5KU5mEcqpSIISvYrn9YO4pawEunQoQMaNmwYVmmKhcTwi9o9e/bUF7Vj4WErjyKQTUBWwQnYutXaBQ+hdO/o0qULqFiUbipiL/bq1auDdazjRF4D3XGy0uw4WXa0PD0pFQV4kmlpaVi1apXH7Ny50+/u3bt3e/ytLHco8BVMSkoKSo732nCszXt9w2NarL+1mRZfuR07dvjFu2fPHl8xcBtaG461A21Lu23bNr/wuOuRb4Dccs+GY+1AW/BxBw3rn9eOVd5yVj4lJcVgY1ukAAAQAElEQVQ3WmzevNkvfYHkuC2dDcfa3K7ON8CNGzf6hXfw4EFfMbPblg3H2txi1ldw/fr1fuEdOnTIVwzr1q3zk+P76Cu4du1aP7lAW9quWbPGTy5Sd17xZaBrERCByCKQnpG17faO5B344O8PPOaNGW/g5fkv5zBvTn/T429l35jpL/dWQLkROcJi2G9NeytAeMHJjZj1ul94b0972y+8gHJTCy/32pzX/OJ95593/OLNT+6PBX/k+qIEqusC1WGsp20dZ+1AcmFY1+XI++rVq/3qxEA7NNk8ets5AnIveJ+3P88ZvusV9X9SKgrwiNmIW758Oaxhw9v3dioB1t/abMj7ylEJsP7W5r2+crzX+ls7kBzTYv2tHUj5oBJg/a0dSPmgEmD9rR1I+QgkF0j5oBJgw7F2IOWDSoD1ZwPZl4e9ZoFn5awdSElhgWf9rR1IWWCBZ/2tHUiOjXvrb+1ASgXTbv2tHaigZePe+ls7kFLBAsn6W5vvo+Vh7UBygZQKFnI2HGtLqbAUZYuACJQkgbTMNBOdk+Ygfnu8x/y54k88OOvBHOafVf94/K3sqGWjcsjwnmmrpvvJ/bzsFz+5WWtn+cl9v/QHP7n5a+b7yX275Ds/uYVrF/rJfbH4Sz+5peuW+sl9svhTP7lV61f5yX288GM/uTUb1vrJvbfgfT851omWW9yeOOT2ryh1XaA6LFBdF2wdFkguUB0WqK5buXKlp82WV123YsUKPzkqB758bBjetq8M7/P25znD95WLxuvc36hozG0R88T5eyeccAKsadOmjV+IzZo18/hbuVatWvnJtWjRwk+ObvD5x3ttONZu3ry5jxTAtFh/azdt2tRPrl27dn7xNm7c2E+OU6ZsONYONIWqY8eOfuHVr1/fL7zOnTv7yQUaLu7atatH7uijj/YLxzp0797dI2fTx7Uj1t/aHBq1/tbmkKn1t3avXr38wqtWrZr19thHHnmkn1yVKlU8/vaEabfxWZvToay/tfv06eMXXoUKFay3xz722GP95Pg+egSyT4477jg/Oa7tyPb2WP379/eTK1OmjMdfJyIgAiJQ0gQyKmeg2unVPGbo+UPxwWUf5DBnnXeWx9/KXnHBFTlkeM/p553uJ3fNBdf4yQ05Z4if3PUXXu8nd8K5J/jJ3XThTX5yA88Z6Cd360W3+skdd85xfnL/vei/fnJHn300bD6tffdFd/vJ9T67l5/cAxff7yfX4+weHrmKR+e+DiRQXRdobWagui7Q+pJAdV358uX9XrFg67p+/fr51WGB6roBAwb4yQWq6wYNGuQnFxcX55c+W597275CvM/bn+cM31cuGq/9iUVjLouYJ2qdDMLaPI8kw9GEQKMMkZSHSEwrRybYKxSo9yQS86M0Rx8B5UgEwoGAp27NDIfUFCINm917kl2jvxIlwKnMrGM970+Jxl60yGyarV200MLnbikVQTwL+9CtHcQtYSWyePFicLpQWCUqBhLD6V2zZs0ClYsYyK6yKAIiIAKFI2CVCWsXLpTSu2ueA2wt1ugVeAACnOLNOjYS22aRmOYAj8DPSUqFHxI5iIAIiIAIiIAIiIAIiIAIFISAlIqC0JKsCIiACIiACIiACIiACIiAHwEpFX5I5CACIhCOBJSmghPYf2g/Gj3XGIM+GJTnzZPWTkKX17ug5YstUe+Z+pi9ebZHftj3w9Dg2Qb4bN5nHrdwPLl11K3o8GoHNHm+CQZ/NDgck6g0iYAIiEBUE5BSEdWPNytzderUQaAdirJ8dSwuAgkJCeajg/Hx8cUVhcIVgTwJfDbvU+zYux5/rx6DGRtn5CrbpGoTnNbmdGxMWond+zaDygiF52+djy/mfIBdyZvwxMQn6BS25sSWJ6JOxTrYtmcd1u/ZELbpjIGEFTyLTsFvCas76mcClcMqRTGRGO4exV0pHSfyXiDHibw0B/NSSakIghK3B6OYtXkeSYbbyDZo0CCSkhwVaeW2etz6tly5clGRH2Ui8gi8Pn2EJ9Ejpr/hOfc9aVy1Me457m5fZ7Sv1R4tanU07oNa5D3aYYRK8TC41WCc3e6sUkyBoi4sAcfJbmBFaoukk5vz2q7RX4kSqFatGljHOk72+1OisRctMsfJSrPjZNlFC62wd4f+vkj9CYeehEIUAREQgSgiMHXDVCzYdHh04tN5n2D3gT0FymF8XDxmXjMDC25agecGP1egeyUsAiIgAiIQWwSkVMTW81ZuS4iAohGB0ibw+rTX0bRmW5zY5nSTlIy0/fhozgfmPNjDnyv/xAfuPd8v/h5fL/w6x23cEnHsqrF4deqreHLik/h8/ufYnLwZb814G7eMusWsy1i7ey3enfUOXv7nJTw+8XEkp+7DnoN78cX8L/DClBewdMfSHGHai4XbFuKdmW/j7tF3m3unb5xuvXLYU9ZPwdsz3zHx/7jkR6Rlpufw14UIiIAIiEDJEZBSUXKsFZMIiIAIlAiBnSk78cWCL3HtEdfiul7XeeJ8bfrrnvNgTqhM/PvXG3HP77fizRlveW7Z5yoHfd89FkM+HIhRK0ZhddJqDP9uGJo/Wx8v/vMCXp/yPG77/TYs3r4Y94y+F3f89m887CoI9Gv4bAOjgLw45UV0faUtrvzxSk+46RnpuGHkDej5Wmc8Oekp4/7pvM9w7Fu9wIXYxsE9rElag/7v98eAd47Ge7Pew8a9G9147jHpdL0L8idZERABERCBEBGQUhEESPbIUczaPI8ks3nzZuzevTuSkhwVaT148CDWrVuH9HT1nkbFA42gTLw/+32T2qFdh2Jwy8FoUK2FuV61fRHGr55gzoM5vDLkFVzR/XCj397z8rSXMXP9JDSu0Ro/XvQjXj/1dVx1xNXGu3aF2nj+lNdx69G3gounp1w5xbjz8MLkF7DkxiX47sLvcMvR/6ETPpn9IdIy0sw5Ry/emfYqKiRWA+97bNBjGD98PFrX6YzXpjyHf9b/Y2Qv+PoC/LNmPE5qeyb+umICXjz5Rcy6dhYGtDrZhKNDZBHw1K2ZkZVuT2o3ugnf67nSSbEQ8A80JSXF1LGe98dfJGxdbJqtHbYJLWDCpFQEAcw+dGsHcUtYiSxduhSbNm0KqzTFQmL27duHOXPmIDU1NRayqzyGCQGWUyOmj8CFnS5A9fLV4TgOrul1jSd1r09/zXMezEnFchX9xFbtXGXc6lWqZ2we6leuTwsrdq0wIyQnt85q4FcqV8m483DLMbegQeUGPEWL6i2NjcxD4MgKL151FQraJ7Q8ATXK1+ApysWXw7kdzzXnnILFtSJzNvxjrv/V41+Ic7KqsTJxZdCxdtaicuOpQ+QQcNvkJrHWNhcRdFjgvoPbIii9UZLUpKQkU8eyzIu0LEVimoNh7P4SghGTTDQTUN5EQASihwDXQazbuQxT1k/BeV+eZ8yvS3/1ZPCHRd9iS/IWz3VhTgY0H2BuW7x9sWfxN0cR6Ni/WX9aAU2HWh087lYZsA7cxnZT0hpz+cOCr9DixRYe8/TEp437noN74P0NDaugGE8dREAEREAESpWAlIpSxa/IRUAERCBoAkEJvj79dTSv1R4Xdb4IXep1MWZgi4Ho1eTYrPszDuGd2e9knRfyeBa3bo1LQGp6Kpq/2BT1n6mPUUt+whkdL8CLJ72Ya6jcTSo3zwplK6BaxTrG+/wul2Lh9Qs9Ztsd27DrnhSMOHUE6lfKGhGhYEZmBi0ZERABERCBMCAQFwZpUBJEQAREQARCQGD9nvX4dcmP+G+f/+K+4+7LYZ4+Iau3n9G8Of0tFKVBPmndJMBVTqZdNQ0Tr5iEMUPHuI3+ffj83M/NlCvGURgzKPtbGNM2TkNCfAISyyR6zIjpI/D8lOcxsPkgwIkH/y3ctpCWx9i1GR4HnYhAqRBQpCIQmwSkVMTAc69Xrx6qVq0aAzkNrywmJCSgcePGKFOmTHglTKmJOgIH0w5i3pZ5uH/s/UBmulmvsCtllyefVCCqJlZFlQp1jduW3avxwewPzO5MB9IOgLs5cSqT8XQPa5LWYMf+HWaa1IY9G1wXGJmVu1aa84pluc4iA91GdMNFX1+ES769FAPeH4CTPz4Zj018DFRuKMhGvne463avw9Z9W7Ft3zas27OOIsYs27EMew8m4/5+96Ne1WZYtX0Rbv39VrNFLeWfnvQ07hp1KxpXaYyqiVXwnz53mPsenfCoiSvlUAo+nfcp3pr5tnHnblDTNkwDd5MyDjqEN4FI//5Xw0w4VcIbcTSmrkKFCmAd6ziR9wI5TuSlOZh3KOKUimAyFWqZuLgsTNYOdfjFHV6bNm1Qv/7hKQPFHZ/CzyJQsWJFdO3aFWXLls1y0FEEionAh3M+Qu83euLLBV+hTNlKOPHD4/HJ3I89sY1bPQ7dX+uEPQeSjH/ZcpXN1q3dX+uCVbtWgSMDfd7sBadMBcS55orvhmLEjBE44/Mz8O3CrDBnbpphlAYG2qRqE1RIrIGMtP1YunUuFm2eidkbpmDcit/wyOh70H1Ed2zauwk/uqMmx793nAk33k3XLb/9B09MfAKvTX8N/x55A+jG+I53FZJflv2MNjXb4O9//Y3zulyGn5b8bLaobfpsA7w3+z28d84nOK/jeYwe3BXqldPeQnxcPFq/0Aw1nqyGh8Y9hKZVmxr/Awd34bi3e2NHyg5zrUN4E3Cc7AZWVlUb3okNlLoOri5fK5CH3IqTQNWqVU0d6zjZ709xRhbisB0nK82Ok2WHOPhSCy5Sf8KlBkwRi4AIBEVAQiVI4KqeVyLl/lQk370Xe12Tct8B3HDkjZ4UDGw+0PVPA93pv+euPdh3zz7XLRXta7cHF1enPJCO/a6bdb/n2HvMtq4p96VmhXlvitkOlruWHPdeP+w/uBfPDXkNIy8fgz+GTzT2i6e+YbavTU7Zju8Xf4+z258NGy7TxvifG/wcHuj3gJuWrPTa+C7sdKFJL3eR+vCsD7HspqXY9N9d2HV3MhZevxDW3wi5h3/1uNK473DzsuX27SZtk6+cjLW3bcXWO/diz72pqJO9RsMV118pEOBoF0fJSiFqRSkCIlAKBKRUlAJ0RSkCIiACkUrgYPpBrE9ajYRyFXBJl0vBnaD6Nulj7Kt7Xo3T2p5mssZpVeakCIdqidXMmoq8guAC7yoJlY0I7doVa6NyQiWUjdcIoYGS76H4BDiSdOqnp+K+sfdj0bZFxReRQhYBEQgLAlIqwuIxKBEiIAIiEBkEuHj6wYGP4GDqfhzxZk88PvFxfDbvM7wx/Q1c+PWFeOOfl9GmThdc6ioc0L+YJkClcMSpI/DMhEfQ47UO6PVWL7w/+32zPiemwSjzIlAYAhFwj5SKIB4Sh/spZm2eR5LZuHEjdu06vGgzktIeyWk9ePAg1qxZg/T09EjOhtIuAn4E/tv3v1h280rc1PtGcEH4yGUjMXPTTLSp1RbfX+KeXzMTHDHwu1EOMUeA62/+umqayff8jdNxl2L48AAAEABJREFU3Q//Qq2na2Ho90MxYc1fxt1Tt2aay8g7rHeTvNc1+itRAvv37zd1rOf9KdHYixaZTbO1ixZa+NwtpSKIZ2EfurWDuCWsRJYvX44tW7aEVZrCLDHFkpx9+/Zh3rx5SE1NLZbwFagIlCaBRlUamXUbT53wFD46+yO8cdobeLj/QxjcarBZQF2aaQsm7v/99T/0fqu3TAkw+L9f/g+Na7TOfiwZQNoBfDn3Ewx+/ziUf8jB7i57gPKud6QqFYscYJubfv2VKIHdu3ebOjYS22aRmOZgHq6UimAoSUYEREAERCAMCIQuCW1rtEW/pv1kSohBzkXzbtMjM0uD6NGoDxI3JwApoXu2CkkERKB0CLi/7NKJWLGKgAiIgAiIQGkR4Pa0T5/4NGSKn0GlhEqYwQ8mZj/sHo2Pxogz3sX2u5Ix6V8TkbDFVSqy/WRFCQFlIyYJSKmIyceuTIuACIiACIhA8RMYs2oMnhj3oInojn73Y8FNKzDpiokY2m0oKparaNx1EAERiA4CUioi7zkWOMUNGjRA9erVC3yfbigagcTERDRt2hRlypQpWkC6WwREQAQikMDKXSvx7aJv8fNlf2L//Rl4qP9DaFG9hX9OnGwna2dfRozVOBNOlYhJbdQklB+YZR3rOJH34jhO5KU5mBdHSkUQlOyXtK0dxC1hJdKqVSvUrVs3rNIUC4mpUKECOnfuDH1ROxaedqA8yk0EYpsAFYhXhryCQS0GwXFyb0Q5TrZfthVx1NoBmbUiLtURn+AqVaqYOtZxIu/FcZysNDtOlh3xDyM7A1IqskHIEgEREIFYIfD5/M9x1NtH5WrO+OwM3PzbzfhywZdIzyidLZGX7liKbiO6odVLrVD/mfoYvXJ0rDwe5bOkCSg+ERCBkBCQUhESjApEBERABCKHQJe6XXDdEdehcZXGmLPhH2M618lyu6zLpWhWrRnem/Uuhn59Adq90h7zt84v8czVqlALl3S+BBt2rUDSvs3YmbKzxNOgCEVABERABIInUNxKRfApkaQIiIAIiECJEOhQu4NZKNujQQ9PfKe3Pc24Xd/7Brx48ouYMDzrw2Trdy3DaZ+ehpLeV71G+Rq4vc/tSEzQejDPQ9KJCIiACIQxASkVQTwcW5laO4hbwkpkw4YN2LlTvXwl/VAOHDiAlStXIi0traSjLkR8ukUEchLoWq8rKleobRw3714NTkcyFzqIQIgJeOrWzBAHXFLBrXUj2uMa/ZUoAX5glnWs5/0p0diLFplNs7WLFlr43C2lIohnYR+6tYO4JaxEVqxYga1bt4ZVmmIhMfv378fChQtx6NChWMiu8hhlBA6kHUBy6r6sXMUngl/Qzro4fFy4bSHemfk27h59N17+5yVM3zj9sKd7xjLzl6W/4OEJD+POP+/EmzPexNQNU3EoPfBvYtu+bfh+8fd4etLTeG3aa1idtNoNRX85CETjRWZ2pqydfRkx1hIH2B4xqY2ahO7Zs8fUsSxnIi1TkZjmYBhLqQiGkmREQAREIIYIbN+/HTeMvAGZafvdXMfhyeOfyPFNAS7epn/P1zrjyUlPuTLAp/M+w7Fv9cKto2411zz8+7d/49zPTsW3C79FYtlEvDDlBfR7+0gc+96x2LovZ0fHK/+8jJYvtcSl316GVUmrMGHNBHCh9oGDuxiUjAiIgAhEFIFYTKyUilh86sqzCIiACPgQGPr95WjxYgvUfLImGj9dG5/MegeIS8Avl/+Bm476dw5pKgfvTHsVFRKrYcqVU/DYoMcwfvh4tK7TGa9NeQ7/rP/HyM/bMs/YbWq2wYP9HsSsa2ahTNlKZmH4K+79xtM9/LTkJ9z+20045I6MTHDDee2U1/D5uZ/jx4t+cn31JwIiIAIiEAkEpFREwlNSGn0I6FIERCDUBJ458RmjIEy7ehqG9bw2K/iMg2CDP+vi8PHVbIXghJYngAuq6VMuvhzO7XguT/H1wq+NzQXfNx1zBx4Z8IhZ6L3/0H70btjb+E3fMM3YPLw89RVa6Fi/O3rU72HOeejpdc5rGREQAREQgfAlIKUiiGfjOE4QUuEr0qhRI9SoUSN8ExilKeMXtVu0aAF9UTtKH3CUZat2hTqoU7GO+eIxRwpa1Opocjhi6stYk7TGnPNAxWBT9vUPC74yoxsc4aB5euLTFMGeg1mrVptXa2GUjuE/DEeVx6ug02ud8c+GqUaGU6jMiXuYvXmWewQaVG5gbB1ijICtYq0dadlvmglok7ISf2oVK1YE61jHibwXx3EiL83BPGApFUFQcpyshx8XF5m4+KOrU6dOEDmVSCgJVKhQAR06dEDZsmVDGazCEoFiJ+A4DlrWaJEVT2Y6Zm+enXXuHiuUrYBqrvLhnuL8Lpdi4fULPWbbHduw654UjDh1BFbsXIGWL7XAg6PvREJ8AiZfORkbbl2P09qcyltzmPqV6pvrjMwMY+sQWwQcJ6uORbYVcblv46Y4wpUKNwcR91elShVTxzpO5L04jpOVZsfJsiMOfi4JjsvFXc4iIAIiIAIxTKBWhVqe3C/ascicr05ajT9X/olBLQaZ62kbpxmFIbFMIqwZMX0Enp/yPD6c+yH27t9m5L447wt0qtPJnG/Zt8XY3gdOo+L1wm0LaXlMWka651wnIiACIiAC4U1ASkWxPx9FIAIiIALhRWBXyi4s3r4Y3MLVpmz9nvXGjdOb6Na6ZmtaxoxcOtLYD49/GM9Nfg7397sf9ao2w6rti3Dr77dic/Jms5sTt4K9a9StaFylMaonHu66/Xvd3+b+X5f9ismrx5pzKiizNs0Ct5e9puc1qOuGtylpFZ6f/DzSMtKwZPsSXP7dZUaWhynrp2DDng08lREBERABEQhDAlIqwvChKEkiIAKlQCCGovx64Tfo/no3vDXjLZQtV9mY2/+4w3XrAqsA3Nj7JlzQdSgQXw7T1v6Fuk/XMwuw7+p7N7ib09//+hvndbkMPy35Gc2frY+mzzbAe7Pfw3vnfILzOp6H6464Dtcd9R9ULF8LF31xFmo9VRuPTHgE31/yG9rW7Yq1u1bhmDePwKR1k0AF5p8rp+DENqe7Mg+j8mMV0dP1i3MOV1HcVerRvx6NoaekrIqACIhAZBE4XGJHVrpLNLX2IyXWLtHIQxDZunXrsGPHjhCEpCAKQiAlJQXLly9HWlpaQW6TrAgUO4Grel6JlPsOYO/de7Hnrj3GJLvnKfel4vgWx5v4K5WriPfPfB9Jd+7Bon+vwp+X/4Htd2zHsU37Gv/6levjw7M+xLKblmLTf3dh193JZm3FhZ0uNP4JZRLw3ODn3Hu2YeMdO7H65tWgIjK41WDMvnY2Uu5PRcoD6ejfrL+Rr1upLn646AfsdMNa+5/1bpp247sLv8O627Pu33n3frw65FUjq0N0EfDUrZnB5SvspFa7KdrtGv2VKIHk5GRTx3renxKNvWiR2TRbu2ihhc/dUiqCeBb2oVs7iFvCSmTVqlXYtm1bWKUpFhJDpWLx4sU4dOhQLGRXeYxSAlQOmlVrhs51O4PbxgbKZrXEamZNRSA/ulUvXx1UUngejKldsTbsKEWtCrXA+8uXLQ/HcYK5XTKRRsAqE9aOtPQvc99L9duV+FPbu3cvWMdGYtssEtMczAP2USqCuUUyIiACIiACIiACIiACIiACInCYgJSKwyx0JgKRQ0ApFQEREAEREAEREIEwIiClogAPY/PmzejcubPHdOnSBd26dcth6OYtw3O6dSsFua5du5q0XXHFFTjzzDPNOdPCNHkbK9fNK43hIMf02Mdzzz33mPTTjSYc0mcZ5savb9++uPHGG9GrVy/zzuQmZ8OxdlHkbBjedqDwvP3tubfce++9Z9HLFgERKAIB3Ro8gXXz1+G6jtcdNp3c884+hm7eMjynW2HleL+vCRSerwyvs+VuvOYGXD/kelzHNNDd12TLGX/K0PjK8LqE5R4Y/IDn4Vx55ZURVcey3ho4cKCpY1l38Zo22wfehu6+Jlg573DsuW9YvLZ+3jbdfY23/2+//eZhH00nUiqCeJqXX345EhISjOT8+fNhzbx58zBnzpwchm7W39p0Kw25uXPnmrStWLECS5YsMedMi02Xta2cdxrDQc4A9zqEW/ry47do0SKsXr0aCxYsMO9MSXC2afK2A8Xr7W/PveUs9ltuuQXlypWzl7JFQAREIOQEzht2nifM9QvXw2MWuOfzfQzdvGV4TrfCyvF+XxMoPF8ZXmfLrV61GmuXrMV6poHuviZbzvhThsZXhtclLOeBnn0SSXUs6y2up1jt1rE8p/Guw2xe6O5rcpGDr5wNw9v2leG1t789p7uvsX60s5Hj5ptvtqdRYUupCOIxXnvttXjttdfQsmXLHKZVq1Zo06ZNDkO3cJFr3bp1jrQxrYHSJ7k2EJeW8H0Phg8fjmeffRZly5YN4lciEREQAREoHIHH7n0Mtzx4C5o0b5LTtHCvW/oYukWDnG8eeM28+eaX7r4mxHLNWjaL2raCb3uM1751HdtGdPc1wcrxfl/jGxavfWVuu+02PPpodG2TLaUiyDKQU4i4Pai3WbZsmRkB4CiANXTzluE53ay/telGP29DN+tvbbp5y/Ccbtbf2nSjn7dZunRpUOkrdjl3lETpW45I4/zuu+8G+euQmAiIgAgUnkBimUQ8+8CzWLNyTU6zwr1e7mPoFg1yvnngNfPmm1+6+5oQy61avipq2wrebSJ7Hqgutn7edrByth3mbXuHY8+9/Xn+9NNPF/5HE6Z3SqkI0wejZImACIhASRNQfCIgAiIgAiJQWAJSKgpLTveJgAiIgAiIgAiIQMkTUIwiEJYEpFSE5WNRokRABERABERABERABEQgcghIqfB9VroWAREQAREQAREQAREQAREoEAEpFQXCJWEREIFwIaB0iIAIiIAIiIAIhA8BKRXh8yyUEhEQAREQARGINgLKjwiIQIwQkFIRIw9a2RQBERABERABERABERCBwASK7iqlougMFYIIiIAIiIAIiIAIiIAIxDQBKRUx/fiV+ZIioHhEQAREQAREQAREIJoJSKmI5qervImACIiACBSEgGRFQAREQAQKSUBKRSHB6TYREAEREAEREAEREIHSIKA4w5GAlIpwfCpKkwiIgAiIgAiIgAiIgAhEEAEpFRH0sEoqqYpHBERABERABERABERABApCQEpFQWhJVgREQATCh4BSIgIiIAIiIAJhQ0BKRdg8CiVEBERABERABEQg+ggoRyIQGwSkVMTGc1YuRUAEREAEREAEREAERKDYCES8UlFsZBSwCIiACIiACIiACIiACIhAUASkVASFSUIiIAJFJKDbRUAEREAEREAEopiAlIoofrjKmgiIgAiIgAgUjICkRUAERKBwBKRUFI6b7hIBERABERABERABERCB0iEQhrFKqQjDh6IkiYAIiIAIiIAIiIAIiEAkEZBSEUlPS2ktKQKKRwREQAREQMudiEYAABAASURBVAREQAREoAAEpFQUAJZERUAEREAEwomA0iICIiACIhAuBKRUhMuTUDpEQAREQAREQAREIBoJKE8xQUBKRUw8ZmVSBERABERABERABERABIqPgJSK4mNbUiErHhEQAREQAREQAREQAREoVQJSKkoVvyIXARGIHQLKqQiIgAiIgAhELwEpFdH7bJUzERABERABERCBghKQvAiIQKEISKkoFDbdJAIiIAIiIAIiIAIiIAIiYAmUtFJh45UtAiIgAiIgAiIgAiIgAiIQJQSkVETJg1Q2RCC0BBSaCIiACIiACIiACARPQEpF8KwkKQIiIAIiIALhRUCpEQEREIEwISClIkwehJIhAiIgAiIgAiIgAiIQnQRiIVdSKmLhKSuPIiACIiACIiACIiACIlCMBKRUFCNcBV1SBBSPCIiACIiACIiACIhAaRKQUlGa9BW3CIiACMQSAeVVBERABEQgaglIqYjaR6uMiYAIiEDJEPjhhx/w0ksv+UWWmZmJjIwMj7v3ucexmE8OHTqESy65BLt27QpJTOnp6TnCKY085UiALkSgGAgoSBEoDAEpFYWhpntEQAREoIgErrjiClx22WV+5rbbbsO7776LTZs2FTGG4G//6KOPcNJJJ6FXr16YNm1a8De6kmPGjME999wD5se9NH8bNmzAueeei759++LYY481YY8fPx4XXnghFi9ebGSCOezevRtnnXUWjjvuOFx//fXB3OInU7ZsWVxwwQU49dRTcfDgQT//YBx+/vlnk5/u3bujT58+Jj/PPvssqGCceeaZmD9/Pj7//HOcfPLJhiGZBBNuKGTeeustk54jjjjCpKOgYV511VV+76B9L++44w6Tr7S0tIIGW+zyKSkpOPvss9GvXz/861//Cml8W7duxTfffAMqxSENWIGJQJQTkFJR4g9YEYqACIgATEN80KBB+Pjjj01D+9577zVugwcPxujRo9GmTRuw4VoSrE5yFQqa6dOnF6hHn43+oUOH4tNPP0WlSpVMUvfv328aep07d8akSZOMoZJ0880346uvvgL9jWAQB4b58MMPY/369ViyZEkQdwQWOf3009GzZ0888MADgQVycd23bx+uvPJKM9IxYMAA/PPPP5gyZQqoZOzZswfM408//QQ2cE844QQwHjLcu3dvLiGG3vmMM84wvGfMmIGkpKQCR3DXXXeB7xzfwxUrVuDuu+/GQw89ZN7Fjh07GmWua9eu5hkUOPBivCEhIQF8N6gALFq0KKQx3XjjjUaJ/O2330IargITgWgnIKUi2p+w8icCIlA4AsV8V8uWLU0PM6Np3Lgx2rZti3bt2oGN008++cT0iHPU4ssvv6RIsZratWvjtNNOM3EUpHf2scceMyMRXbp0Mffy8Nlnn2Hjxo34z3/+w0tjGjRogF9++QXlypUz18Ee4uPjTcO9W7duwd6Sq9ydd96JV199FWvWrMlVxtfj3//+N9555x288cYbpnFt01+mTBnT8OazsvfUrFnTKBX2uqTsOnXq4JRTTil0dC1atDAjLAygefPmaN++PejGd5EK4xNPPIGFCxfiuuuuo0jYmLi4OHTq1AkcoSnIOxtMBjhSc+aZZ8L7vQ7mPsmIQKwTkFIR62+A8i8CIhCWBNgzzoR99913tIrdOI5j4nCcLNtc5HFgQ+7DDz/EDTfckEOKDVA6sNFH2xoqFoVt/DpOcGmycQWyGT+nJ33xxReBvP3c/vjjD6NQcMoWjZ+A6/C///0P7DF3T82f4xQ9nSagAh4cp/ji7d+/v0kNR2nMSZgdHMcBTSiTxaly/N01bNgwlMEqLBGIegJSKqL+ESuDIiACkUiAU2+Ybs7jp+1tuPiYax84j3/lypXeXuacc/059YjTdDg9xDgGODCOqVOnYtu2bQF883ZiI5PTfo4++ugcgh06dDDTgW655RYwfG9PLphu1KiRcdqyZYvpAee0HePgHphWTmVh2O5lwD9OhaJMQM98HNnzzMZiPmLG+/XXXzf2pZdeauxAh8qVK5t1I4H8Dhw4AE6F4rQt78XcweZ78+bNZo2EXYOSnJxspl8FM8WJayD4jlDxox0ofcG6/frrr0b0xBNPNLb3gVPA/vzzT4wbN86sL/H2K8588p2fPXu2X5ze8ef1G+B7yalenMrGd5is58yZY27nupvVq1ebZ7dz507jZg9kmdvvjsz//vtvcKSO7++8efMKvYbHxidbBCKNQD5KRaRlR+kVAREQgcgnwIW/nHJz1FFH+S1C5UgA57hzihQbRJzHz2k6Nte///47Bg4ciFWrVoEKAxvSb775pvU2NhtVHGE45phjwB75++67z2/EwQjmceBaAi7s9u0l5mhE3bp1wQXErVu3xvDhw/Hee++BDbRzzjkHnK7DBhgXT3Na0/nnn++J5fLLL0ePHj0CTudhfo4//ngzesAREipbXMvhuTmIE65TYYOPykt+4jNnzjQiTKM5yeXw8ssvm6lr3t4LFiwAla0PPvgAnEZz5JFHgnmmCSbfVo4srrnmGrO+gQoZFR2OuHBtind8vudPPfWUmWrG0SIulvf1D+aaCgmVhSeffNLkhVPdvO/j8+U6laVLl4KGC9iff/55I2LT361bN+T1fK1csPkcP368Yf3oo4+C7zmnn1llwEScfaBfXr8Bpn3AgAEmX9y1jIyZVi5M573cGIDv9tixY7NDhFGAc/vdcWcxPuNZs2aB7z6n+jE8/o49AehEBGKAgJSKGHjIymIMEFAWI5oAez+5IJhzuXv37m3WEXBNwoQJE8C5+jZz7EXlwlw2qF944QXTYOeaCzaMbAPm8ccfBxu15513nlnIevHFF4MLT9nDb8Phjj/sYZ44caJpsI4YMcIs1qU/G5O08zPsLeY6EF+5evXqYeTIkWZePnewev/9983OUByhoCJCea5JGOf2blPJ4LU1XBh70UUX2cscNkdT2IDnYmvm8bnnngPn/DOuHIJ5XFCpYP5Wuz3ReYiZERa79sKbf6B7uAahSpUqObyoxP31119geqn8ccSC6Qw235RjA3rIkCHgO8BpONy2lyypWDH/OSL0uWjSpAm4yJr3Uony8c71kmnme0hDZYjvENfb8B1jmPZGjoBxjQXz9H//93+4+uqr8e2334LKKXcSY/qDeb6UCzaf7PnnaAnXGXGdCxUA7tDEERw+U5s22uST12+AmwZQMaUsRxYYDllT4eXaIioW9LMmv98d88xpcNyhjMoMF5BTebT3yxaBWCEgpSJWnrTyKQJhRoA97tzlyNtwGoJvMtk48JbhOadW+Mqx0UE/b8OGqK8ceza9ZXi+Y8cOXzGw15F+3oY9kn6CIXBgr+jbb78NNk44QsEGIRs93F3JO3g2fpYvX252prHunG7UqlUr2Gkq7O3n6AUbOZRhwz01NRV2yhDzxYYUG46cvkMZGm7NSTtYw97+qlWrBhRnzzN7rzm9hD3d3OWKoypsaNnpPLyRuzvR9ja5NeK5cJgNeCvLnma6Pf3009YpX5sjJxQK9P7Q3ZrExESwwctrjurQLojh6IzNG58N7yUv2jTWj+fWBMo3G7ncEpdKoJVjeN5hWXdrc6ctPl+y53a+1j0Ym/J8D2moxPD3w5EGLuBmQ9mGwZ3KOBJjedKdIygcOeDz5jVNKPPJOCtWrIhhw4YxaGOqV69uFHBz4XXI7zdAUcubu57xWXN0gQqLtx/PafL73fG3NnnyZNx6661mihpHYPjMuBkD75cRgVghIKUiVp608ikCYUagVq1aaNq0aQ7DRoNvMoOVY4+qb3gVKlTwDc5Mv/GVK1++vJ8cpzH4yrGx6ScYYgeOQrz22mtGCWBDyjt4ziPnNXus2VNsDUcH2GinHxu0XAfA3mM28tizT3cqFrRtzzWnRfHa1/hOZ/L1t9fsIQ7EbdmyZWaqD3du4pQQ9ihzVIRTd5jGH3/80QaBYOPy3OBzwpESNp45hYu96t6GOz35iIPvV7Vq1cyaD18/72um3TaYV+czqsGF3+vWrfO+HWyEWweGxXP2dtOmKUi+OcJDxYL30TA877DoZs39998PNmbZyA3UoLdyBbH5/DgSw/eIU8K4VoSjYtyxzDccjmawE8Aq6aHMJ99bKpHeLGz8vvHk9xuw99HmFEDaeZn8fndUYvhb4+gZpyxS2eHUKXLLK1z5hTUBJa4QBKRUFAKabhEBESg6AfbCstfV23j3nNsY2Lj3luF5oAYTG9b08zZsRNpwrF2/fn14y/A8kPLBnlf6eZtAjWgbbiht2yPvOw2jRo0aJhruOsRGujWctsIGJT25QJojFZwmw8Y2p0nR3Rq7Laq9LqzNnt5A32NgfFyc7BsuR0aoIHKkxNfP+9p7UbO3e6BzytL8P3tnASVHsQXQmo+7uwW34O7B3d0hWHCXBA1ugQRycAluwTVISHAnBAlOEgguwYLD31uf2t+ZzO7MbmZnd2bvOVvb3eV1q+ec96req8Z3gP5kQ0MCHTsPKBaF6svGoZjxnHaAuC8U2BXIf2/zhVzK5ZvoEJcNjCP7nO6bUhf+E/hb4IzObkOqY2yurOKzK0EdKHD0B8EeR2jisiGNgZX6bHz2PuXJxnFPvVyzIcusKe9tsd9Ato30m8rG5d+nPA397tjBZKcDpYv3H3+KU045JfoS5dflswRqmYBKRS3PrmOTgASqkkAyK2KVnBN2GASr/Jj8cM8pM1xTQFBGuGWXAGdZvnCN2QrpqTz33bt3D+yEcM+KMtcUsgJcimvsilKIeUx+Hurp27dvfnQ0J0L4zq7iI7CSP5sZ5/Lsc7rPz0c8pmyYWmFehV9INuCfQp5sQAniBB/6no0vdI8fCo7ImFdxUlChPOxQMC+lKCnZ8k0Zd7ZcsXu+Ys5XptmtwG+A96FYmWLpjA/fG/Kh7LILwmo85obEZcPgwYMDuxUsBBBfznF26tQpOoTnKyz57wVjLvYbQAmgf6WGYr87HL8J/LY4NAEln10zTKJKbaOd5HOYNU5ApaLGJ9jhSUACbZdAEvjzfSdYlWVHBYEJJ19GgLCI7wXX3r17h2wZ7N0xiUEJIS/luBKwjeeK4oEgiLMy5hoIQQjZpJE/mQsV8zcgPwEBCqGe+/yAoyxH2mbjEbQQwjnFKMVjXjZixIhoLkUcjt0IsPQLYZa4FBBYk18IcazIM94ePXrwWFKgv+xKYTZVrAAKELsU7IDhC0LfsmVQqFBc+Dhcik/HvWbnJsWNGjUqZYsmf6WMm7K8I8xPKkwcilFWuCaO9HQ999xz4+lP+NPQT9IaC6m/vEPZfLTRtWvX+K7hwIxgT/pJJ50UcNRPjvfE4XCNskvbPBNKnV/6XWyc7D4wbkz/qJuAYoxDePad5Z0gLcss/zfA74s2yUe7XLMhpWHiR3yx3x27LxyWQP/IT6B9HMu5N0igvRBQqaiVmXYcEpBAVRFYd911Ax/ZQnhFccD/gJX2NIhevXoFTHgQ6jC7QBmdADC1AAAQAElEQVTAzIdjVDFzwcEUQYtTehCScGBlRZXTeDjClZNo2K3g5CBMj/C/oD5Mwjg9h1NqMJPCbwPhM/lcHHTQQdHhNPWjoSvtYlufL/yTH8UBW3zqx86cNviAHIoMTujkIXTu3Dng24HQzi4KR3ty1C196dixY8AXg3wElB76duSRRwaUIgTbfv36hVJs4ilPwC4fQa9UMzb8dDAh69ChQ8DHokuXLoF5oR98SA+FAuWKujE34jhd+GL6wnzgbM/xpMSx48HqNXmLjZtxUS/mb/CFGavemCGxC4Tp0SKLLBL5oMDBmza4ItRzmhgKAoI/+XA0pt1CAV8A+kgaSgK7MzDlimKL0M4RtbSbTJRw6sfECkE6+S+wO8JBA2mM1FfOcWJShD8OSgX+Qrz7xxxzTMAZmnHCCLO7Un4D9BtWMKMuxkN/CZzcxTtCWrdu3QLvb7HfHX4u7H6xa0a/OEkKJZLfCHUaJNBeCKhUtJeZdpwSkECbIJA6geDICUmslLJSjMCL0JrSEc5Y2edoWQRDhEXSMD/Bl4KdAAQiHIURYknD3AThm3oxw0DIQwBGmMcshJOBUj52OxDa8R3gyFBOnWLlF8Ese4IP+QsFVqER/NkxyKajvOBoTv84UhWBfOmll46naSFoZ/MitCMAo0wh5CE0YrpFX/koHmZN5EcoRjFCyIYRwjx9xW+E9FIDHwvkSN5S85MPcx4Eak6toiz29QjQmGlhBkQeAv3jWxqsbrNyjvCL8oPpFHHsGCRWxcaNUIvvCSvmlOWkNBQKFAtMd4hj14bxo3QOHz48EMf7wglGcGNnijhOVENppI+FAkoTxw2zso4iAnd2HLjSZ5yUUeRQXLPlUaooi7KE0ss7h+KYzVPOcVIvwj87WbzvKK0oQbwbcMffgx2oUn4DWWaMnV0W6ifwm6I+2LGbhJJAfGO/O0zl+vfvH+gDShbKJD4W+cyoxyCBWiagUlHLs+vYJCCBqibAzgKCGsIszrHZwbB6ymk4CDvZeO5ZUWZnI5uGMExaNrDCSh0IYqzeczwtK648Z/M1dM+uAgpBNh0FKLWL4oG5Fiu21JvNl71H+MQhnjjMjbjHT4FxEJcCz6xM40Sf4kq9IvAhYLN7U2qZbD5OYUKQxuSJMcI/m96c+1LH3Zy6K1UG53sUx8baK/c48dngPaFNfHRonx0/nlPgXSnlN5DyF7gWjGLe+c2kd5xMnPbEb4bdDd4NmBBvkEB7I6BS0d5m3PFKQAISKBMBdh5QTDD9KVOVLVIN5lTs2ODAi09FizRipRKQgATaOYHWUSraOXSHLwEJSKAWCLAajEkPduf4V7TVMWEShA8JpyO11T7aLwlIQALVTkClotpn0P5LoAUJWLUEihHADAXb+mHDhhXL2irpnMiDMzh+GK3SARuVgAQk0E4IqFS0k4l2mBKQgARaigAnTHHyUUvVPzb14ouCE/XY1FEFZe2iBCQggVYnoFLR6lNgByQgAQlIQAISkIAEap9AbY9QpaK259fRSUACEpCABCQgAQlIoMUJqFS0OGIbqBQB25GABCQgAQlIQAISaB0CKhWtw91WJSABCbRXAo5bAhKQgARqkIBKRQ1OqkOSgAQkIAEJSEACY0fA0hJoGgGViqbxMrcEJCABCUhAAhKQgAQkkEdApSIPSKUebUcCEpCABCQgAQlIQAK1QkClolZm0nFIQAItQcA6JSABCUhAAhIogYBKRQmQzCIBCUhAAhKQQFsmYN8kIIHWJqBS0dozYPsSkIAEJCABCUhAAhKocgIlKRVVPka7LwEJSEACEpCABCQgAQm0IAGVihaEa9USqDABm5OABCQgAQlIQAKtQkClolWw26gEJCABCbRfAo5cAhKQQO0RUKmovTl1RBKQgAQkIAEJSEACY0vA8k0ioFLRJFxmloAEJCABCUhAAhKQgATyCahU5BPxuVIEbEcCEmgHBP7666/RRvn333+P9uyDBCQgAQnUBgGVitqYR0chAQlUGYHXX3897LbbbmGfffYJ++23X7yef/75cRS333576Ny5c+jSpUtM23PPPUO/fv1i2qhRo+rLUZa0mNDAv0cffTS8++67DaSWEt30PPfdd1/YaqutwhJLLBFWWmmlsN5664UePXoEFIzNNtssvPHGG+Hmm28O66+/flhmmWVC//79m95IM0p8//33YfPNNw+rrrpq2H///ZtRQwh77bVX2HnnnQuGo446Ko7rzz//bFbdLVnol19+CVtssUVYbbXVwh577FHWpr788svAO/vPP/+UtV4rk4AEqouASkV1zZe9lYAEaoTA3HPPHQ477LAwcODAcPHFF4cpppgibLnllnF0CL0Iv9dcc01M22CDDaLwTeKEE04YBfannnoqDBo0KOy+++5EFwxDhgwJa6+9dhSA8zM8+OCD+VFj/fzzzz8HlJwdd9wxrL766uH5558Pzz33XEDJ+OGHH8IiiywS7r333oCAS7822WST8NJLL4Uff/xxrNsupYJJJ500nHzyyeGTTz4J77zzTilFxsjTtWvXsO6664brr78+fPDBB6Fbt26he/fu4dhjjw0LL7xwVFYWW2yx2MYYhVsxYoIJJohjRwHgvShnVw488MD4Tj700EPlrNa6mkrA/BJoZQIqFa08ATYvAQm0TwITTzxxWHTRRcMBBxwQAUw00URh9tlnj/fTTTdd2HjjjeMKPxHTTjttmHrqqbkN//nPf8JGG20UEFxvuummuBMQEwr8m3feecP2228/hlLx8ccfhzPPPLNAibGLOvjgg8OVV14ZLr300ihcjz/++LHCcccdNwreKBIxou7fNNNME1Aq6m4r9jfOOONExWbxxRdvdptzzTVX3GGhgjnnnDMsuOCCgbgFFlgg7LrrrpHrW2+9Ffbdd1+ytJnAe9OxY8ew9NJLh3LvKLBzww4U73ObGbAdkYAEKk5ApaI05OaSgAQk0CIE2JHI5XKhb9++Y9Q/cuTIGJef9ttvv4VvvvkmCrMxQwP/EOZvvPHGesUlZXvmmWfSbdmujzzySFQotttuu0AoVPGpp54aWDFPablcLt1W9JrLtVy7nTp1imNhlybetLF/uVwu5HK5svYKJffOO+8Ms8wyS1nrtTIJSKC6CKhUVNd82VsJ1AwBhNBevXqF8obK1PfAAw+UbR4QxFZYYYXA6vbbb79dXy8KBSvKrPbfcccdo60uP/zww2GdddaJcZ999lkYPHhwNOfBlv+VV14J3333Xaznq6++ivW+9tpr8Zl/+FdgqvP7779HsyNMj/744w+S6gN13n333dGPg3z1CY3cYMJF8k477cSlYJhsssnCyiuvXDDt119/jaZQmCVlnbm/+OKLOIaXX365vlwy4ckK7p9//nn01UgMf/rpp2h+Bcf6gg3c0DZtEvD7aCBbSdHJrIz5yS+ACRg+LgMGDIj+Jdn0lhznhx9+GE3lGhsb7fOevfjii/G9yvYNszZMvTBlw3QN1umdQsEdOnRonLtvv/02WyzwXlEf/jP0IZvIu4pyy24b84iPEXVl83gvAQlUFwGViuqaL3srgZohcMMNN4RDDjmkKsPVV19d1nlIvhTZHQl8DzArWXPNNcOIESOib0JqFOGPMgjM22yzTVhyySXD8ccfHzbddNPo0zDbbLMFFApMn3CWToL+m2++GQ466KCAwzJKzNZbbx0ITz75ZKwaJeaEE06I/hAoG9jIY+JDuZihkX8oMyQXMy268MILw/zzz0/W+kD9KFb4kGBGs9xyywWETsK2224bqJNxpgK77LJLHPOGG24Yo1I+OOC8jtKEXweKzswzzxxuu+22mK/QPwRfTLEwjcLsDCG3UL5icbBDWTjrrLMCYzn99NNHK3L55ZeHpZZaKjrNo9jhwJ4c81P/yz3OgQMHRtannXZaQBHF/CwpA6lzjJ/dsp49e0ZflyOPPDKsssoqASUj5aHv+MgwrgsuuCAeKkBfcUynXnyAcLh//PHHU5GoCGKix+8cRQRTN8zjyIDSu1zdHL/66qthhhlmCPfff3+cYxz4STdIQALVSUClojrnzV5LQAI1RAAFgeFklQp2ChCwOUUpm4YQiKNxhw4dwlRTTRVQCFj9R3DGSRoTo/nmmy/6XrAqjsBIeQKOxCgKCICLL7544J6wxhprkByuuuqqcMYZZwRMWVBEEHqp+8QTT4zpDf1jJXvYsGExGQE93jTwDx+EySeffLTUyy67LI4DhQMhFOdtdoMw30JQT3xSIfqMwpSeyYcAjUP7E088Ec1w4NenT5+w1lprxTGlvPlXBHp8T44++ujAGBh3fp6GnmEPcwJzhYKGYnLrrbfW+8dQFkd1fCwYEyd97b333gHFEEXwuuuuC/R/QN3uRTnHyco/uyVHHHFENEtDAeCEJnZwUIDoF4HdgrvuuivQbxQx8rzwwgvRB4Z0wiGHHBKuvfZabgNKF3lgPf3000ffHxSLmPjvP3YcUHBRaFFWOEwAJigkKA6MGTM4TuDi3cN5HuXx3+JeJCCBNkSgKV1RqWgKLfNKQAJlI4BAhUBTjYGV37KBqKtojjnmiA60rCK///77ASEdwQwBHeEMoRMhtC5rYDUYQYz7FMjHSjsr0Qh77BoQR3q6cl8ssMq+0EILRefjlJcVZpQThO8Ul3/lRCr6SDx959qUgNDJyUyUmWeeebgETJziTd2/lFZ3W/9XaFwIueONN1489jVlpL5sXSmeK7s5HGuLUIvjeqF2yNdQYEX/iiuuCASUGOpjR4XdHepM5Y477ri4e4HykuLYQWG+YJ7iCrXf3HHS5iSTTBJ22223VH1UQjmBqz6i7obdEw4FYJen7jHQHsfO8i7ynALx3HM8MHPN7gIKC3EpjXsCSgfvcVKIieO9Yi4wD0OhePbZZ8Phhx8eTdR4tziqlxPRyGuQgASqk4BKRXXOm72uCQLtexAIEJhTVGNgJ6Dcs5cEMAQyVrRRDmgDga1Tp05h6L9266Tnr2iTj5N3CgmlpJUSUAYQBL/++ut4chEr64R77rknMF4E5obqQaFJAjP9bCgf8bfcckvg9CnuU0AIT/fUxT1KFVdCLle6Y/Gss84aUCwoR6C+bF3EERjriiuuGFBq8WshrhyBHQF2YtjdQbnDX4PVeUzS8uvntC9Mv3C6Jy2XK9848VPgRKosC9og5HL/b4d3BrMzfB9451AiMY1ryJcGZtTRWBg0aFBMZqeIdyiFGWecMZpYYb6GQnXeeeeF5ZdfPio7KMtwiwX9JwEJVCUBlYqqnDY7LQEJ1BqBpChgAoX5UdZsKaVhQoLDLEJ+/vjTkbP58cWecU5GUWH1mJVthDx8EVJACcAcZqaZZmq0qmQ2xEp0Yxkxn8FhO5snl/u/kJvisyY6KS57pd/Z53Sfy5VWF/Wz2o4ytsMOOwQ+KpjqGOPahAhW8VGUKYJjcy6Xi0pOISfpNAZW6slfKKQ8+Wm5XOPjxME/v0yhZxQezJQw8VOpqgAACZFJREFU5eI4YJTI1P9C+Ut5z1IeTPHSe8SVNvDZwacC9ihdPXv2jP4Up5xySii3r1Kh/hsnAQm0HAGVipZja80SkIAESiaAaQiOrfgT4DPBqm4qjILBdwbOOeec6ESb4pt7pS6EasojTGNTjzCM/wSCcEojnUB6Wk3nuVDgA2iY0tBHFJ9CedihwGl3yimnLJTcYBx9y+8Tik6DBUpIoK8oZyg5OMIfeuihJZQqnoXx8WFCcuI/grKGooaPA3HZwKld7FbgrEx8OcfJ7hYO4fkKSz5HzL44lQoOSXHkmf4w5wj73Dcl4NRNfnaBuKbAbhjt4PhNQJnBeRtlA38UTKJSXq8SaA4By7QuAZWK1uVv6xKQgATqCaQdiS222KI+jhuETgR+7lMe7lMYOXJkSIJgiktX0nDOTc9c+QDa0KFDA6vgKDGs1hOPnwv5syvGn376aeAkoymmmIIsDQZ2H9ilQBnC6ZZjabOZMZ/iNCuE2BRPW9xzGhVXQopD2eGZgM8Jgn8SkKkbwZ0TqhDiyUOgLByygjNxOLensikf8dxjtoWTNs7ifLSPuGIh9TffrIo2unbtGk/Xwk8BwZ66TjrppMCRqpzoxTMBx3KE7nPPPZfHGMo5zsMOOywe6dqnT59YN/8wtRowYMBoJzt99NFHAV65XI4sAbYI9ygAzD2MSUi84MtzNqS09J5xEADvcO/evSOLlBc/E5jx3uG0zbykNPqAY3l69ioBCVQfAZWK6puzVuqxzUpAAi1NICkMCGT5beFzwVGsHTt2rE9iJZlTnFi1xw4eZ1tO1iEDQhorwZiZ4KiMIpGEYWzaca7mxCJW6NkJoQz28jgcI4iycowjLYoANvesopOnscAJQnx/pEOHDgFhvUuXLvE7JBxji0M0CgV9og6cmzkSFpMrVsNxKOaEIY4nJY4dD/pA3s6dOwcUH5SV7t27xyNNDzjggIDdPzz69esXqJdTiFAyOOUKwRgzHszJMD3CQRlnd3jhP0CgDEwQ8CeaaKKAIJ4UAdotFPAFoI+koSSw4wE3rqz0D6gT2s8+++z4McNc7n+COo7PmLQhSOOUjqkYjsnMVRoj9ZVrnDjWM05MmZhLTptibMccc0zAlwkFB0Z8EwSFkfcKB2xOeSIvO1McKctHDHnvOBGMk6GYF9IZD/0loIyhDJDWrVu3gJ8EO2F8dBFFmHppm3cNkyzmGT8XnOp5X0nDlwMlkjzUaZCABKqTgEpFdc6bvZaABGqQACfkIOBiEpM/PAROBLVsPE7cOMWymswqMTbqO++8c8ySy+UC3wFgFZlVZ3Yk0m7DXHPNFT8Uh5CP6QmmV7FQ3T+OYOVY1osuuiigcDz22GOjnQZVl6XRP/qOIM9H6Di5Cvt6BGgUH8yAUmGOYWWVnH4jwCL8ouxgOkUcOxsck0t+lBWUgEsuuSQg3CIs8y2K9957L/BRPITaNFbKomChUKBYYL9P3JAhQwLKGrxYbYcZZWCCEI5QCyeUAtpsKKA0YZ6G0saqO+2z48CVPlM/33pAgM7WgVJFWZQlfA0wTUJoz+Yp1ziZQ+pF+GdHB6UNB3KUIPxn4I6ZG8oEebgnDQGfK99GGT58eHx/ll122cDuC89wZOy8o9RPQMmgPtLYTaIO4jH7wo/i6aefDuTBN4d+kIapXP/+/eO3V1CyUCZRfvOZkdcgAQlUDwGViuqZK3sqAQm0AwJpFTx/qKwEs7KbH9/cZwQ4lBhO/ylUBwIuJykVSisljrII0ux0sEvA6nUp5RrLQ5+SAoSZFff4Z+Ry/9sRaKxsW0mbdtppQ4cOHRrtTrnHifkcvGiUk7ZoH3M1nlNAuUIhTM+8b7wj6bm5V+adU6hQMlIdfF+FnS/a4N2ASUprT1fHKoFaI6BSUWsz6ngkIAEJSEACEpCABCRQYQI1qlRUmKLNSUACEpCABCQgAQlIoB0TUKlox5Pv0CXQ6gTsgAQkIAEJSEACNUFApaImptFBSEACEpCABFqOgDVLQAISKEZApaIYIdMlIAEJSEACEpCABCTQ9gm0ag9VKloVv41LQAISkIAEJCABCUig+gmoVFT/HDqCShGwHQlIQAISkIAEJCCBggRUKgpiMVICEpCABKqVgP2WgAQkIIHKE1CpqDxzW5SABCQgAQlIQALtnYDjrzECKhU1NqEORwISkIAEJCABCUhAApUmoFJRaeKVas92JCABCUhAAhKQgAQkUCECKhUVAm0zEpCABAoRME4CEpCABCRQCwRUKmphFh2DBCQgAQlIQAItScC6JSCBIgRUKooAMlkCEpCABCQgAQlIQAISaJxA21AqGu+jqRKQgAQkIAEJSEACEpBAGyagUtGGJ8euSaCtEbA/EpCABCQgAQlIoBABlYpCVIyTgAQkIAEJVC8Bey4BCUig4gRUKiqO3AYlIAEJSEACEpCABCRQWwRUKmprPh2NBCQgAQlIQAISkIAEKk5ApaLiyG2wUgRsRwISkIAEJCABCUigMgRUKirD2VYkIAEJSKAwAWMlIAEJSKAGCKhU1MAkOgQJSEACEpCABCTQsgSsXQKNE1CpaJyPqRKQgAQkIAEJSEACEpBAEQIqFUUAVSrZdiQgAQlIQAISkIAEJFCtBFQqqnXm7LcEJNAaBGxTAhKQgAQkIIECBFQqCkAxSgISkIAEJCCBaiZg3yUggUoTUKmoNHHbk4AEJCABCUhAAhKQQI0RaJZSUWMMHI4EJCABCUhAAhKQgAQkMBYEVCrGAp5FJdDGCdg9CUhAAhKQgAQkUBECKhUVwWwjEpCABCQggYYIGC8BCUig+gmoVFT/HDoCCUhAAhKQgAQkIIGWJmD9jRJQqWgUj4kSkIAEJCABCUhAAhKQQDECKhXFCJleKQK2IwEJSEACEpCABCRQpQRUKqp04uy2BCQggdYhYKsSkIAEJCCBMQmoVIzJxBgJSEACEpCABCRQ3QTsvQQqTEClosLAbU4CEpCABCQgAQlIQAK1RkClonkzaikJSEACEpCABCQgAQlI4F8CKhX/gvAiAQnUIgHHJAEJSEACEpBAJQioVFSCsm1IQAISkIAEJNAwAVMkIIGqJ6BSUfVT6AAkIAEJSEACEpCABCTQ8gQaa0GlojE6pklAAhKQgAQkIAEJSEACRQmoVBRFZAYJVIqA7UhAAhKQgAQkIIHqJKBSUZ3zZq8lIAEJSKC1CNiuBCQgAQmMQUClYgwkRkhAAhKQgAQkIAEJVDsB+19ZAioVleVtaxKQgAQkIAEJSEACEqg5Av8FAAD//3xPUwgAAAAGSURBVAMA7rTqMbfaEPUAAAAASUVORK5CYII=)" + ], + "metadata": { + "id": "SJ5y5VS4rL4Z" + }, + "id": "SJ5y5VS4rL4Z" + }, + { + "cell_type": "markdown", + "source": [ + "The chief drawback of this feature is that it must be specified at **save** time, meaning that you must have some insight into the various ways in which the checkpoint will be loaded with resharding in order to optimize.\n", + "\n", + "If such patterns are not known in advance, or are variable depending on the use case, it is still valuable to specify a `chunk_byte_size` 10MB or greater." + ], + "metadata": { + "id": "fRdXdLxcs6oo" + }, + "id": "fRdXdLxcs6oo" + }, + { + "cell_type": "markdown", + "source": [ + "The `StorageOptions` config can be used to limit the size of chunks. This\n", + "config can apply to all arrays globally, or can be used with `scoped_storage_options_creator` to apply differently to individual weights." + ], + "metadata": { + "id": "txIvTnaaFTVX" + }, + "id": "txIvTnaaFTVX" + }, + { + "cell_type": "code", + "source": [ + "import inspect\n", + "\n", + "doc = inspect.getdoc(ocp.options.ArrayOptions.Saving.StorageOptions)\n", + "print(doc)" + ], + "metadata": { + "id": "RpJKQFMKqS1r" + }, + "execution_count": null, + "outputs": [], + "id": "RpJKQFMKqS1r" + }, + { + "cell_type": "markdown", + "source": [ + "In the following example, we limit the chunk byte size to 16 bytes. For a 128-element array sharded across 16 devices, the shard shape (called `write_shape` below) is `(8,)`. Limited to 16 bytes, the shape becomes `(4,)`, as you see below with `chunk_shape`." + ], + "metadata": { + "id": "uSGcefQ87a_p" + }, + "id": "uSGcefQ87a_p" + }, + { + "cell_type": "code", + "source": [ + "arr = jnp.ones(\n", + " (128,), # 128 * 4 bytes\n", + " device=jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('x',)),\n", + " jax.sharding.PartitionSpec('x'),\n", + " )\n", + ")" + ], + "metadata": { + "id": "2-uTR8bg6lpy" + }, + "execution_count": null, + "outputs": [], + "id": "2-uTR8bg6lpy" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "with ctx:\n", + " path = root_directory / 'ckpt_not_subchunked'\n", + " ocp.save(path, {'a': arr}, overwrite=True)\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "b0XeIJdB2F3I" + }, + "execution_count": null, + "outputs": [], + "id": "b0XeIJdB2F3I" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.storage_options.chunk_byte_size = 16\n", + "with ctx:\n", + " path = root_directory / 'ckpt_subchunked'\n", + " ocp.save(path, {'a': arr}, overwrite=True)\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "VYfd4rIz2sgg" + }, + "execution_count": null, + "outputs": [], + "id": "VYfd4rIz2sgg" + }, + { + "cell_type": "markdown", + "source": [ + "Let's imagine our model has the following (32, 32) array, with shard shapes of (4, 16):" + ], + "metadata": { + "id": "FJSJiUBZ615K" + }, + "id": "FJSJiUBZ615K" + }, + { + "cell_type": "code", + "source": [ + "arr = jnp.ones(\n", + " (32, 32),\n", + " device=jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(np.array(jax.devices()).reshape(8, 2), ('x', 'y')),\n", + " jax.sharding.PartitionSpec('x', 'y')\n", + " )\n", + ")" + ], + "metadata": { + "id": "1d1_DCd96-GW" + }, + "execution_count": null, + "outputs": [], + "id": "1d1_DCd96-GW" + }, + { + "cell_type": "code", + "source": [ + "arr.sharding.shard_shape(arr.shape)" + ], + "metadata": { + "id": "IgsoAWlD7MtM" + }, + "execution_count": null, + "outputs": [], + "id": "IgsoAWlD7MtM" + }, + { + "cell_type": "markdown", + "source": [ + "Now let's suppose we want to load it with the following sharding, giving a shard shape of (32, 2):" + ], + "metadata": { + "id": "QZ1fc_-37Sce" + }, + "id": "QZ1fc_-37Sce" + }, + { + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(np.array(jax.devices()).reshape(1, 16), ('x', 'y')),\n", + " jax.sharding.PartitionSpec(None, 'y')\n", + ")\n", + "sharding.shard_shape(arr.shape)" + ], + "metadata": { + "id": "H4wqwXav7c8Y" + }, + "execution_count": null, + "outputs": [], + "id": "H4wqwXav7c8Y" + }, + { + "cell_type": "markdown", + "source": [ + "Without subchunking, in order to read a single shard of shape (32, 2), we would have to read all 16 columns in the origin chunk, discarding 14, in order to obtain the 2 we actually want." + ], + "metadata": { + "id": "waWYFMUi7CRg" + }, + "id": "waWYFMUi7CRg" + }, + { + "cell_type": "markdown", + "source": [ + "To fix this, we can ensure that our checkpoint is saved with sub-chunking along the column dimension. Now the `chunk_shape` becomes `(4, 1)` while `write_shape` (the original shard shape) remains the same." + ], + "metadata": { + "id": "X3udRKEP5R12" + }, + "id": "X3udRKEP5R12" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.storage_options.chunk_byte_size = 16\n", + "ctx.array.saving.storage_options.shard_axes = (1,)\n", + "\n", + "with ctx:\n", + " path = root_directory / 'ckpt_subchunked_with_shard_axes'\n", + " ocp.save(path, {'a': arr}, overwrite=True) # 32 * 32 * 4 bytes\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "W2oOq6-A5fa2" + }, + "execution_count": null, + "outputs": [], + "id": "W2oOq6-A5fa2" + }, + { + "cell_type": "markdown", + "source": [ + "A large-scale performance evaluation of this feature on Llama 3.1 70B model shows positive results for various resharding patterns, as shown below." + ], + "metadata": { + "id": "cv-ojAhH87w8" + }, + "id": "cv-ojAhH87w8" + }, + { + "cell_type": "markdown", + "source": [ + "| Model | Origin Topology | Sharding (data, fsdp, tensor) | Target Topology | Sharding (data, fsdp, tensor) | Subchunked | Not Subchunked | Speedup |\n", + "| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n", + "| Llama 3.1 70B | v5p-256 | (4, 8, 4) | v5p-256 | (1, 128, 1) | 13.41 ± 1.44 | 45.23 ± 2.69 | **3.37×** |\n", + "| Llama 3.1 70B | v5p-256 | (4, 8, 4) | v5p-64 | (1, 32, 1) | 14.40 ± 0.51 | 34.40 ± 2.84 | **2.39×** |\n", + "| Llama 3.1 70B (baseline) | v5p-256 | (4, 8, 4) | v5p-256 | (4, 8, 4) | 15.99 ± 1.50 | - | - |\n", + "| Llama 3.1 8B | v5p-32 | (8, 2, 1) | v5p-32 | (1, 16, 1) | 3.59 ± 0.17 | 20.40 ± 2.56 | **5.68×** |\n", + "| Llama 3.1 8B | v5p-32 | (8, 2, 1) | v5p-8 | (1, 4, 1) | 10.88 ± 0.49 | 21.07 ± 3.12 | **1.94×** |\n", + "| Llama 3.1 8B (baseline) | v5p-32 | (8, 2, 1) | v5p-32 | (8, 2, 1) | 26.20 ± 3.44 | - | - |" + ], + "metadata": { + "id": "UpSdyFebKswg" + }, + "id": "UpSdyFebKswg" + }, + { + "cell_type": "markdown", + "source": [ + "### Memory Usage" + ], + "metadata": { + "id": "F8kDkkDGdr2r" + }, + "id": "F8kDkkDGdr2r" + }, + { + "cell_type": "markdown", + "source": [ + "#### Loading" + ], + "metadata": { + "id": "hZ6q8zs8PBkt" + }, + "id": "hZ6q8zs8PBkt" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e770d3fb" + }, + "source": [ + "When loading, a checkpoint may consume substantial amounts of RAM before being transferred to accelerators. Depending on the ratio of available accelerator to host memory, as well as the size of the checkpoint, you may find that it is necessary to set a cap on RAM usage.\n", + "\n", + "The feature can also be secondarily useful purely for rate-limiting traffic to storage.\n", + "\n", + "Note that is not possible to set a cap on accelerator memory usage, as this is the ultimate destination and must fit the entire model. RAM is merely a transient resource for loading.\n", + "\n", + "To lower concurrent RAM usage, you can enable `read_concurrent_bytes` to ensure no more than the specified limit can be in use at once before transferring loaded weights to the accelerators." + ], + "id": "e770d3fb" + }, + { + "cell_type": "code", + "metadata": { + "id": "a4b1eb45" + }, + "source": [ + "ctx = ocp.Context()\n", + "with ctx:\n", + " path = root_directory / 'ckpt_memory_test'\n", + " arrs = [jnp.ones((1024, 1024)) for _ in range(150)] # 10 x ~4MB arrays\n", + " ocp.save(path, arrs, overwrite=True)" + ], + "execution_count": null, + "outputs": [], + "id": "a4b1eb45" + }, + { + "cell_type": "code", + "metadata": { + "id": "2b9c4e43" + }, + "source": [ + "def load_unlimited():\n", + " ctx = ocp.Context()\n", + " with ctx:\n", + " ocp.load(path)\n", + "\n", + "def load_limited():\n", + " ctx = ocp.Context()\n", + " ctx.memory_options.read_concurrent_bytes = 1024 * 1024 * 5\n", + " with ctx:\n", + " ocp.load(path)\n", + "\n", + "load_unlimited()\n", + "load_limited()" + ], + "execution_count": null, + "outputs": [], + "id": "2b9c4e43" + }, + { + "cell_type": "markdown", + "source": [ + "#### Saving" + ], + "metadata": { + "id": "ol0nsGU2PEfJ" + }, + "id": "ol0nsGU2PEfJ" + }, + { + "cell_type": "markdown", + "source": [ + "Memory-limited saving is more complex than loading because saving is typically assumed to be asynchronous, while loading blocks experiment start-up. This means that limiting the transfer from accelerators to storage at saving time in order to limit RAM consumption necessarily introduces storage I/O wait time into the blocking path of save, transforming an asynchronous save into a synchronous one.\n", + "\n", + "Orbax team is currently developing features for memory-limited saving as well as recommendations for their use - please stay tuned!" + ], + "metadata": { + "id": "HGdT-xhqPH2j" + }, + "id": "HGdT-xhqPH2j" + }, + { + "cell_type": "markdown", + "source": [ + "## Performance at Scale" + ], + "metadata": { + "id": "eJbY7B0AdLr1" + }, + "id": "eJbY7B0AdLr1" + }, + { + "cell_type": "markdown", + "source": [ + "For our purposes \"at scale\" refers to training workloads of thousands of devices, often using multiple TPU pods and partitioning the model using multiple\n", + "data-parallel replicas. This scale presents unprecedented challenges, but also offers unique opportunities for performance improvements. Below, we show a few key features that Orbax provides to unlock checkpointing performance at scale." + ], + "metadata": { + "id": "jJTB2787iXMJ" + }, + "id": "jJTB2787iXMJ" + }, + { + "cell_type": "markdown", + "source": [ + "For saving and loading benchmarks, we use `n` slices of 64 n2-standard-4 machines with\n", + "a Llama 3.1 70B model fully sharded 64 ways and replicated `n` ways for `n ∈ {2, 4, 8, 16, 32}`.\n", + "This CPU-only approach to benchmarking exercises the actual resources required for large-\n", + "scale checkpointing, namely network and distributed storage, without incurring substantial\n", + "monetary cost from accelerator usage. While device / host transfer throughput cannot\n", + "be measured with this approach, inter-slice communication between slices use DCN (data\n", + "center network) connections regardless of whether TPU or CPU hosts are used." + ], + "metadata": { + "id": "6QcU6UthhLRX" + }, + "id": "6QcU6UthhLRX" + }, + { + "cell_type": "markdown", + "source": [ + "### Load-Balanced Saving (Replica-Parallel Mode)" + ], + "metadata": { + "id": "IAGWtvZ-dXs4" + }, + "id": "IAGWtvZ-dXs4" + }, + { + "cell_type": "markdown", + "source": [ + "For models with more than one data-parallel replica, traditional checkpointing avoids duplicate work by saving only one copy of the model, typically model replica 0 (located on the set of hosts corresponding to physical slice 0, or more precisely, the set of array shards with `replica_id == 0`). Even if most model weights are fully-sharded, a fraction may still be fully replicated, in which case the same de-duplication logic applies.\n", + "\n", + "By default, Orbax improves on the traditional behavior with its *replica-parallel* feature, which slices shards on device to balance saving work\n", + "across hosts. Concretely, this means that for an array shard with `n` replicas, Orbax obtains `n` unique sub-shards that are then transferred and written from each host independently. This translates to reduced I/O and memory pressure on all hosts.\n", + "\n", + "Currently this behavior is enabled by default based on our evaluations, which show write-time speedups for many levels of replication. However, the behavior may require tuning for specific scenarios, as slicing shards too finely results in an elevated number of queries per second (QPS) for a constant data volume, potentially degrading performance for some topologies or storage backends." + ], + "metadata": { + "id": "dazMPJTsBunw" + }, + "id": "dazMPJTsBunw" + }, + { + "cell_type": "markdown", + "source": [ + "Options for tuning can be found under `ocp.Context.array_options.saving`." + ], + "metadata": { + "id": "a8NV3hsWK-q5" + }, + "id": "a8NV3hsWK-q5" + }, + { + "cell_type": "markdown", + "source": [ + "| Slices (`n` X 64-n2-standard-4) | Single-Slice | Replica-Parallel | Speedup |\n", + "| :--- | :--- | :--- | :--- |\n", + "| 2 | 49.20 ± 12.28 | 29.40 ± 3.00 | **1.67×** |\n", + "| 4 | 49.63 ± 6.86 | 25.12 ± 7.19 | **1.98×** |\n", + "| 8 | 57.46 ± 20.19 | 24.47 ± 6.97 | **2.35×** |\n", + "| 16 | 77.53 ± 18.88 | 32.51 ± 13.88 | **2.39×** |\n", + "| 32 | 131.83 ± 20.37 | 77.00 ± 19.85 | **1.71×** |" + ], + "metadata": { + "id": "pAybgNi6LJ1s" + }, + "id": "pAybgNi6LJ1s" + }, + { + "cell_type": "markdown", + "source": [ + "### Scalable Loading (Load+Broadcast Mode)" + ], + "metadata": { + "id": "EBh0ljmJdcTK" + }, + "id": "EBh0ljmJdcTK" + }, + { + "cell_type": "markdown", + "source": [ + "For large training runs where the model is partitioned into data-parallel replicas, parallel reads from all hosts introduce excessive load on the distributed storage. Since our model is replicated, however, we can reduce duplicate reads by a factor of `n` (the number of model replicas). Instead, only hosts belong corresponding to the primary model replica need to perform a read from disk. After being loaded to the primary-replica devices, it can be broadcast to all other devices.\n", + "\n", + "For example, we may have 32 slices, each consisting of 1024 TPUs (256 hosts), giving 32768 chips total. All 256 hosts of slice 0 perform the read from disk. After loading the model onto the corresponding 1024 TPUs, the model is broadcast to all 31 other slices over the DCN (data-center network).\n", + "\n", + "Note that the load+broadcast feature is tied to the logical notion of a model replica rather than any physical notion of TPU pods / slices. Replication is determined exclusively from the device mesh of the arrays (assuming SPMD) along with a `replica_axis_index` parameter. In our example, the mesh might have axes like: `('replica', 'fsdp', 'tensor'): [32, 1024, 1]` with `replica_axis_index=0`." + ], + "metadata": { + "id": "xasMShwDgg8I" + }, + "id": "xasMShwDgg8I" + }, + { + "cell_type": "markdown", + "source": [ + "| Slices (`n` X 64-n2-standard-4)| Broadcast | No Broadcast | Speedup |\n", + "| :--- | :--- | :--- | :--- |\n", + "| 2 | 35.31 ± 1.69 | 27.88 ± 2.10 | 0.79× |\n", + "| 4 | 33.33 ± 2.55 | 51.77 ± 3.22 | **1.55×** |\n", + "| 16 | 38.45 ± 1.45 | 156.03 ± 13.68 | **4.06×** |\n", + "| 32 | 46.85 ± 3.33 | 231.13 ± 10.11 | **4.93×** |" + ], + "metadata": { + "id": "KCMVqxSLLNOP" + }, + "id": "KCMVqxSLLNOP" + }, + { + "cell_type": "markdown", + "source": [ + "As we can see, load+broadcast mode offers a significant speedup relative to the baseline approach. The relative benefit improves with larger scale. Note that at the 2-slice scale, the overhead of broadcasting outweighs the benefit." + ], + "metadata": { + "id": "dCxr9f9QyrAw" + }, + "id": "dCxr9f9QyrAw" + }, + { + "cell_type": "markdown", + "source": [ + "In reality, the performance gains from load+broadcast mode are likely understated by this benchmark, since TPU hosts are equipped with higher-throughput network interfaces. We were also forced to rely on gloo, which is largely unproven performance-wise, rather than megascale as our collectives implementation for this benchmark." + ], + "metadata": { + "id": "JTcfOhAJhCd6" + }, + "id": "JTcfOhAJhCd6" + }, + { + "cell_type": "markdown", + "source": [ + "Let's take a look at an example. In this example, we have a mesh with shape `(2, 8)`, where the row dimension represents a \"slice\" where each row has a replica of our \"model\"." + ], + "metadata": { + "id": "G_SFxPguHnBh" + }, + "id": "G_SFxPguHnBh" + }, + { + "cell_type": "code", + "source": [ + "num_replicas = 2\n", + "num_devices_per_replica = 8\n", + "assert num_replicas * num_devices_per_replica == jax.device_count()\n", + "\n", + "device_array = np.asarray(jax.devices()).reshape(\n", + " (num_replicas, num_devices_per_replica)\n", + ")\n", + "mesh = jax.sharding.Mesh(device_array, ('replica', 'model'))\n", + "pspec = jax.sharding.PartitionSpec('model')\n", + "sharding = jax.sharding.NamedSharding(mesh, pspec)\n", + "arr = jnp.arange(1024, device=sharding)\n", + "abstract_arr = ocp.arrays.to_shape_dtype_struct(arr)\n", + "\n", + "jax.debug.visualize_array_sharding(arr)" + ], + "metadata": { + "id": "u8TcxqvBFuaG" + }, + "execution_count": null, + "outputs": [], + "id": "u8TcxqvBFuaG" + }, + { + "cell_type": "markdown", + "source": [ + "As we can see, each array shard is replicated onto one device from each \"slice\"." + ], + "metadata": { + "id": "IqcLnJruHgti" + }, + "id": "IqcLnJruHgti" + }, + { + "cell_type": "code", + "source": [ + "path = root_directory / 'load_and_broadcast'\n", + "ocp.save(path, [arr], overwrite=True)" + ], + "metadata": { + "id": "LSVOs0F0HEcO" + }, + "execution_count": null, + "outputs": [], + "id": "LSVOs0F0HEcO" + }, + { + "cell_type": "markdown", + "source": [ + "```{warning}\n", + "Make sure to specify **`replica_axis_index`**. This must correspond to the replicated axis of your mesh.\n", + "```" + ], + "metadata": { + "id": "dEXIZIxmIu1U" + }, + "id": "dEXIZIxmIu1U" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array_options.loading.use_load_and_broadcast = True\n", + "ctx.array_options.loading.load_and_broadcast_options.replica_axis_index = 0\n", + "with ctx:\n", + " loaded = ocp.load(path, [abstract_arr])\n", + "\n", + "jax.debug.visualize_array_sharding(loaded[0])" + ], + "metadata": { + "id": "fcZqsZ4hIJn6" + }, + "execution_count": null, + "outputs": [], + "id": "fcZqsZ4hIJn6" + } + ] +} diff --git a/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb b/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb index 2ce451c2bc..053649a4b8 100644 --- a/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb +++ b/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb @@ -288,7 +288,7 @@ }, "cell_type": "code", "source": [ - "!ls {directory / 'experiment'}" + "import os; print(sorted(os.listdir(directory / 'experiment')))" ], "outputs": [], "execution_count": null diff --git a/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb b/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb index 69d5cc66e0..4480ea8065 100644 --- a/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb +++ b/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb @@ -146,7 +146,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/migration/root_dir/0" + "print(sorted([p.name for p in (root_dir / '0').iterdir()]))" ], "outputs": [ { @@ -282,7 +282,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/migration/custom_checkpoint/my_checkpoint" + "print(sorted([p.name for p in my_checkpoint_dir.iterdir()]))" ], "outputs": [ { diff --git a/docs/guides/checkpoint/v1/training.ipynb b/docs/guides/checkpoint/v1/training.ipynb index 8f5b8262c8..b012f07c61 100644 --- a/docs/guides/checkpoint/v1/training.ipynb +++ b/docs/guides/checkpoint/v1/training.ipynb @@ -313,7 +313,7 @@ }, "outputs": [], "source": [ - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -374,7 +374,7 @@ }, "outputs": [], "source": [ - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -1128,7 +1128,7 @@ " # Update model and optimizer separately\n", " nnx.update(model, loaded_state['params'])\n", " nnx.update(optimizer, loaded_state['optimizer'])\n", - " last_step = loaded_state['optimizer']['step'].value\n", + " last_step = loaded_state['optimizer']['step'].get_value()\n", " else:\n", " last_step = 0\n", "\n", @@ -1146,7 +1146,7 @@ " model, optimizer, last_step = init_or_restore(ckptr, abs_state, ckpt_path)\n", "\n", " # Main training loop\n", - " with mesh:\n", + " with jax.set_mesh(mesh):\n", " for step, batch in enumerate(train_ds, start=last_step + 1):\n", " if step >= train_steps or (stop_fn and stop_fn(step)):\n", " break\n", @@ -1184,7 +1184,7 @@ "train(stop_fn=simulated_failure)\n", "latest = training.Checkpointer(root_directory).latest\n", "assert latest is not None and latest.step == 50\n", - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -1230,7 +1230,7 @@ "assert latest is not None and latest.step == 80\n", "assert (root_directory / '80').exists()\n", "train(ckpt_path=root_directory / '80')\n", - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] } ], diff --git a/docs/index.rst b/docs/index.rst index 0794543563..9d60f4de99 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -106,6 +106,7 @@ API that is **easy to use**, **highly performant**, and **maximimally compatible guides/checkpoint/v1/async_checkpointing guides/checkpoint/v1/checkpoint_format + guides/checkpoint/v1/maximizing_performance .. toctree:: :hidden: