From 2f27fadd2d032d2afff60d1058406fb5010db1e0 Mon Sep 17 00:00:00 2001 From: Orbax Authors Date: Tue, 16 Jun 2026 08:53:45 -0700 Subject: [PATCH] Internal PiperOrigin-RevId: 933114409 --- .../serialization/jax_array_restore_args.py | 18 +- .../experimental/tiering_service/db_lib.py | 250 ++ .../tiering_service/db_lib_test.py | 199 ++ .../tiering_service/gcp_storage_client.py | 455 ++++ .../gcp_storage_client_test.py | 281 ++ .../tiering_service/storage_backend.py | 11 +- .../tiering_service/storage_backend_test.py | 7 +- .../v1/_src/layout/orbax_layout.py | 2 +- checkpoint/pyproject.toml | 1 + .../checkpoint/v1/async_checkpointing.ipynb | 4 +- .../checkpoint/v1/checkpoint_format.ipynb | 2 +- .../checkpoint/v1/checkpointing_pytrees.ipynb | 2346 +++++++++-------- .../v1/maximizing_performance.ipynb | 848 ++++++ .../checkpoint/v1/orbax_checkpoint_101.ipynb | 2 +- .../v1/orbax_v0_to_v1_migration.ipynb | 4 +- docs/guides/checkpoint/v1/training.ipynb | 12 +- docs/index.rst | 1 + 17 files changed, 3288 insertions(+), 1155 deletions(-) create mode 100644 checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py create mode 100644 checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py create mode 100644 docs/guides/checkpoint/v1/maximizing_performance.ipynb diff --git a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py index 20b7dab544..768bf8d26a 100644 --- a/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py +++ b/checkpoint/orbax/checkpoint/_src/serialization/jax_array_restore_args.py @@ -101,11 +101,13 @@ class SingleReplicaArrayRestoreArgs(ArrayRestoreArgs): def __post_init__(self): super().__post_init__() - logging.log_first_n( - logging.WARNING, - '`single_replica_sharding` is deprecated and will be removed in a' - ' future version. It is not needed, as Orbax code will automatically' - ' construct a single-replica sharding used for restoring before' - ' broadcasting.', - 1, - ) + if self.single_replica_sharding is not None: + logging.log_first_n( + logging.WARNING, + '`single_replica_sharding` is deprecated and will be removed in a' + ' future version. It is not needed, as Orbax code will automatically' + ' construct a single-replica sharding used for restoring before' + ' broadcasting.', + 1, + ) + diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib.py index 48df80cce4..a1bda60d32 100644 --- a/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib.py +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib.py @@ -15,10 +15,12 @@ """Database initialization utilities for Tiering Service.""" import contextlib +import datetime import sqlite3 from orbax.checkpoint.experimental.tiering_service import db_schema from orbax.checkpoint.experimental.tiering_service.proto import tiering_service_pb2 +import sqlalchemy from sqlalchemy import event from sqlalchemy.dialects.sqlite.aiosqlite import AsyncAdapt_aiosqlite_connection from sqlalchemy.engine import Engine @@ -27,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.future import select +import sqlalchemy.orm from sqlalchemy.orm import sessionmaker @@ -228,3 +231,250 @@ async def async_verify_db(config: tiering_service_pb2.ServerConfig) -> None: f" prefix: DB has {db_backend.prefix!r}, config expects" f" {instance.prefix!r}" ) + + +async def get_active_jobs( + session: AsyncSession, hostname: str, pid: int +) -> list[db_schema.AssetJob]: + """Returns all active PROCESSING jobs owned by this worker.""" + stmt = ( + select(db_schema.AssetJob) + .options( + sqlalchemy.orm.selectinload( + db_schema.AssetJob.target_tier_path + ).selectinload(db_schema.TierPath.storage_backend), + sqlalchemy.orm.selectinload(db_schema.AssetJob.asset) + .selectinload(db_schema.Asset.tier_paths) + .selectinload(db_schema.TierPath.storage_backend), + ) + .where( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.worker_host == hostname, + db_schema.AssetJob.worker_pid == pid, + ) + ) + result = await session.execute(stmt) + return list(result.scalars().all()) + + +async def _has_eligible_jobs( + session: AsyncSession, + backend_id: int | None, + now: datetime.datetime, +) -> bool: + """Checks if there are any eligible jobs for the backend without locking.""" + active_assets_subquery = ( + select(db_schema.AssetJob.asset_uuid) + .where( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.expiration_at >= now, + ) + .scalar_subquery() + ) + + if backend_id is None: + backend_cond = db_schema.AssetJob.target_tier_path_id.is_(None) + else: + backend_cond = db_schema.TierPath.storage_backend_id == backend_id + + stmt = ( + select(db_schema.AssetJob.id) + .join( + db_schema.TierPath, + db_schema.AssetJob.target_tier_path_id == db_schema.TierPath.id, + isouter=True, + ) + .where( + sqlalchemy.or_( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_QUEUED, + sqlalchemy.and_( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.expiration_at < now, + ), + ), + ~db_schema.AssetJob.asset_uuid.in_(active_assets_subquery), + backend_cond, + ) + .limit(1) + ) + result = await session.execute(stmt) + return result.scalar() is not None + + +async def _try_lock_backend( + session: AsyncSession, + backend_id: int, + max_active: int, + now: datetime.datetime, +) -> bool: + """Locks the backend and returns True if it has capacity for more jobs.""" + # Lock only this specific backend to avoid race conditions. + backend_stmt = ( + select(db_schema.StorageBackend) + .where(db_schema.StorageBackend.id == backend_id) + .with_for_update() + ) + await session.execute(backend_stmt) + + # Re-evaluate active capacity under the backend lock. + active_count_stmt = ( + select(sqlalchemy.func.count(db_schema.AssetJob.id)) + .join( + db_schema.TierPath, + db_schema.AssetJob.target_tier_path_id == db_schema.TierPath.id, + ) + .where( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.expiration_at >= now, + db_schema.TierPath.storage_backend_id == backend_id, + ) + ) + active_count_result = await session.execute(active_count_stmt) + active_count = active_count_result.scalar() + + return active_count < max_active + + +async def _claim_eligible_job( + session: AsyncSession, + backend_id: int | None, + lease_duration: datetime.timedelta, + hostname: str, + pid: int, + now: datetime.datetime, +) -> db_schema.AssetJob | None: + """Fetches and claims the next eligible job, if any.""" + active_assets_subquery = ( + select(db_schema.AssetJob.asset_uuid) + .where( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.expiration_at >= now, + ) + .scalar_subquery() + ) + + if backend_id is None: + backend_cond = db_schema.AssetJob.target_tier_path_id.is_(None) + else: + backend_cond = db_schema.TierPath.storage_backend_id == backend_id + + # Fetch the next eligible job, filtering for: + # 1. Jobs that are queued or whose execution lease has expired (stale jobs). + # 2. Jobs targeting assets that aren't already actively being processed by + # another job, preventing concurrency conflicts on the same asset. + # 3. Jobs belonging to the requested storage backend. + # We select the oldest job (FIFO order) and use SKIP LOCKED concurrency + # control to prevent multiple workers from matching or blocking on the same + # job. + stmt = ( + select(db_schema.AssetJob) + .options( + sqlalchemy.orm.selectinload( + db_schema.AssetJob.target_tier_path + ).selectinload(db_schema.TierPath.storage_backend), + sqlalchemy.orm.selectinload(db_schema.AssetJob.asset) + .selectinload(db_schema.Asset.tier_paths) + .selectinload(db_schema.TierPath.storage_backend), + ) + .join( + db_schema.TierPath, + db_schema.AssetJob.target_tier_path_id == db_schema.TierPath.id, + isouter=True, + ) + .where( + sqlalchemy.or_( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_QUEUED, + sqlalchemy.and_( + db_schema.AssetJob.status + == db_schema.JobStatus.JOB_STATUS_PROCESSING, + db_schema.AssetJob.expiration_at < now, + ), + ), + ~db_schema.AssetJob.asset_uuid.in_(active_assets_subquery), + backend_cond, + ) + .order_by(db_schema.AssetJob.created_at.asc()) + .limit(1) + .with_for_update(skip_locked=True) + ) + + result = await session.execute(stmt) + job = result.scalars().first() + + if job: + # Atomically claim the job + job.status = db_schema.JobStatus.JOB_STATUS_PROCESSING + job.expiration_at = now + lease_duration + job.worker_host = hostname + job.worker_pid = pid + job.last_updated_at = now + session.add(job) + if ( + job.request_type == db_schema.RequestType.REQUEST_TYPE_COPY + and job.target_tier_path + ): + job.target_tier_path.state = db_schema.TierPathState.IN_PROGRESS + session.add(job.target_tier_path) + + return job + + +async def acquire_next_job( + session_maker: sessionmaker, + backend_id: int | None, + lease_duration: datetime.timedelta, + hostname: str, + pid: int, + max_active: int, +) -> db_schema.AssetJob | None: + """Queries the database for the next eligible job on the given backend and claims it. + + Args: + session_maker: A session maker or session factory. MUST be configured with + `expire_on_commit=False` to prevent returned Job relationships from being + expired upon transaction commit. + backend_id: The ID of the storage backend. + lease_duration: Lease duration for the claimed job. + hostname: Hostname of the claiming worker. + pid: PID of the claiming worker. + max_active: Maximum active jobs allowed on this backend. + + Returns: + The claimed AssetJob instance, or None if no eligible jobs are available or + if capacity is full. + """ + now = datetime.datetime.now(datetime.timezone.utc) + + async with session_maker() as session: + await session.begin() + try: + # Check if there are any jobs at all before acquiring locks + if not await _has_eligible_jobs(session, backend_id, now): + await session.rollback() + return None + + if backend_id is not None: + if not await _try_lock_backend(session, backend_id, max_active, now): + # no jobs available on this backend, release lock and return None + await session.rollback() + return None + + job = await _claim_eligible_job( + session, backend_id, lease_duration, hostname, pid, now + ) + if job is None: + await session.rollback() + return None + + await session.commit() + return job + except Exception: + await session.rollback() + raise diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib_test.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib_test.py index 5102411c5a..dec845dda7 100644 --- a/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib_test.py +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/db_lib_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import textwrap import unittest @@ -19,8 +20,12 @@ import aiosqlite # pylint: disable=unused-import import greenlet # pylint: disable=unused-import from orbax.checkpoint.experimental.tiering_service import db_lib +from orbax.checkpoint.experimental.tiering_service import db_schema from orbax.checkpoint.experimental.tiering_service import server_config from sqlalchemy import exc as sqlalchemy_exc +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy.orm import sessionmaker import yaml @@ -170,6 +175,200 @@ async def test_sqlite_url_translation(self): ) await engine.dispose() + async def test_acquire_and_get_active_jobs(self): + tmp_file = self.create_tempfile() + db_url = f"sqlite+aiosqlite:///{tmp_file.full_path}" + yaml_content = textwrap.dedent(f"""\ + db_connection_str: {db_url} + storage_backends: + - level: 1 + backend_type: BACKEND_TYPE_GCS + prefix: gs://my-bucket + region: us-central1 + """) + config_dict = yaml.safe_load(yaml_content) + config = server_config.parse_config(config_dict) + await db_lib.async_initialize_db(config) + + engine = db_lib.get_async_engine(config) + + async_session = sessionmaker( + engine, expire_on_commit=False, class_=AsyncSession + ) + # Setup + async with async_session() as session: + # Create dependencies for valid AssetJob payload + asset = db_schema.Asset( + asset_uuid="dummy-uuid", + path="/experiment/dummy-path", + user="testuser", + ) + backend = db_schema.StorageBackend( + level=1, + zone="us-central1-a", + backend_type=db_schema.BackendType.BACKEND_TYPE_GCS, + prefix="gs://my-bucket", + ) + session.add_all([asset, backend]) + await session.commit() + + tier_path = db_schema.TierPath( + asset_uuid="dummy-uuid", + storage_backend_id=backend.id, + path="/path1", + ) + session.add(tier_path) + await session.commit() + + # Create valid job + job = db_schema.AssetJob( + asset_uuid="dummy-uuid", + request_type=db_schema.RequestType.REQUEST_TYPE_COPY, + status=db_schema.JobStatus.JOB_STATUS_QUEUED, + request_id="req-123", + target_tier_path_id=tier_path.id, + ) + session.add(job) + await session.commit() + backend_id = backend.id + + # Call + acquired_job = await db_lib.acquire_next_job( + session_maker=async_session, + backend_id=backend_id, + lease_duration=datetime.timedelta(minutes=5), + hostname="test-host", + pid=1234, + max_active=10, + ) + self.assertIsNotNone(acquired_job) + self.assertEqual(acquired_job.request_id, "req-123") + self.assertEqual( + acquired_job.status, db_schema.JobStatus.JOB_STATUS_PROCESSING + ) + self.assertEqual(acquired_job.worker_host, "test-host") + + # Verification + async with async_session() as session: + # Get active jobs + active_jobs = await db_lib.get_active_jobs( + session=session, hostname="test-host", pid=1234 + ) + self.assertLen(active_jobs, 1) + self.assertEqual(active_jobs[0].request_id, "req-123") + await engine.dispose() + + async def test_acquire_next_job_capacity_full_releases_lock(self): + tmp_file = self.create_tempfile() + db_url = f"sqlite+aiosqlite:///{tmp_file.full_path}" + yaml_content = textwrap.dedent(f"""\ + db_connection_str: {db_url} + storage_backends: + - level: 1 + backend_type: BACKEND_TYPE_GCS + prefix: gs://my-bucket + region: us-central1 + """) + config_dict = yaml.safe_load(yaml_content) + config = server_config.parse_config(config_dict) + await db_lib.async_initialize_db(config) + + engine = db_lib.get_async_engine(config) + async_session = sessionmaker( + engine, expire_on_commit=False, class_=AsyncSession + ) + + async with async_session() as session: + # Get backend ID + result = await session.execute(select(db_schema.StorageBackend)) + backend = result.scalars().first() + backend_id = backend.id + + # Create an active job for this backend to consume the capacity + asset = db_schema.Asset( + asset_uuid="uuid-active", + path="/path/active", + user="test-user", + state=db_schema.AssetState.ASSET_STATE_STORED, + ) + session.add(asset) + await session.commit() + + tier_path = db_schema.TierPath( + asset_uuid="uuid-active", + storage_backend_id=backend_id, + path="/mnt/lustre/active", + ) + session.add(tier_path) + await session.commit() + + # Expiration is in the future + future_now = datetime.datetime.now( + datetime.timezone.utc + ) + datetime.timedelta(minutes=10) + active_job = db_schema.AssetJob( + asset_uuid="uuid-active", + request_type=db_schema.RequestType.REQUEST_TYPE_COPY, + status=db_schema.JobStatus.JOB_STATUS_PROCESSING, + expiration_at=future_now, + target_tier_path_id=tier_path.id, + ) + session.add(active_job) + await session.commit() + + # Create another queued job that we want to try to acquire + asset2 = db_schema.Asset( + asset_uuid="uuid-queued", + path="/path/queued", + user="test-user", + state=db_schema.AssetState.ASSET_STATE_STORED, + ) + session.add(asset2) + await session.commit() + + tier_path2 = db_schema.TierPath( + asset_uuid="uuid-queued", + storage_backend_id=backend_id, + path="/mnt/lustre/queued", + ) + session.add(tier_path2) + await session.commit() + + queued_job = db_schema.AssetJob( + asset_uuid="uuid-queued", + request_type=db_schema.RequestType.REQUEST_TYPE_COPY, + status=db_schema.JobStatus.JOB_STATUS_QUEUED, + target_tier_path_id=tier_path2.id, + ) + session.add(queued_job) + await session.commit() + + # Call acquire_next_job (simulating job_worker.py) + # With max_active = 1, this should return None because 1 job is active + acquired_job = await db_lib.acquire_next_job( + session_maker=async_session, + backend_id=backend_id, + lease_duration=datetime.timedelta(minutes=5), + hostname="test-host", + pid=1234, + max_active=1, # Max active is 1 + ) + self.assertIsNone(acquired_job) + + # Now verify that the lock is released by trying to modify the backend in + # a new session. If the lock was not released, this will block or fail. + async with async_session() as session2: + async with session2.begin(): + result = await session2.execute( + select(db_schema.StorageBackend) + .where(db_schema.StorageBackend.id == backend_id) + .with_for_update() + ) + backend_row = result.scalar() + backend_row.prefix = "gs://new-bucket-name" # Modifying prefix + session2.add(backend_row) + await engine.dispose() + if __name__ == "__main__": absltest.main() diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py new file mode 100644 index 0000000000..6971b69ea1 --- /dev/null +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client.py @@ -0,0 +1,455 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GCP storage clients for Checkpoint Tiering Service (CTS).""" + +import abc +import asyncio +import dataclasses +import datetime +import enum +import os +from typing import Any +import google.auth +from google.auth import exceptions as auth_exceptions +from google.auth import impersonated_credentials +from google.auth import transport +import httpx + + +class OperationStatus(enum.Enum): + """The status of a job or operation.""" + + IN_PROGRESS = "IN_PROGRESS" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + + +@dataclasses.dataclass +class Result: + status: OperationStatus + detail_info: dict[str, Any] + + +@dataclasses.dataclass +class TransferContext: + job_request_id: str + source_path: str + destination_path: str + transfer_status: dict[str, Any] + + +class HttpxRequest(transport.Request): + """A google-auth compatible transport request using HTTPX (sync).""" + + def __init__(self, client: httpx.Client): + self._client = client + + def __call__( + self, + url: str, + method: str = "GET", + body: bytes | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + **kwargs, + ) -> transport.Response: + try: + response = self._client.request( + method=method, + url=url, + headers=headers, + content=body, + timeout=timeout, + **kwargs, + ) + + class HttpxResponse(transport.Response): + """A google-auth compatible transport response using HTTPX.""" + + def __init__(self, resp): + self._resp = resp + + @property + def status(self) -> int: + return self._resp.status_code + + @property + def headers(self) -> dict[str, str]: + return dict(self._resp.headers) + + @property + def data(self) -> bytes: + return self._resp.content + + return HttpxResponse(response) + except httpx.TimeoutException as e: + raise auth_exceptions.TransportError(f"Timeout: {e}") + except httpx.RequestError as e: + raise auth_exceptions.TransportError(f"Request error: {e}") + + +class GCPStorageClient(abc.ABC): + """Client interface to interact with GCP storage backend (e.g. + + Lustre, GCS). + """ + + def __init__( + self, + project: str | None = None, + location: str | None = None, + instance: str | None = None, + service_account: str | None = None, + ): + self.project = project + self.location = location + self.instance = instance + self.service_account = service_account + self._credentials = None + self._async_client = None + + @property + def async_client(self) -> httpx.AsyncClient: + if self._async_client is None: + self._async_client = httpx.AsyncClient() + return self._async_client + + async def close(self): + if self._async_client is not None: + await self._async_client.aclose() + self._async_client = None + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _get_token_and_project(self) -> tuple[str, str]: + """Gets authentication credentials and projects.""" + if not self._credentials: + base_credentials, detected_project = await asyncio.to_thread( + google.auth.default, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + if not self.project: + self.project = detected_project + + if self.service_account: + self._credentials = impersonated_credentials.Credentials( + source_credentials=base_credentials, + target_principal=self.service_account, + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + else: + self._credentials = base_credentials + + if not self._credentials.valid: + with httpx.Client() as client: + await asyncio.to_thread(self._credentials.refresh, HttpxRequest(client)) + + if not self.project: + raise ValueError("GCP Project ID must be specified or auto-detected.") + + return self._credentials.token, self.project + + @abc.abstractmethod + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers copy and returns operation name.""" + pass + + @abc.abstractmethod + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls operation status and returns a Result object.""" + pass + + +def _parse_gcs_path(gcs_path: str) -> tuple[str, str]: + """Parses a GCS path like gs://bucket/prefix/file into (bucket, prefix).""" + path_no_scheme = gcs_path.replace("gs://", "") + parts = path_no_scheme.split("/", 1) + bucket = parts[0] + prefix = parts[1] if len(parts) > 1 else "" + return bucket, prefix + + +class GcsToGcsClient(GCPStorageClient): + """Client implementation for GCS-to-GCS operations using Storage Transfer Service.""" + + def __init__( + self, + project: str | None = None, + service_account: str | None = None, + ): + super().__init__(project=project, service_account=service_account) + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers GCS-to-GCS transfer using GCP Storage Transfer Service (STS).""" + token, project = await self._get_token_and_project() + url = "https://storagetransfer.googleapis.com/v1/transferJobs" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + src_bucket, src_prefix = _parse_gcs_path(source_path) + dest_bucket, dest_prefix = _parse_gcs_path(destination_path) + + now = datetime.datetime.now(datetime.timezone.utc) + payload = { + "projectId": project, + "transferSpec": { + "gcsDataSource": { + "bucketName": src_bucket, + "path": src_prefix, + }, + "gcsDataSink": { + "bucketName": dest_bucket, + "path": dest_prefix, + }, + "transferOptions": { + "overwriteObjectsAlreadyExistingInSink": True, + }, + }, + "schedule": { + "scheduleStartDate": { + "year": now.year, + "month": now.month, + "day": now.day, + }, + }, + "status": "DISABLED", + } + + response = await self.async_client.post(url, json=payload, headers=headers) + + if response.status_code != 200: + raise RuntimeError( + f"Failed to create Storage Transfer Job: {response.status_code} -" + f" {response.text}" + ) + + job_name = response.json()["name"] + + # Run the job immediately. This returns the operation name directly. + run_url = f"https://storagetransfer.googleapis.com/v1/{job_name}:run" + run_payload = {"projectId": project} + run_response = await self.async_client.post( + run_url, json=run_payload, headers=headers + ) + + if run_response.status_code != 200: + raise RuntimeError( + f"Failed to run Storage Transfer Job {job_name}:" + f" {run_response.status_code} - {run_response.text}" + ) + + operation_name = run_response.json()["name"] + return operation_name + + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls Storage Transfer Service operation status.""" + token, _ = await self._get_token_and_project() + url = f"https://storagetransfer.googleapis.com/v1/{operation_name}" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + response = await self.async_client.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to poll Storage Transfer operation: {response.status_code} -" + f" {response.text}" + ) + + data = response.json() + done = data.get("done", False) + + if not done: + metadata = data.get("metadata", {}) + counters = metadata.get("counters", {}) + bytes_transferred = int(counters.get("bytesTransferredToSink", 0)) + bytes_found = int(counters.get("bytesFoundToTransfer", 0)) + return Result( + status=OperationStatus.IN_PROGRESS, + detail_info={ + "bytes_copied": bytes_transferred, + "total_bytes": bytes_found, + }, + ) + + if "error" in data: + return Result( + status=OperationStatus.FAILED, + detail_info={"error": data["error"]}, + ) + + metadata = data.get("metadata", {}) + op_status = metadata.get("status") + + if op_status == "SUCCESS": + return Result( + status=OperationStatus.SUCCESS, + detail_info={}, + ) + else: + error_msg = f"STS Operation ended with status: {op_status}" + return Result( + status=OperationStatus.FAILED, + detail_info={"error": error_msg}, + ) + + +class GcpLustreBaseClient(GCPStorageClient): + """Base client interface to interact with GCP Managed Lustre API via REST.""" + + def __init__( + self, + project: str | None = None, + location: str | None = None, + instance: str | None = None, + service_account: str | None = None, + ): + location = location or os.environ.get("CTS_LUSTRE_LOCATION") + instance = instance or os.environ.get("CTS_LUSTRE_INSTANCE") + + if not location or not instance: + raise ValueError("Lustre location and instance must be specified.") + + super().__init__( + project=project, + location=location, + instance=instance, + service_account=service_account, + ) + + async def poll_operation( + self, + operation_name: str, + context: TransferContext | None = None, + ) -> Result: + """Polls operation status and returns a Result object.""" + del context # Unused for Lustre + token, _ = await self._get_token_and_project() + url = f"https://lustre.googleapis.com/v1/{operation_name}" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + response = await self.async_client.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to poll operation: {response.status_code} - {response.text}" + ) + data = response.json() + done = data.get("done", False) + if done: + if "error" in data: + return Result( + status=OperationStatus.FAILED, + detail_info={"error": data["error"]}, + ) + else: + return Result( + status=OperationStatus.SUCCESS, + detail_info=data.get("response", {}), + ) + else: + return Result( + status=OperationStatus.IN_PROGRESS, + detail_info=data.get("metadata", {}), + ) + + +class GcsToLustreClient(GcpLustreBaseClient): + """Client implementation to trigger GCS-to-Lustre imports.""" + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers import from GCS to Lustre and returns the Operation name.""" + token, project = await self._get_token_and_project() + url = ( + f"https://lustre.googleapis.com/v1/projects/{project}" + f"/locations/{self.location}/instances/{self.instance}:importData" + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + payload = { + "gcsPath": {"uri": source_path}, + "lustrePath": {"path": destination_path}, + "requestId": request_id, + } + response = await self.async_client.post(url, json=payload, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to trigger import: {response.status_code} - {response.text}" + ) + return response.json()["name"] + + +class LustreToGcsClient(GcpLustreBaseClient): + """Client implementation to trigger Lustre-to-GCS exports.""" + + async def trigger_copy( + self, + request_id: str, + source_path: str, + destination_path: str, + ) -> str: + """Triggers export from Lustre to GCS and returns the Operation name.""" + token, project = await self._get_token_and_project() + url = ( + f"https://lustre.googleapis.com/v1/projects/{project}" + f"/locations/{self.location}/instances/{self.instance}:exportData" + ) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + payload = { + "lustrePath": {"path": source_path}, + "gcsPath": {"uri": destination_path}, + "requestId": request_id, + } + response = await self.async_client.post(url, json=payload, headers=headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to trigger export: {response.status_code} - {response.text}" + ) + return response.json()["name"] diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py new file mode 100644 index 0000000000..99caea1778 --- /dev/null +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/gcp_storage_client_test.py @@ -0,0 +1,281 @@ +# Copyright 2026 The Orbax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for GCP Storage Clients.""" + +import os +import unittest +from unittest import mock +import httpx +from orbax.checkpoint.experimental.tiering_service import gcp_storage_client + + +class GCPStorageClientTest(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + super().setUp() + self.mock_creds = mock.MagicMock() + self.mock_creds.valid = True + self.mock_creds.token = "dummy_auth_token" + self.mock_default_auth = mock.patch( + "google.auth.default", return_value=(self.mock_creds, "dummy-project") + ) + self.mock_default_auth.start() + + # Mock httpx.AsyncClient + self.mock_client = mock.AsyncMock(spec=httpx.AsyncClient) + self.mock_client_patcher = mock.patch( + "httpx.AsyncClient", return_value=self.mock_client + ) + self.mock_client_patcher.start() + + def tearDown(self): + self.mock_default_auth.stop() + self.mock_client_patcher.stop() + super().tearDown() + + async def test_gcs_to_gcs_trigger_copy_success(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + # 1. Mock the first POST call to create transfer job + mock_post_resp_1 = mock.MagicMock(spec=httpx.Response) + mock_post_resp_1.status_code = 200 + mock_post_resp_1.json.return_value = {"name": "transferJobs/job-123"} + + # 2. Mock the second POST call to run transfer job + mock_post_resp_2 = mock.MagicMock(spec=httpx.Response) + mock_post_resp_2.status_code = 200 + mock_post_resp_2.json.return_value = {"name": "transferOperations/op-456"} + + self.mock_client.post.side_effect = [mock_post_resp_1, mock_post_resp_2] + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path/to/src", + destination_path="gs://dest-bucket/path/to/dest", + ) + + self.assertEqual(op_name, "transferOperations/op-456") + self.assertEqual(self.mock_client.post.call_count, 2) + + async def test_gcs_to_gcs_trigger_copy_sts_fail(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 400 + mock_post_resp.text = "Invalid arguments" + self.mock_client.post.return_value = mock_post_resp + + with self.assertRaises(RuntimeError) as ctx: + await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path", + destination_path="gs://dest-bucket/path", + ) + self.assertIn("Failed to create Storage Transfer Job", str(ctx.exception)) + + async def test_gcs_to_gcs_poll_operation_in_progress(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": False, + "metadata": { + "counters": { + "bytesTransferredToSink": "500", + "bytesFoundToTransfer": "1000", + } + }, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual( + result.status, gcp_storage_client.OperationStatus.IN_PROGRESS + ) + self.assertEqual(result.detail_info["bytes_copied"], 500) + self.assertEqual(result.detail_info["total_bytes"], 1000) + + async def test_gcs_to_gcs_poll_operation_success(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "metadata": {"status": "SUCCESS"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.SUCCESS) + + async def test_gcs_to_gcs_poll_operation_failed(self): + client = gcp_storage_client.GcsToGcsClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "metadata": {"status": "FAILED"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("transferOperations/op-456") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.FAILED) + self.assertIn("error", result.detail_info) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_gcs_to_lustre_trigger_copy(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 200 + mock_post_resp.json.return_value = {"name": "operations/import-123"} + self.mock_client.post.return_value = mock_post_resp + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="gs://src-bucket/path", + destination_path="/lustre/path", + ) + self.assertEqual(op_name, "operations/import-123") + self.mock_client.post.assert_called_once() + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_to_gcs_trigger_copy(self): + client = gcp_storage_client.LustreToGcsClient(project="test-project") + + mock_post_resp = mock.MagicMock(spec=httpx.Response) + mock_post_resp.status_code = 200 + mock_post_resp.json.return_value = {"name": "operations/export-123"} + self.mock_client.post.return_value = mock_post_resp + + op_name = await client.trigger_copy( + request_id="req-1", + source_path="/lustre/path", + destination_path="gs://dest-bucket/path", + ) + self.assertEqual(op_name, "operations/export-123") + self.mock_client.post.assert_called_once() + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_done_success(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "response": {"some_metadata": "val"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.SUCCESS) + self.assertEqual(result.detail_info, {"some_metadata": "val"}) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_done_fail(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": True, + "error": {"message": "import failed"}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual(result.status, gcp_storage_client.OperationStatus.FAILED) + self.assertEqual(result.detail_info["error"], {"message": "import failed"}) + + @mock.patch.dict( + os.environ, + { + "CTS_LUSTRE_LOCATION": "us-central1-a", + "CTS_LUSTRE_INSTANCE": "lustre-1", + }, + ) + async def test_lustre_poll_operation_in_progress(self): + client = gcp_storage_client.GcsToLustreClient(project="test-project") + + mock_get_resp = mock.MagicMock(spec=httpx.Response) + mock_get_resp.status_code = 200 + mock_get_resp.json.return_value = { + "done": False, + "metadata": {"percent_complete": 42}, + } + self.mock_client.get.return_value = mock_get_resp + + result = await client.poll_operation("operations/import-123") + self.assertEqual( + result.status, gcp_storage_client.OperationStatus.IN_PROGRESS + ) + self.assertEqual(result.detail_info, {"percent_complete": 42}) + + @mock.patch("google.auth.impersonated_credentials.Credentials") + async def test_service_account_token_impersonation( + self, mock_impersonated_creds_class + ): + mock_imp_creds = mock.MagicMock() + mock_imp_creds.valid = True + mock_imp_creds.token = "impersonated_token" + mock_impersonated_creds_class.return_value = mock_imp_creds + + client = gcp_storage_client.GcsToGcsClient( + project="test-project", + service_account="sa@test.iam.gserviceaccount.com", + ) + + token, project = await client._get_token_and_project() + self.assertEqual(token, "impersonated_token") + self.assertEqual(project, "test-project") + + # Verify that impersonated credentials were constructed correctly + mock_impersonated_creds_class.assert_called_once_with( + source_credentials=self.mock_creds, + target_principal="sa@test.iam.gserviceaccount.com", + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend.py index 121eb63392..c3fd2d5624 100644 --- a/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend.py +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend.py @@ -30,20 +30,21 @@ async def find_backends_by_level( session: AsyncSession, - level: int = 0, + level: int | None = 0, ) -> Sequence[db_schema.StorageBackend]: """Queries backends with the given level from the database. Args: session: The database session. - level: The tiering level to filter by (default is 0). + level: The tiering level to filter by. If None, returns all backends. Returns: A sequence of StorageBackend objects matching the level. """ - result = await session.execute( - select(db_schema.StorageBackend).filter_by(level=level) - ) + stmt = select(db_schema.StorageBackend) + if level is not None: + stmt = stmt.filter_by(level=level) + result = await session.execute(stmt) return result.scalars().all() diff --git a/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend_test.py b/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend_test.py index f3a8958b42..8237327d45 100644 --- a/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend_test.py +++ b/checkpoint/orbax/checkpoint/experimental/tiering_service/storage_backend_test.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - - import unittest from absl.testing import absltest from orbax.checkpoint.experimental.tiering_service import db_schema @@ -76,6 +74,11 @@ async def test_find_backends_by_level(self): self.assertLen(backends_l1, 1) self.assertEqual(backends_l1[0].prefix, "gs://bucket") + backends_all = await storage_backend.find_backends_by_level( + session, level=None + ) + self.assertLen(backends_all, 3) + async def test_locate_closest_backend(self): async with self.session_maker() as session: await self._setup_backends(session) diff --git a/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py b/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py index e882fc1732..c53e78f4de 100644 --- a/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py +++ b/checkpoint/orbax/checkpoint/experimental/v1/_src/layout/orbax_layout.py @@ -56,7 +56,7 @@ class CheckpointVersion(enum.Enum): ORBAX_CHECKPOINT_INDICATOR_FILE = "orbax.checkpoint" CHECKPOINT_METADATA = "_CHECKPOINT_METADATA" -_OCDBT_MANIFEST_FILE = "ocdbt.manifest" +_OCDBT_MANIFEST_FILE = "manifest.ocdbt" _ZARRAY_FILE = ".zarray" diff --git a/checkpoint/pyproject.toml b/checkpoint/pyproject.toml index f4490abe08..49c3e116d3 100644 --- a/checkpoint/pyproject.toml +++ b/checkpoint/pyproject.toml @@ -81,6 +81,7 @@ tiering_service = [ 'fire', 'greenlet', 'grpcio-tools>=1.80.0', + 'httpx', 'pysqlite3', 'pytimeparse', 'sqlalchemy>=1.4.0', diff --git a/docs/guides/checkpoint/v1/async_checkpointing.ipynb b/docs/guides/checkpoint/v1/async_checkpointing.ipynb index da315d7c10..ce11493998 100644 --- a/docs/guides/checkpoint/v1/async_checkpointing.ipynb +++ b/docs/guides/checkpoint/v1/async_checkpointing.ipynb @@ -139,7 +139,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/sync_checkpoint" + "print(sorted([p.name for p in path.iterdir()]))" ], "outputs": [], "execution_count": 9 @@ -175,7 +175,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/async_checkpoint" + "print(sorted([p.name for p in path.iterdir()]))" ], "outputs": [], "execution_count": 11 diff --git a/docs/guides/checkpoint/v1/checkpoint_format.ipynb b/docs/guides/checkpoint/v1/checkpoint_format.ipynb index e5851d0ed4..949b760bcb 100644 --- a/docs/guides/checkpoint/v1/checkpoint_format.ipynb +++ b/docs/guides/checkpoint/v1/checkpoint_format.ipynb @@ -321,7 +321,7 @@ }, "cell_type": "code", "source": [ - "!ls {directory / 'ckpt-0'}" + "print(sorted([p.name for p in (directory / 'ckpt-0').iterdir()]))" ], "outputs": [], "execution_count": null diff --git a/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb b/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb index 165de5b4c2..f0553319cb 100644 --- a/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb +++ b/docs/guides/checkpoint/v1/checkpointing_pytrees.ipynb @@ -1,1131 +1,1223 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "GYCcRRZas1PS" - }, - "source": [ - "# Working with PyTree Checkpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iJwUDQVA7GIV" - }, - "source": [ - "A [PyTree](https://jax.readthedocs.io/en/latest/pytrees.html) is the most common way of representing a training state in JAX. While Orbax is designed to be as generic as possible, and provides customization options for all manner of checkpointable objects, PyTrees naturally have pride of place. Furthermore, the standard object used to represent large, sharded arrays is the {py:class}`jax.Array `. This, too, has extensive first-class support." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Vfj9mDHsb7QF" - }, - "source": [ - "## Exclusive APIs to checkpoint PyTrees" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "e3JtbxFhVq_6" - }, - "source": [ - "The following APIs can be used to checkpoint PyTrees exclusively.\n", - "\n", - "To save:\n", - "\n", - "* `ocp.save(...)`\n", - "* `ocp.save_async(...)`\n", - "* `training.Checkpointer.save(...)`\n", - "* `training.Checkpointer.save_async(...)`\n", - "\n", - "To load:\n", - "* `ocp.load(...)`\n", - "* `ocp.load_async(...)`\n", - "* `training.Checkpointer.load(...)`\n", - "* `training.Checkpointer.load_async(...)`\n", - "\n", - "Of course, the `save_checkpointables(...)` and `load_checkpointables(...)`\n", - "flavor APIs can be used to save a PyTree too." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ej99peApVq_7" - }, - "source": [ - "Let's setup a PyTree of jax.Array to play with these APIs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6NknwqBeVq_7" - }, - "outputs": [], - "source": [ - "from etils import epath\n", - "import jax\n", - "import numpy as np\n", - "import orbax.checkpoint.experimental.v1 as ocp" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "b0j19QxuVq_7" - }, - "outputs": [], - "source": [ - "sharding = jax.sharding.NamedSharding(\n", - " jax.sharding.Mesh(jax.devices(), ('model',)),\n", - " jax.sharding.PartitionSpec(\n", - " 'model',\n", - " ),\n", - ")\n", - "create_sharded_array = lambda x: jax.device_put(x, sharding)\n", - "pytree = {\n", - " 'a': np.arange(16, dtype=np.int32),\n", - " 'b': np.ones(16, dtype=np.int32),\n", - "}\n", - "pytree = jax.tree_util.tree_map(create_sharded_array, pytree)\n", - "pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Kf_OCzGbVq_8" - }, - "source": [ - "## Basic Checkpointing" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sU2YIn10Vq_8" - }, - "source": [ - "Let's use `ocp.save_*`/`ocp.load_*` to work with the pytree created earlier." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Szq9yBPhVq_8" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/basic/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "# Simple save using default options:\n", - "ocp.save(path, pytree)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Kr6nrYg3QR73" - }, - "source": [ - "We can easily restore using the following snippet.\n", - "\n", - "**Warning: do not use for production-sensitive cases.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "CtEMu9BBRHYW" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path)\n", - "loaded" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "F-CPPqeaRRks" - }, - "source": [ - "It is not recommended to load this way for production-sensitive cases because the user cannot make any guarantees about what they are loading. If the shapes of some arrays have changed in the model since the checkpoint was saved, errors can be seen when attempting to create the model. If the device topology has changed, we will see errors when attempting to place arrays on devices.\n", - "\n", - "It is therefore recommended that users always specify an **abstract pytree** when loading." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R1Sbx3DYQMpS" - }, - "source": [ - "### Understanding Abstract Trees and Leaves" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4f8Ze-q3R5XE" - }, - "source": [ - "An **abstract PyTree** is just a normal PyTree, but with abstract leaves. An **abstract leaf** is a cheap representation of a leaf type (such as an array) that contains only metadata, and does not represent the real values. (Contrast with a *concrete* PyTree, which contains real data in the form of large arrays, and other types.)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PVyCvC7VaisK" - }, - "source": [ - "Let's create an abstract PyTree matching the structure of the PyTree we originally saved." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "u5JJaHrYVq_7" - }, - "outputs": [], - "source": [ - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", - " 'b': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", - "}\n", - "abstract_state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "0GCij-iuVq_8" - }, - "outputs": [], - "source": [ - "# Load using abstract_state.\n", - "loaded = ocp.load(path, abstract_state)\n", - "loaded" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TjOlZAkTVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tQ6L_wtnVq_8" - }, - "source": [ - "The `metadata` method returns a `CheckpointMetadata` object with a number of properties, but the core `metadata` property is just an abstract PyTree. This can also be used for loading as shown below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_sJOi2I3Vq_8" - }, - "outputs": [], - "source": [ - "metadata = ocp.metadata(path).metadata\n", - "metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "1SQMyU44Vq_8" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path, metadata)\n", - "loaded" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jGTdk0tVVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "U7URSieWVq_8" - }, - "source": [ - "Note that it is also valid to provide a \"concrete\" PyTree for loading rather than an \"abstract\" target, since by definition, the concrete leaves contain all the same properties provided by the abstract leaves.\n", - "\n", - "However, this requires that you fully initialize the target train state\n", - "before loading from the checkpoint, which is inefficient or impractical for real use cases. It is better practice to only initialize metadata (either by manually creating `jax.ShapeDtypeStruct`s or using `jax.eval_shape`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3JfwbnA_Vq_8" - }, - "outputs": [], - "source": [ - "ocp.load(path, pytree)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "wjpnDhAqb6t0" - }, - "source": [ - "### Standard Leaf Types" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "p6yM2hBGdNXZ" - }, - "source": [ - "The following standard leaf types are supported by Orbax by default. Each concrete leaf type has a corresponding abstract leaf type. Most abstract types are implemented as `Protocol`'s, so that any object implementing the required properties can be accepted as a valid abstract type." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G0oCkcDZb8g_" - }, - "source": [ - "| `Leaf` Type | `AbstractLeaf` Type | Properties |\n", - ":------- | :-------- | :-------- |\n", - "|`jax.Array`|`AbstractShardedArray` (`jax.ShapeDtypeStruct`) |`shape`, `dtype`, `sharding`|\n", - "|`np.ndarray`|`AbstractArray` (`np.ndarray`) |`shape`, `dtype`|\n", - "|`int`|`int`| |\n", - "|`float`|`float`| |\n", - "|`bytes`|`bytes`| |\n", - "|`str`|`str`| |" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "01zrVpfcdq7m" - }, - "source": [ - "`None` is always a valid abstract leaf; it serves as an indication that the leaf should be restored using metadata stored in the checkpoint.\n", - "\n", - "`Type[AbstractLeaf]` is also always a valid abstract leaf; it again serves as an indication that the leaf should be restored using the metadata, but with the additional constraint to load as the indicated type. For example, instead of specifying `jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=...)`, it is sufficient to pass `jax.ShapeDtypeStruct`. Similarly, instead of passing `0` to restore as an `int`, the type itself may be passed." - ] - }, - { - "metadata": { - "id": "nRn_2IgV09xT" - }, - "cell_type": "markdown", - "source": [ - "To summarize, here are the ways you can load a PyTree using abstract leaves, with the way we most recommend at the top, and the way we least recommend at the bottom.\n", - "\n", - "**1. Fully-specified abstract values**\n", - "\n", - "This provides the most loading validations and requires the least amount of\n", - "unnecessary metadata reads.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=jax.sharding.NamedSharding(...))\n", - "}\n", - "```\n", - "\n", - "**2. Only types specified**\n", - "\n", - "This guarantees that each leaf will be loaded with the indicated type, but metadata\n", - "will be used to restore specific properties for each leaf.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct,\n", - " 'b': int,\n", - " 'c': np.ndarray,\n", - "}\n", - "```\n", - "\n", - "**3. `None` specified (per-leaf)**\n", - "\n", - "This is essentially the same as (2), but metadata will also be used to decide\n", - "which type each leaf should be loaded as.\n", - "\n", - "```\n", - "abstract_state = {\n", - " 'a': None,\n", - " 'b': None,\n", - "}\n", - "```\n", - "\n", - "**4. `None` specified**\n", - "\n", - "This loads the PyTree structure without any checks, and can lead to errors later\n", - "in your code if the checkpoint does not have the structure you expect.\n", - "\n", - "```\n", - "abstract_state = None\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UR34KTImVq_8" - }, - "source": [ - "### Customizing Loaded Properties for Arrays" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TErvR-zDVq_8" - }, - "source": [ - "#### Array dtype" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZDSJhefZVq_8" - }, - "outputs": [], - "source": [ - "def set_loading_dtype(x: jax.ShapeDtypeStruct) -> jax.ShapeDtypeStruct:\n", - " return x.update(dtype=np.int16)\n", - "\n", - "\n", - "cast_dtype_abstract_state = jax.tree_util.tree_map(\n", - " set_loading_dtype, abstract_state\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "scZbqSQ4Vq_8" - }, - "outputs": [], - "source": [ - "ocp.load(path, cast_dtype_abstract_state)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rhkTT9wMVq_8" - }, - "source": [ - "#### Change sharding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NLjuYubRVq_8" - }, - "source": [ - "**NOTE: This can often be a particularly sharp edge.**\n", - "\n", - "Sharding commonly needs to be changed when loading a checkpoint saved on one topology to a different topology.\n", - "\n", - "**If changing topologies, you MUST specify sharding when loading.**\n", - "\n", - "Unless you are loading on the exact same topology, Orbax does not make any decisions about shardings on your behalf. If you have the exact same topology,\n", - "however, it is possible to avoid specifying the sharding when loading. This is demonstrated below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "atPqix9IVq_8" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Pap_vpN9Vq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "T_T3QjRLVq_8" - }, - "source": [ - "In the example below, we alter the sharding while loading." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "J0tvS901Vq_8" - }, - "outputs": [], - "source": [ - "sharding = jax.sharding.NamedSharding(\n", - " jax.sharding.Mesh(jax.devices(), ('x',)),\n", - " jax.sharding.PartitionSpec(),\n", - ")\n", - "\n", - "\n", - "def set_sharding(x: jax.ShapeDtypeStruct) -> jax.ShapeDtypeStruct:\n", - " return x.update(sharding=sharding)\n", - "\n", - "\n", - "change_sharding_abstract_state = jax.tree_util.tree_map(\n", - " set_sharding, abstract_state\n", - ")\n", - "loaded = ocp.load(path, change_sharding_abstract_state)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "M84bx2smVq_8" - }, - "outputs": [], - "source": [ - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zA_Vdck2Vq_8" - }, - "source": [ - "We can use pytree metadata instead of the abstract pytree." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cAN_1Xj_Vq_8" - }, - "outputs": [], - "source": [ - "metadata = ocp.metadata(path).metadata\n", - "change_sharding_metadata = jax.tree_util.tree_map(\n", - " lambda x: jax.ShapeDtypeStruct(shape=x.shape, dtype=x.dtype, sharding=sharding), metadata\n", - ")\n", - "loaded = ocp.load(path, change_sharding_metadata)\n", - "(loaded['a'].sharding, loaded['b'].sharding)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sIj3AYAidwvh" - }, - "source": [ - "#### Change leaf type" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "01BAYxDGez1T" - }, - "source": [ - "The abstract leaf type dictates the loaded type for each leaf. If we save a value as a `jax.Array` but provide an abstract leaf without the required `sharding` property, Orbax will load as `np.ndarray`. Similarly, we can save as an `int` and load as a `float` if we specify `float` as the abstract leaf." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cDdy1bk0dzDk" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/change-type/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "pytree_with_scalars = {\n", - " 'a': np.asarray(12),\n", - " 'b': 13.5,\n", - " 'c': create_sharded_array(np.arange(8)),\n", - "}\n", - "ocp.save(path, pytree_with_scalars)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "UxWeE0E8eg52" - }, - "outputs": [], - "source": [ - "abstract_state_with_scalars = {\n", - " 'a': float,\n", - " 'b': int,\n", - " 'c': np.empty((8,)),\n", - "}\n", - "ocp.load(path, abstract_state_with_scalars)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LBSbal0hVq_8" - }, - "source": [ - "### Partial Loading\n", - "\n", - "You may wish to load part of a PyTree contained within a saved checkpoint. For example, consider the following item:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "96Zk1nrzVq_8" - }, - "outputs": [], - "source": [ - "original_item = {\n", - " 'params': {\n", - " 'layer1': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " 'layer2': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " },\n", - " 'opt_state': [np.arange(8), np.arange(8)],\n", - " 'step': 101,\n", - "}\n", - "\n", - "path = epath.Path('/tmp/checkpointing-pytrees/partial/')\n", - "path.rmtree(missing_ok=True)\n", - "\n", - "ocp.save(path / '1', original_item)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "g9q0W8GoVq_8" - }, - "source": [ - "If we want to load only a subset of PyTree nodes (`params.layer2` and `step`, for example), we can use Placeholder values." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "s9WFY0vuVq_8" - }, - "source": [ - "#### Placeholder\n", - "\n", - "To load part of a PyTree item, we can specify which nodes to ignore during loading by using `...` (`ocp.PLACEHOLDER`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NHreKKn8Vq_9" - }, - "outputs": [], - "source": [ - "reference_item = {\n", - " 'params': {\n", - " 'layer1': {\n", - " 'kernel': ...,\n", - " 'bias': ...,\n", - " },\n", - " 'layer2': {\n", - " 'kernel': np.arange(8),\n", - " 'bias': np.arange(8),\n", - " },\n", - " },\n", - " 'opt_state': [..., ...],\n", - " 'step': 101,\n", - "}\n", - "\n", - "ocp.load(path / '1', reference_item)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MH3hBSxPVq_9" - }, - "source": [ - "## Advanced Customizations" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nZcttkVHVq_9" - }, - "source": [ - "`ocp.Context` enables more customizations.\n", - "\n", - "For customized save/load behavior, these APIs should be invoked within a `ocp.Context`\n", - "instance, which in turn can be configured with a number of options like Saving, Loading,\n", - "FileOptions etc.\n", - "\n", - "The usage pattern is as follows:\n", - "```\n", - "with ocp.Context(\n", - " pytree_options=PyTreeOptions(...),\n", - " file_options=FileOptions(...),\n", - "):\n", - " ocp.save(path, pytree)\n", - "```\n", - "\n", - "Let's explore few examples. Please also take a look at API Reference for specific option details." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CY0cRK32Vq_9" - }, - "source": [ - "### Saving" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DpuH7FMHVq_9" - }, - "source": [ - "#### Customizing Array dtype" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bq3Hj3-aVq_9" - }, - "source": [ - "we can customize the on-disk type used to save individual arrays. First, let's save and load as normal." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "FZt44Li_Vq_9" - }, - "outputs": [], - "source": [ - "path = epath.Path('/tmp/checkpointing-pytrees/advanced/')\n", - "path.rmtree(missing_ok=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "68BXPuRcVq_9" - }, - "outputs": [], - "source": [ - "ocp.save(path / '1', pytree)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LcXASaTWVq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '1')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "WnCRCzlOVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yhb3Py_9Vq_9" - }, - "source": [ - "Now, let's set the dtype of selective array when saving." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9VpoTnHmVq_9" - }, - "outputs": [], - "source": [ - "def scoped_storage_options_creator(keypath, value):\n", - " del value\n", - " last_key = keypath[-1]\n", - " # Override 'a' to int16\n", - " if isinstance(last_key, jax.tree_util.GetAttrKey) and last_key.name == 'a':\n", - " return ocp.options.ArrayOptions.Saving.StorageOptions(\n", - " dtype=np.dtype(np.int16)\n", - " )\n", - " # Return None to use global default storage_options for other leaves\n", - " return None\n", - "\n", - "ctx = ocp.Context()\n", - "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", - "\n", - "with ctx:\n", - " ocp.save(path / '2', pytree, overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "mHh_ucu_Vq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '2')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sDHCc2s4Vq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P58IxTVHVq_9" - }, - "source": [ - "Now, let's set the dtype of all arrays when saving." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "W-k7jRWFVq_9" - }, - "outputs": [], - "source": [ - "scoped_storage_options_creator = (\n", - " lambda k, v: ocp.options.ArrayOptions.Saving.StorageOptions(\n", - " dtype=np.dtype(np.int16)\n", - " )\n", - ")\n", - "\n", - "ctx = ocp.Context()\n", - "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", - "\n", - "with ctx:\n", - " ocp.save(path / '3', pytree, overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Cgs_vBOWVq_9" - }, - "outputs": [], - "source": [ - "loaded = ocp.load(path / '3')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9acbLs7CVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].dtype, loaded['b'].dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XZZq8Nxelxwn" - }, - "source": [ - "#### High Throughput with `ocdbt` option" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GhMBAVqDn1s2" - }, - "source": [ - "For high throughput and avoid creating separate subdirectories for each leaf, enable `use_ocdbt`. Please note that it is enabled by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "W_FD_od_mb8z" - }, - "outputs": [], - "source": [ - "ctx = ocp.Context()\n", - "ctx.array.saving.use_ocdbt = True\n", - "with ctx:\n", - " ocp.save(path / '4', pytree, overwrite=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G6Cm0PYcoIF4" - }, - "source": [ - "A checkpoint created with this option enabled can be identified by presence of files `manifest.ocdbt` and subdirs like `ocdbt.process_*`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "IA4tTKDGoGlf" - }, - "outputs": [], - "source": [ - "!ls /tmp/checkpointing-pytrees/advanced/4/pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GqA8Kdk6orZc" - }, - "source": [ - "However, for use cases like large stacked models, disabling this option may be more efficient." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qJvm7QxCpUOT" - }, - "outputs": [], - "source": [ - "ctx = ocp.Context()\n", - "ctx.array.saving.use_ocdbt = False\n", - "with ctx:\n", - " ocp.save(path / '5', pytree, overwrite=True)\n", - "\n", - "!ls /tmp/checkpointing-pytrees/advanced/5/pytree" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-OZF-uwBpb8y" - }, - "source": [ - "Please note how each leaf is written in its own subdir when `use_ocdbt=False`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_AnAJYT8Vq_9" - }, - "source": [ - "### Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5aTJ6eUJVq_9" - }, - "source": [ - "#### Pad / truncate shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2Qg0HbH3Vq_9" - }, - "source": [ - "Ordinarily, specifying a target array with a different shape than in the\n", - "checkpoint results in an error." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p1I0sfNEVq_9" - }, - "outputs": [], - "source": [ - "# Original shape.\n", - "loaded = ocp.load(path / '1')\n", - "\n", - "(loaded['a'].shape, loaded['b'].shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2X8mMPBkVq_9" - }, - "outputs": [], - "source": [ - "different_shape_abstract_state = {\n", - " 'a': jax.ShapeDtypeStruct(\n", - " shape=(8,),\n", - " dtype=abstract_state['a'].dtype,\n", - " sharding=abstract_state['a'].sharding,\n", - " ),\n", - " 'b': jax.ShapeDtypeStruct(\n", - " shape=(32,),\n", - " dtype=abstract_state['b'].dtype,\n", - " sharding=abstract_state['b'].sharding,\n", - " ),\n", - "}\n", - "\n", - "try:\n", - " ocp.load(path / '1', different_shape_abstract_state)\n", - "except BaseException as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HMPpVIiLVq_9" - }, - "source": [ - "We can pad or truncate arrays as they are loaded by specifying `enable_padding_and_truncation=True`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Rbu-q1upVq_9" - }, - "outputs": [], - "source": [ - "\n", - "ctx = ocp.Context()\n", - "ctx.array.loading.enable_padding_and_truncation = True\n", - "with ctx:\n", - " loaded = ocp.load(path / '1', different_shape_abstract_state)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HP6gm2HEVq_9" - }, - "outputs": [], - "source": [ - "(loaded['a'].shape, loaded['b'].shape)" - ] - } - ], - "metadata": { - "colab": { - "last_runtime": { - "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", - "kind": "private" - }, - "private_outputs": true, - "provenance": [ - { - "file_id": "1QNxBBBN16Br9Xj-a7LvtJzJWjOBhjFps", - "timestamp": 1686159333109 - } - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" + "cells": [ + { + "id": "6914e6f6", + "cell_type": "markdown", + "source": [ + "# Working with PyTree Checkpoints" + ], + "metadata": { + "id": "GYCcRRZas1PS" + }, + "execution_count": null + }, + { + "id": "c510117b", + "cell_type": "markdown", + "source": [ + "A [PyTree](https://jax.readthedocs.io/en/latest/pytrees.html) is the most common way of representing a training state in JAX. While Orbax is designed to be as generic as possible, and provides customization options for all manner of checkpointable objects, PyTrees naturally have pride of place. Furthermore, the standard object used to represent large, sharded arrays is the {py:class}`jax.Array \u003cjax.Array\u003e`. This, too, has extensive first-class support." + ], + "metadata": { + "id": "iJwUDQVA7GIV" + }, + "execution_count": null + }, + { + "id": "60fd24e1", + "cell_type": "markdown", + "source": [ + "## Exclusive APIs to checkpoint PyTrees" + ], + "metadata": { + "id": "Vfj9mDHsb7QF" + }, + "execution_count": null + }, + { + "id": "af8bc7b5", + "cell_type": "markdown", + "source": [ + "The following APIs can be used to checkpoint PyTrees exclusively.\n", + "\n", + "To save:\n", + "\n", + "* `ocp.save(...)`\n", + "* `ocp.save_async(...)`\n", + "* `training.Checkpointer.save(...)`\n", + "* `training.Checkpointer.save_async(...)`\n", + "\n", + "To load:\n", + "* `ocp.load(...)`\n", + "* `ocp.load_async(...)`\n", + "* `training.Checkpointer.load(...)`\n", + "* `training.Checkpointer.load_async(...)`\n", + "\n", + "Of course, the `save_checkpointables(...)` and `load_checkpointables(...)`\n", + "flavor APIs can be used to save a PyTree too." + ], + "metadata": { + "id": "e3JtbxFhVq_6" + }, + "execution_count": null + }, + { + "id": "c2677229", + "cell_type": "markdown", + "source": [ + "Let's setup a PyTree of jax.Array to play with these APIs." + ], + "metadata": { + "id": "ej99peApVq_7" + }, + "execution_count": null + }, + { + "id": "5bb3f520", + "cell_type": "code", + "source": [ + "from etils import epath\n", + "import jax\n", + "import numpy as np\n", + "import orbax.checkpoint.experimental.v1 as ocp" + ], + "metadata": { + "id": "6NknwqBeVq_7" + }, + "execution_count": null + }, + { + "id": "87e1bb5b", + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('model',)),\n", + " jax.sharding.PartitionSpec(\n", + " 'model',\n", + " ),\n", + ")\n", + "create_sharded_array = lambda x: jax.device_put(x, sharding)\n", + "pytree = {\n", + " 'a': np.arange(16, dtype=np.int32),\n", + " 'b': np.ones(16, dtype=np.int32),\n", + "}\n", + "pytree = jax.tree_util.tree_map(create_sharded_array, pytree)\n", + "pytree" + ], + "metadata": { + "id": "b0j19QxuVq_7" + }, + "execution_count": null + }, + { + "id": "76928efc", + "cell_type": "markdown", + "source": [ + "## Basic Checkpointing" + ], + "metadata": { + "id": "Kf_OCzGbVq_8" + }, + "execution_count": null + }, + { + "id": "345282a3", + "cell_type": "markdown", + "source": [ + "Let's use `ocp.save_*`/`ocp.load_*` to work with the pytree created earlier." + ], + "metadata": { + "id": "sU2YIn10Vq_8" + }, + "execution_count": null + }, + { + "id": "35052c1d", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/basic/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "# Simple save using default options:\n", + "ocp.save(path, pytree)" + ], + "metadata": { + "id": "Szq9yBPhVq_8" + }, + "execution_count": null + }, + { + "id": "61ba3a26", + "cell_type": "markdown", + "source": [ + "We can easily restore using the following snippet.\n", + "\n", + "**Warning: do not use for production-sensitive cases.**" + ], + "metadata": { + "id": "Kr6nrYg3QR73" + }, + "execution_count": null + }, + { + "id": "7abc5fef", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path)\n", + "loaded" + ], + "metadata": { + "id": "CtEMu9BBRHYW" + }, + "execution_count": null + }, + { + "id": "0dc43b69", + "cell_type": "markdown", + "source": [ + "It is not recommended to load this way for production-sensitive cases because the user cannot make any guarantees about what they are loading. If the shapes of some arrays have changed in the model since the checkpoint was saved, errors can be seen when attempting to create the model. If the device topology has changed, we will see errors when attempting to place arrays on devices.\n", + "\n", + "It is therefore recommended that users always specify an **abstract pytree** when loading." + ], + "metadata": { + "id": "F-CPPqeaRRks" + }, + "execution_count": null + }, + { + "id": "99e12640", + "cell_type": "markdown", + "source": [ + "### Understanding Abstract Trees and Leaves" + ], + "metadata": { + "id": "R1Sbx3DYQMpS" + }, + "execution_count": null + }, + { + "id": "5a6792e2", + "cell_type": "markdown", + "source": [ + "An **abstract PyTree** is just a normal PyTree, but with abstract leaves. An **abstract leaf** is a cheap representation of a leaf type (such as an array) that contains only metadata, and does not represent the real values. (Contrast with a *concrete* PyTree, which contains real data in the form of large arrays, and other types.)" + ], + "metadata": { + "id": "4f8Ze-q3R5XE" + }, + "execution_count": null + }, + { + "id": "0353949d", + "cell_type": "markdown", + "source": [ + "Let's create an abstract PyTree matching the structure of the PyTree we originally saved." + ], + "metadata": { + "id": "PVyCvC7VaisK" + }, + "execution_count": null + }, + { + "id": "95dc146f", + "cell_type": "code", + "source": [ + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", + " 'b': jax.ShapeDtypeStruct(shape=(16,), dtype=np.int32, sharding=sharding),\n", + "}\n", + "abstract_state" + ], + "metadata": { + "id": "u5JJaHrYVq_7" + }, + "execution_count": null + }, + { + "id": "d6332cfd", + "cell_type": "code", + "source": [ + "# Load using abstract_state.\n", + "loaded = ocp.load(path, abstract_state)\n", + "loaded" + ], + "metadata": { + "id": "0GCij-iuVq_8" + }, + "execution_count": null + }, + { + "id": "299c34c5", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "TjOlZAkTVq_8" + }, + "execution_count": null + }, + { + "id": "8917eb60", + "cell_type": "markdown", + "source": [ + "The `metadata` method returns a `CheckpointMetadata` object with a number of properties, but the core `metadata` property is just an abstract PyTree. This can also be used for loading as shown below." + ], + "metadata": { + "id": "tQ6L_wtnVq_8" + }, + "execution_count": null + }, + { + "id": "0664d045", + "cell_type": "code", + "source": [ + "metadata = ocp.metadata(path).metadata\n", + "metadata" + ], + "metadata": { + "id": "_sJOi2I3Vq_8" + }, + "execution_count": null + }, + { + "id": "205df646", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path, metadata)\n", + "loaded" + ], + "metadata": { + "id": "1SQMyU44Vq_8" + }, + "execution_count": null + }, + { + "id": "37966ae1", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "jGTdk0tVVq_8" + }, + "execution_count": null + }, + { + "id": "b2e2b86a", + "cell_type": "markdown", + "source": [ + "Note that it is also valid to provide a \"concrete\" PyTree for loading rather than an \"abstract\" target, since by definition, the concrete leaves contain all the same properties provided by the abstract leaves.\n", + "\n", + "However, this requires that you fully initialize the target train state\n", + "before loading from the checkpoint, which is inefficient or impractical for real use cases. It is better practice to only initialize metadata (either by manually creating `jax.ShapeDtypeStruct`s or using `jax.eval_shape`)." + ], + "metadata": { + "id": "U7URSieWVq_8" + }, + "execution_count": null + }, + { + "id": "4fb8d9da", + "cell_type": "code", + "source": [ + "ocp.load(path, pytree)" + ], + "metadata": { + "id": "3JfwbnA_Vq_8" + }, + "execution_count": null + }, + { + "id": "bd7690cf", + "cell_type": "markdown", + "source": [ + "### Standard Leaf Types" + ], + "metadata": { + "id": "wjpnDhAqb6t0" + }, + "execution_count": null + }, + { + "id": "5968ec28", + "cell_type": "markdown", + "source": [ + "The following standard leaf types are supported by Orbax by default. Each concrete leaf type has a corresponding abstract leaf type. Most abstract types are implemented as `Protocol`'s, so that any object implementing the required properties can be accepted as a valid abstract type." + ], + "metadata": { + "id": "p6yM2hBGdNXZ" + }, + "execution_count": null + }, + { + "id": "4573a1ae", + "cell_type": "markdown", + "source": [ + "| `Leaf` Type | `AbstractLeaf` Type | Properties |\n", + ":------- | :-------- | :-------- |\n", + "|`jax.Array`|`AbstractShardedArray` (`jax.ShapeDtypeStruct`) |`shape`, `dtype`, `sharding`|\n", + "|`np.ndarray`|`AbstractArray` (`np.ndarray`) |`shape`, `dtype`|\n", + "|`int`|`int`| |\n", + "|`float`|`float`| |\n", + "|`bytes`|`bytes`| |\n", + "|`str`|`str`| |" + ], + "metadata": { + "id": "G0oCkcDZb8g_" + }, + "execution_count": null + }, + { + "id": "da09a3b7", + "cell_type": "markdown", + "source": [ + "`None` is always a valid abstract leaf; it serves as an indication that the leaf should be restored using metadata stored in the checkpoint.\n", + "\n", + "`Type[AbstractLeaf]` is also always a valid abstract leaf; it again serves as an indication that the leaf should be restored using the metadata, but with the additional constraint to load as the indicated type. For example, instead of specifying `jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=...)`, it is sufficient to pass `jax.ShapeDtypeStruct`. Similarly, instead of passing `0` to restore as an `int`, the type itself may be passed." + ], + "metadata": { + "id": "01zrVpfcdq7m" + }, + "execution_count": null + }, + { + "id": "41d64797", + "cell_type": "markdown", + "source": [ + "To summarize, here are the ways you can load a PyTree using abstract leaves, with the way we most recommend at the top, and the way we least recommend at the bottom.\n", + "\n", + "**1. Fully-specified abstract values**\n", + "\n", + "This provides the most loading validations and requires the least amount of\n", + "unnecessary metadata reads.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(shape=..., dtype=..., sharding=jax.sharding.NamedSharding(...))\n", + "}\n", + "```\n", + "\n", + "**2. Only types specified**\n", + "\n", + "This guarantees that each leaf will be loaded with the indicated type, but metadata\n", + "will be used to restore specific properties for each leaf.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct,\n", + " 'b': int,\n", + " 'c': np.ndarray,\n", + "}\n", + "```\n", + "\n", + "**3. `None` specified (per-leaf)**\n", + "\n", + "This is essentially the same as (2), but metadata will also be used to decide\n", + "which type each leaf should be loaded as.\n", + "\n", + "```\n", + "abstract_state = {\n", + " 'a': None,\n", + " 'b': None,\n", + "}\n", + "```\n", + "\n", + "**4. `None` specified**\n", + "\n", + "This loads the PyTree structure without any checks, and can lead to errors later\n", + "in your code if the checkpoint does not have the structure you expect.\n", + "\n", + "```\n", + "abstract_state = None\n", + "```" + ], + "metadata": { + "id": "nRn_2IgV09xT" + }, + "execution_count": null + }, + { + "id": "b6a7681c", + "cell_type": "markdown", + "source": [ + "### Customizing Loaded Properties for Arrays" + ], + "metadata": { + "id": "UR34KTImVq_8" + }, + "execution_count": null + }, + { + "id": "9369d5dd", + "cell_type": "markdown", + "source": [ + "#### Array dtype" + ], + "metadata": { + "id": "TErvR-zDVq_8" + }, + "execution_count": null + }, + { + "id": "971b5f81", + "cell_type": "code", + "source": [ + "def set_loading_dtype(x: jax.ShapeDtypeStruct) -\u003e jax.ShapeDtypeStruct:\n", + " return x.update(dtype=np.int16)\n", + "\n", + "\n", + "cast_dtype_abstract_state = jax.tree_util.tree_map(\n", + " set_loading_dtype, abstract_state\n", + ")" + ], + "metadata": { + "id": "ZDSJhefZVq_8" + }, + "execution_count": null + }, + { + "id": "0b59e9a7", + "cell_type": "code", + "source": [ + "ocp.load(path, cast_dtype_abstract_state)" + ], + "metadata": { + "id": "scZbqSQ4Vq_8" + }, + "execution_count": null + }, + { + "id": "803e5cc5", + "cell_type": "markdown", + "source": [ + "#### Change sharding" + ], + "metadata": { + "id": "rhkTT9wMVq_8" + }, + "execution_count": null + }, + { + "id": "238e155f", + "cell_type": "markdown", + "source": [ + "**NOTE: This can often be a particularly sharp edge.**\n", + "\n", + "Sharding commonly needs to be changed when loading a checkpoint saved on one topology to a different topology.\n", + "\n", + "**If changing topologies, you MUST specify sharding when loading.**\n", + "\n", + "Unless you are loading on the exact same topology, Orbax does not make any decisions about shardings on your behalf. If you have the exact same topology,\n", + "however, it is possible to avoid specifying the sharding when loading. This is demonstrated below:" + ], + "metadata": { + "id": "NLjuYubRVq_8" + }, + "execution_count": null + }, + { + "id": "196eef08", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path)" + ], + "metadata": { + "id": "atPqix9IVq_8" + }, + "execution_count": null + }, + { + "id": "ea0ac0bf", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "Pap_vpN9Vq_8" + }, + "execution_count": null + }, + { + "id": "31011e33", + "cell_type": "markdown", + "source": [ + "In the example below, we alter the sharding while loading." + ], + "metadata": { + "id": "T_T3QjRLVq_8" + }, + "execution_count": null + }, + { + "id": "ca7af6eb", + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('x',)),\n", + " jax.sharding.PartitionSpec(),\n", + ")\n", + "\n", + "\n", + "def set_sharding(x: jax.ShapeDtypeStruct) -\u003e jax.ShapeDtypeStruct:\n", + " return x.update(sharding=sharding)\n", + "\n", + "\n", + "change_sharding_abstract_state = jax.tree_util.tree_map(\n", + " set_sharding, abstract_state\n", + ")\n", + "loaded = ocp.load(path, change_sharding_abstract_state)" + ], + "metadata": { + "id": "J0tvS901Vq_8" + }, + "execution_count": null + }, + { + "id": "23a43ee2", + "cell_type": "code", + "source": [ + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "M84bx2smVq_8" + }, + "execution_count": null + }, + { + "id": "b73ad6a0", + "cell_type": "markdown", + "source": [ + "We can use pytree metadata instead of the abstract pytree." + ], + "metadata": { + "id": "zA_Vdck2Vq_8" + }, + "execution_count": null + }, + { + "id": "28cc7757", + "cell_type": "code", + "source": [ + "metadata = ocp.metadata(path).metadata\n", + "change_sharding_metadata = jax.tree_util.tree_map(\n", + " lambda x: jax.ShapeDtypeStruct(shape=x.shape, dtype=x.dtype, sharding=sharding), metadata\n", + ")\n", + "loaded = ocp.load(path, change_sharding_metadata)\n", + "(loaded['a'].sharding, loaded['b'].sharding)" + ], + "metadata": { + "id": "cAN_1Xj_Vq_8" + }, + "execution_count": null + }, + { + "id": "5cc090f6", + "cell_type": "markdown", + "source": [ + "#### Change leaf type" + ], + "metadata": { + "id": "sIj3AYAidwvh" + }, + "execution_count": null + }, + { + "id": "758157c2", + "cell_type": "markdown", + "source": [ + "The abstract leaf type dictates the loaded type for each leaf. If we save a value as a `jax.Array` but provide an abstract leaf without the required `sharding` property, Orbax will load as `np.ndarray`. Similarly, we can save as an `int` and load as a `float` if we specify `float` as the abstract leaf." + ], + "metadata": { + "id": "01BAYxDGez1T" + }, + "execution_count": null + }, + { + "id": "dd9a8ce2", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/change-type/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "pytree_with_scalars = {\n", + " 'a': np.asarray(12),\n", + " 'b': 13.5,\n", + " 'c': create_sharded_array(np.arange(8)),\n", + "}\n", + "ocp.save(path, pytree_with_scalars)" + ], + "metadata": { + "id": "cDdy1bk0dzDk" + }, + "execution_count": null + }, + { + "id": "c91174ff", + "cell_type": "code", + "source": [ + "abstract_state_with_scalars = {\n", + " 'a': float,\n", + " 'b': int,\n", + " 'c': np.empty((8,)),\n", + "}\n", + "ocp.load(path, abstract_state_with_scalars)" + ], + "metadata": { + "id": "UxWeE0E8eg52" + }, + "execution_count": null + }, + { + "id": "153334fc", + "cell_type": "markdown", + "source": [ + "### Partial Loading\n", + "\n", + "You may wish to load part of a PyTree contained within a saved checkpoint. For example, consider the following item:" + ], + "metadata": { + "id": "LBSbal0hVq_8" + }, + "execution_count": null + }, + { + "id": "90272421", + "cell_type": "code", + "source": [ + "original_item = {\n", + " 'params': {\n", + " 'layer1': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " 'layer2': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " },\n", + " 'opt_state': [np.arange(8), np.arange(8)],\n", + " 'step': 101,\n", + "}\n", + "\n", + "path = epath.Path('/tmp/checkpointing-pytrees/partial/')\n", + "path.rmtree(missing_ok=True)\n", + "\n", + "ocp.save(path / '1', original_item)" + ], + "metadata": { + "id": "96Zk1nrzVq_8" + }, + "execution_count": null + }, + { + "id": "2a00857d", + "cell_type": "markdown", + "source": [ + "If we want to load only a subset of PyTree nodes (`params.layer2` and `step`, for example), we can use Placeholder values." + ], + "metadata": { + "id": "g9q0W8GoVq_8" + }, + "execution_count": null + }, + { + "id": "b84eaaeb", + "cell_type": "markdown", + "source": [ + "#### Placeholder\n", + "\n", + "To load part of a PyTree item, we can specify which nodes to ignore during loading by using `...` (`ocp.PLACEHOLDER`)." + ], + "metadata": { + "id": "s9WFY0vuVq_8" + }, + "execution_count": null + }, + { + "id": "d1c17716", + "cell_type": "code", + "source": [ + "reference_item = {\n", + " 'params': {\n", + " 'layer1': {\n", + " 'kernel': ...,\n", + " 'bias': ...,\n", + " },\n", + " 'layer2': {\n", + " 'kernel': np.arange(8),\n", + " 'bias': np.arange(8),\n", + " },\n", + " },\n", + " 'opt_state': [..., ...],\n", + " 'step': 101,\n", + "}\n", + "\n", + "ocp.load(path / '1', reference_item)" + ], + "metadata": { + "id": "NHreKKn8Vq_9" + }, + "execution_count": null + }, + { + "id": "6bfc7009", + "cell_type": "markdown", + "source": [ + "## Advanced Customizations" + ], + "metadata": { + "id": "MH3hBSxPVq_9" + }, + "execution_count": null + }, + { + "id": "e4a66e76", + "cell_type": "markdown", + "source": [ + "`ocp.Context` enables more customizations.\n", + "\n", + "For customized save/load behavior, these APIs should be invoked within a `ocp.Context`\n", + "instance, which in turn can be configured with a number of options like Saving, Loading,\n", + "FileOptions etc.\n", + "\n", + "The usage pattern is as follows:\n", + "```\n", + "with ocp.Context(\n", + " pytree_options=PyTreeOptions(...),\n", + " file_options=FileOptions(...),\n", + "):\n", + " ocp.save(path, pytree)\n", + "```\n", + "\n", + "Let's explore few examples. Please also take a look at API Reference for specific option details." + ], + "metadata": { + "id": "nZcttkVHVq_9" + }, + "execution_count": null + }, + { + "id": "da3e1ed1", + "cell_type": "markdown", + "source": [ + "### Saving" + ], + "metadata": { + "id": "CY0cRK32Vq_9" + }, + "execution_count": null + }, + { + "id": "79a09a4a", + "cell_type": "markdown", + "source": [ + "#### Customizing Array dtype" + ], + "metadata": { + "id": "DpuH7FMHVq_9" + }, + "execution_count": null + }, + { + "id": "a621ff4a", + "cell_type": "markdown", + "source": [ + "we can customize the on-disk type used to save individual arrays. First, let's save and load as normal." + ], + "metadata": { + "id": "bq3Hj3-aVq_9" + }, + "execution_count": null + }, + { + "id": "a2264fa6", + "cell_type": "code", + "source": [ + "path = epath.Path('/tmp/checkpointing-pytrees/advanced/')\n", + "path.rmtree(missing_ok=True)" + ], + "metadata": { + "id": "FZt44Li_Vq_9" + }, + "execution_count": null + }, + { + "id": "7af654a9", + "cell_type": "code", + "source": [ + "ocp.save(path / '1', pytree)" + ], + "metadata": { + "id": "68BXPuRcVq_9" + }, + "execution_count": null + }, + { + "id": "25b249e1", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '1')" + ], + "metadata": { + "id": "LcXASaTWVq_9" + }, + "execution_count": null + }, + { + "id": "038f2721", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "WnCRCzlOVq_9" + }, + "execution_count": null + }, + { + "id": "a5a088ef", + "cell_type": "markdown", + "source": [ + "Now, let's set the dtype of selective array when saving." + ], + "metadata": { + "id": "yhb3Py_9Vq_9" + }, + "execution_count": null + }, + { + "id": "08d2a51e", + "cell_type": "code", + "source": [ + "def scoped_storage_options_creator(keypath, value):\n", + " del value\n", + " last_key = keypath[-1]\n", + " # Override 'a' to int16\n", + " if isinstance(last_key, jax.tree_util.GetAttrKey) and last_key.name == 'a':\n", + " return ocp.options.ArrayOptions.Saving.StorageOptions(\n", + " dtype=np.dtype(np.int16)\n", + " )\n", + " # Return None to use global default storage_options for other leaves\n", + " return None\n", + "\n", + "ctx = ocp.Context()\n", + "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", + "\n", + "with ctx:\n", + " ocp.save(path / '2', pytree, overwrite=True)" + ], + "metadata": { + "id": "9VpoTnHmVq_9" + }, + "execution_count": null + }, + { + "id": "7c4c8c9f", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '2')" + ], + "metadata": { + "id": "mHh_ucu_Vq_9" + }, + "execution_count": null + }, + { + "id": "e1cf3f3b", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "sDHCc2s4Vq_9" + }, + "execution_count": null + }, + { + "id": "70eb8746", + "cell_type": "markdown", + "source": [ + "Now, let's set the dtype of all arrays when saving." + ], + "metadata": { + "id": "P58IxTVHVq_9" + }, + "execution_count": null + }, + { + "id": "48917323", + "cell_type": "code", + "source": [ + "scoped_storage_options_creator = (\n", + " lambda k, v: ocp.options.ArrayOptions.Saving.StorageOptions(\n", + " dtype=np.dtype(np.int16)\n", + " )\n", + ")\n", + "\n", + "ctx = ocp.Context()\n", + "ctx.array.saving.scoped_storage_options_creator = scoped_storage_options_creator\n", + "\n", + "with ctx:\n", + " ocp.save(path / '3', pytree, overwrite=True)" + ], + "metadata": { + "id": "W-k7jRWFVq_9" + }, + "execution_count": null + }, + { + "id": "506d2a46", + "cell_type": "code", + "source": [ + "loaded = ocp.load(path / '3')" + ], + "metadata": { + "id": "Cgs_vBOWVq_9" + }, + "execution_count": null + }, + { + "id": "3e415dcb", + "cell_type": "code", + "source": [ + "(loaded['a'].dtype, loaded['b'].dtype)" + ], + "metadata": { + "id": "9acbLs7CVq_9" + }, + "execution_count": null + }, + { + "id": "03152a87", + "cell_type": "markdown", + "source": [ + "#### High Throughput with `ocdbt` option" + ], + "metadata": { + "id": "XZZq8Nxelxwn" + }, + "execution_count": null + }, + { + "id": "653e3c07", + "cell_type": "markdown", + "source": [ + "For high throughput and avoid creating separate subdirectories for each leaf, enable `use_ocdbt`. Please note that it is enabled by default." + ], + "metadata": { + "id": "GhMBAVqDn1s2" + }, + "execution_count": null + }, + { + "id": "f6daa343", + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = True\n", + "with ctx:\n", + " ocp.save(path / '4', pytree, overwrite=True)" + ], + "metadata": { + "id": "W_FD_od_mb8z" + }, + "execution_count": null + }, + { + "id": "38c16f01", + "cell_type": "markdown", + "source": [ + "A checkpoint created with this option enabled can be identified by presence of files `manifest.ocdbt` and subdirs like `ocdbt.process_*`." + ], + "metadata": { + "id": "G6Cm0PYcoIF4" + }, + "execution_count": null + }, + { + "id": "7c9a8da3", + "cell_type": "code", + "source": [ + "print(sorted([p.name for p in (path / '4' / 'state').iterdir()]))" + ], + "metadata": { + "id": "IA4tTKDGoGlf" + }, + "execution_count": null + }, + { + "id": "bad9abc2", + "cell_type": "markdown", + "source": [ + "However, for use cases like large stacked models, disabling this option may be more efficient." + ], + "metadata": { + "id": "GqA8Kdk6orZc" + }, + "execution_count": null + }, + { + "id": "603bf145", + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = False\n", + "with ctx:\n", + " ocp.save(path / '5', pytree, overwrite=True)\n", + "\n", + "print(sorted([p.name for p in (path / '5' / 'state').iterdir()]))" + ], + "metadata": { + "id": "qJvm7QxCpUOT" + }, + "execution_count": null + }, + { + "id": "ec1db3e0", + "cell_type": "markdown", + "source": [ + "Please note how each leaf is written in its own subdir when `use_ocdbt=False`." + ], + "metadata": { + "id": "-OZF-uwBpb8y" + }, + "execution_count": null + }, + { + "id": "6a9d4943", + "cell_type": "markdown", + "source": [ + "### Loading" + ], + "metadata": { + "id": "_AnAJYT8Vq_9" + }, + "execution_count": null + }, + { + "id": "ef2e649f", + "cell_type": "markdown", + "source": [ + "#### Pad / truncate shape" + ], + "metadata": { + "id": "5aTJ6eUJVq_9" + }, + "execution_count": null + }, + { + "id": "f813d121", + "cell_type": "markdown", + "source": [ + "Ordinarily, specifying a target array with a different shape than in the\n", + "checkpoint results in an error." + ], + "metadata": { + "id": "2Qg0HbH3Vq_9" + }, + "execution_count": null + }, + { + "id": "bcf99098", + "cell_type": "code", + "source": [ + "# Original shape.\n", + "loaded = ocp.load(path / '1')\n", + "\n", + "(loaded['a'].shape, loaded['b'].shape)" + ], + "metadata": { + "id": "p1I0sfNEVq_9" + }, + "execution_count": null + }, + { + "id": "2a8a944e", + "cell_type": "code", + "source": [ + "different_shape_abstract_state = {\n", + " 'a': jax.ShapeDtypeStruct(\n", + " shape=(8,),\n", + " dtype=abstract_state['a'].dtype,\n", + " sharding=abstract_state['a'].sharding,\n", + " ),\n", + " 'b': jax.ShapeDtypeStruct(\n", + " shape=(32,),\n", + " dtype=abstract_state['b'].dtype,\n", + " sharding=abstract_state['b'].sharding,\n", + " ),\n", + "}\n", + "\n", + "try:\n", + " ocp.load(path / '1', different_shape_abstract_state)\n", + "except BaseException as e:\n", + " print(e)" + ], + "metadata": { + "id": "2X8mMPBkVq_9" + }, + "execution_count": null + }, + { + "id": "ded90dfd", + "cell_type": "markdown", + "source": [ + "We can pad or truncate arrays as they are loaded by specifying `enable_padding_and_truncation=True`." + ], + "metadata": { + "id": "HMPpVIiLVq_9" + }, + "execution_count": null + }, + { + "id": "bd96c69c", + "cell_type": "code", + "source": [ + "\n", + "ctx = ocp.Context()\n", + "ctx.array.loading.enable_padding_and_truncation = True\n", + "with ctx:\n", + " loaded = ocp.load(path / '1', different_shape_abstract_state)" + ], + "metadata": { + "id": "Rbu-q1upVq_9" + }, + "execution_count": null + }, + { + "id": "322ae4c3", + "cell_type": "code", + "source": [ + "(loaded['a'].shape, loaded['b'].shape)" + ], + "metadata": { + "id": "HP6gm2HEVq_9" + }, + "execution_count": null + } + ], + "metadata": { + "colab": { + "last_runtime": { + "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", + "kind": "private" + }, + "private_outputs": true, + "provenance": [ + { + "file_id": "1QNxBBBN16Br9Xj-a7LvtJzJWjOBhjFps", + "timestamp": 1686159333109 } + ], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 0 + "language_info": { + "name": "python" + } + }, + "nbformat_minor": 0, + "nbformat": 4 } diff --git a/docs/guides/checkpoint/v1/maximizing_performance.ipynb b/docs/guides/checkpoint/v1/maximizing_performance.ipynb new file mode 100644 index 0000000000..bfedf8de40 --- /dev/null +++ b/docs/guides/checkpoint/v1/maximizing_performance.ipynb @@ -0,0 +1,848 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "colab": { + "provenance": [ + { + "file_id": "1GZkiWMcvYFs_tW-Y-p07R4M5YU8N9K1r", + "timestamp": 1781122549898 + } + ], + "last_runtime": { + "build_target": "//experimental/users/cpgaffney/colab:orbax_colab", + "kind": "private" + }, + "private_outputs": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Maximizing Performance" + ], + "metadata": { + "id": "Z7zpSM8GbuVA" + }, + "id": "Z7zpSM8GbuVA" + }, + { + "cell_type": "markdown", + "source": [ + "Checkpointing performance is a core focus of the Orbax team and we strive to make our library as performant out-of-the-box as possible. However, many performance features are tailored for more specific use cases and are usable on\n", + "an opt-in basis.\n", + "\n", + "When we say \"performance\" with regards to checkpointing, we generally have a few primary metrics:\n", + "\n", + "\n", + "* **Save blocking time:** The time spent blocking the main thread. In an async save, this is largely attributable to the device/host transfer bandwidth (e.g. 32 GB/s with TPU PCIe Gen5) and is minimally attributable to Orbax.\n", + "* **Save total time:** The total time required to save a checkpoint completely. Since the majority of the total time is comprised of disk I/O occuring in a background thread, its performance impact is less obvious. However, we still want to ensure reliably fast checkpoint completions, as frequent checkpointing can [increase overall training goodput](https://developers.googleblog.com/boost-training-goodput-how-continuous-checkpointing-optimizes-reliability-in-orbax-and-maxtext/).\n", + "* **Load total time:** The total time required to load a checkpoint. This is critical for warmstart training, evaluations, and inference. It impacts accelerator utilization, experiment recovery times, and developer velocity. While loads are performed less frequently than saves, their blocking nature makes them more critical.\n", + "* **Memory consumption:** The additional RAM consumed by checkpointing. RAM is a scarce resource just as accelerators are and excess memory consumed by checkpointing inhibits other useful applications of available memory.\n", + "\n" + ], + "metadata": { + "id": "IfDQ4BvjdlUL" + }, + "id": "IfDQ4BvjdlUL" + }, + { + "cell_type": "markdown", + "source": [ + "Below, we will reference various performance benchmarks. These are performed with single- or multi-region GCS buckets using Llama 3.1 models (8B, 70B, and 405B) adapted for JAX\n", + "using [MaxText](https://maxtext.readthedocs.io/). Typically, we use TPU topologies v5p-8, v5p-32, and v5p-128 for each model size, respectively, unless otherwise noted." + ], + "metadata": { + "id": "Ru3PCuGWeS4d" + }, + "id": "Ru3PCuGWeS4d" + }, + { + "cell_type": "code", + "source": [ + "import jax\n", + "from jax import numpy as jnp\n", + "from orbax.checkpoint import v1 as ocp\n", + "import os\n", + "import numpy as np\n", + "from etils import epath\n", + "\n", + "# Use 16 fake devices.\n", + "os.environ[\"XLA_FLAGS\"] = \"--xla_force_host_platform_device_count=16\"\n", + "root_directory = epath.Path('/tmp/maximizing_performance')\n", + "root_directory.rmtree(missing_ok=True)" + ], + "metadata": { + "id": "m4vRKQH9uGpt" + }, + "execution_count": null, + "outputs": [], + "id": "m4vRKQH9uGpt" + }, + { + "cell_type": "markdown", + "source": [ + "## Key Performance Optimizations" + ], + "metadata": { + "id": "wGLp4RbEdFS5" + }, + "id": "wGLp4RbEdFS5" + }, + { + "cell_type": "markdown", + "source": [ + "### Asynchronous Checkpointing" + ], + "metadata": { + "id": "X9q5tTpleOpY" + }, + "id": "X9q5tTpleOpY" + }, + { + "cell_type": "markdown", + "source": [ + "Async checkpointing is THE key checkpointing performance improvement, as it can hide almost all of the cost of slow disk I/O behind training computations. It is strongly recommended that all users should enable this feature. See our [guide](https://orbax.readthedocs.io/en/latest/guides/checkpoint/v1/async_checkpointing.html) for more details." + ], + "metadata": { + "id": "Vvsjf1PWeT_2" + }, + "id": "Vvsjf1PWeT_2" + }, + { + "cell_type": "markdown", + "source": [ + "### Storage Format" + ], + "metadata": { + "id": "sGEcdDp1byZK" + }, + "id": "sGEcdDp1byZK" + }, + { + "cell_type": "markdown", + "source": [ + "Orbax provides two [storage formats](https://orbax.readthedocs.io/en/latest/guides/checkpoint/v1/checkpoint_format.html#pytree-checkpointables) (leveraging [TensorStore](https://google.github.io/tensorstore/)) referred to as `OCDBT` and `non-OCDBT`.\n", + "\n", + "At a high level, the `non-OCDBT` format stores each model weight as a separate directory while the `OCDBT` format packs model weights into a series of uniform files.\n" + ], + "metadata": { + "id": "sIr79qECjwMb" + }, + "id": "sIr79qECjwMb" + }, + { + "cell_type": "markdown", + "source": [ + "#### `OCDBT` Format\n", + "This format packs model weights into a series of uniform files.\n", + "\n", + "**+++**\n", + "\n", + "* Improved performance for large models.\n", + "* More tunable settings for extracting performance on different filesystems.\n", + "\n", + "**---**\n", + "\n", + "* Higher overhead, poorer performance for small models." + ], + "metadata": { + "id": "dABkdAeE0oWQ" + }, + "id": "dABkdAeE0oWQ" + }, + { + "cell_type": "markdown", + "source": [ + "#### `non-OCDBT` Format\n", + "This format stores each model weight as a separate sub-directory.\n", + "\n", + "**+++**\n", + "\n", + "* More human-readable.\n", + "* Low-overhead on small models.\n", + "\n", + "**---**\n", + "\n", + "* Performs poorly for models with many distinct arrays (especially small arrays).\n", + "* Poorer performance for large models." + ], + "metadata": { + "id": "zxj_93sz0qg8" + }, + "id": "zxj_93sz0qg8" + }, + { + "cell_type": "markdown", + "source": [ + "| Operation | Model | Topology | OCDBT | No OCDBT | Speedup |\n", + "| :--- | :--- | :--- | :--- | :--- | :--- |\n", + "| Load | 8B | v5p-8 | 22.85 ± 4.68 | 17.97 ± 4.22 | 0.79×* |\n", + "| Load | 70B | v5p-32 | 41.19 ± 4.17 | 40.04 ± 4.61 | 0.97× |\n", + "| Load | 405B | v5p-128| 59.14 ± 4.06 | 65.18 ± 12.01 | **1.10×** |\n", + "| Save (Background) | v5p-8 | 8B | 59.37 ± 9.04 | 33.71 ± 3.20 | 0.57×* |\n", + "| Save (Background) | v5p-32 | 70B | 62.47 ± 9.09 | 80.84 ± 8.82 | **1.29×** |\n", + "| Save (Background) | v5p-64 | 405B | 91.76 ± 14.91 | 123.33 ± 19.27 | **1.34×** |" + ], + "metadata": { + "id": "Yxmp4zh5i8MR" + }, + "id": "Yxmp4zh5i8MR" + }, + { + "cell_type": "markdown", + "source": [ + "You can configure the storage format following this example:" + ], + "metadata": { + "id": "IfB0aOAIupoi" + }, + "id": "IfB0aOAIupoi" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.use_ocdbt = True # default\n", + "with ctx:\n", + " path = root_directory / 'ckpt_ocdbt'\n", + " ocp.save(path, {'a': jnp.arange(8)})\n", + " assert os.path.exists(path / 'state' / 'manifest.ocdbt')\n", + "\n", + "ctx.array.saving.use_ocdbt = False\n", + "with ctx:\n", + " path = root_directory / 'ckpt_non_ocdbt'\n", + " ocp.save(path, {'a': jnp.arange(8)})\n", + " assert not os.path.exists(path / 'state' / 'manifest.ocdbt')" + ], + "metadata": { + "id": "fMZ4DPnHuoBV" + }, + "execution_count": null, + "outputs": [], + "id": "fMZ4DPnHuoBV" + }, + { + "cell_type": "markdown", + "source": [ + "### Loading with Resharding" + ], + "metadata": { + "id": "qNpl940nd7Ai" + }, + "id": "qNpl940nd7Ai" + }, + { + "cell_type": "markdown", + "source": [ + "When warm-starting or recovering a training experiment, or when performing evaluations or inference on a different topology than the training topology, *resharding* is needed.\n", + "\n", + "To facilitate efficient (fast and low-memory-overhead) checkpoint loading with resharding, we need to first understand a bit about how Orbax uses TensorStore to store large distributed arrays.\n", + "\n", + "Using TensorStore's zarr driver, distributed arrays are represented in a chunked format, where `jax.Array` shards correspond one-to-one with TensorStore shards. If an array is resharded and its new boundaries don't align with the originally saved chunks, the system is forced to load and discard excess data, which increases I/O and memory (RAM) overhead.\n", + "\n", + "However, we can subdivide larger write chunks by configuring granular \"subchunks\" at save time, which reduces overhead." + ], + "metadata": { + "id": "NP9etjit0sM2" + }, + "id": "NP9etjit0sM2" + }, + { + "cell_type": "markdown", + "source": [ + "This is illustrated in the figure below. For example, if our array is shape `(4, 4)`, it may be sharded across 4 devices using shard shapes of `(2, 2)`. The checkpoint will be saved with chunks of the same shape.\n", + "\n", + "If we attempt a reshard operation where we need to load shards of shape `(1, 2)`, each read will have 2x overhead, since the `(2, 2)` chunk is the smallest readable unit.\n", + "\n", + "This can be resolved if we configure `(1, 1)` chunk shapes though, as a `(1, 2)` will read two chunks with no overhead." + ], + "metadata": { + "id": "8RAwtLBhr5h8" + }, + "id": "8RAwtLBhr5h8" + }, + { + "cell_type": "markdown", + "source": [ + "![8K5xDXQpmtFovE9.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAxUAAAG3CAYAAADCVnjGAAAQAElEQVR4AeydBWAUR9/Gn02ABHd3dy9UoMXa0lJ3L9C3+lXevrW3rm/dnbq7Gy1t0UKhuLu7EyAQCJFvn0nmuNxdkktySU4eyOzszvx35Ld7M/Mf27hM/RMBERABERABERABERABERCBIhCIg/6JgAhEAAElUQREQAREQAREQATCl4CUivB9NkqZCIiACIhApBFQekVABEQgRglIqYjRB69si4AIiIAIiIAIiECsElC+Q09ASkXomSpEERABERABERABERABEYgpAlIqYupxl1RmFY8IiIAIiIAIiIAIiEAsEZBSEUtPW3kVAREQAW8COhcBERABERCBEBGQUhEikApGBERABERABERABIqDgMIUgUggIKUiEp6S0igCIiACIiACIiACIiACYUxASgXC+OkoaSIgAiIgAiIgAiIgAiIQAQSkVETAQ1ISRUAEAAiCCIiACIiACIhA2BKQUhG2j0YJEwEREAEREIHII6AUi4AIxCYBKRWx+dyVaxEQAREQAREQAREQgdglEPKcS6kIOVIFKAIiIAIiIAIiIAIiIAKxRUBKRWw9b+W2pAgoHhEQAREQAREQARGIIQJSKmLoYSurIiACIiACOQnoSgREQAREIDQEpFSEhqNCEQEREAEREAEREAERKB4CCjUCCEipiICHpCSKgAiIgAiIgAiIgAiIQDgTkFIRzk+npNKmeERABERABERABERABESgCASkVBQBnm4VAREQgZIkoLhEQAREQAREIFwJSKkI1yejdImACIiACIiACEQiAaVZBGKSgJSKmHzsyrQIiIAIiIAIiIAIiIAIhI5A5CkVocu7QhIBERABERABERABERABEQgBASkVIYCoIERABPwJyEUEREAEREAERCB2CEipiJ1nrZyKgAiIgAiIgC8BXYuACIhASAhIqQgJRgUiAiIgAiIgAiIgAiIgAsVFIPzDlVIR/s9IKRQBERABERABERABERCBsCYgpSKsH48SV1IEFI8IiIAIiIAIiIAIiEDhCUipKDw73SkCIiACIlCyBBSbCIiACIhAmBKQUhGmD0bJEgEREAEREAEREIHIJKBUxyIBKRWx+NSVZxEQAREQAREQAREQAREIIQEpFSGEWVJBKR4REAEREAEREAEREAERCCcCUirC6WkoLSIgAtFEQHkRAREQAREQgZghIKUiZh61MioCIiACIiACIuBPQC4iIAKhICClIhQUFYYIiIAIiIAIiIAIiIAIxDCBYlcqYpitsi4CIiACIiACIiACIiACMUFASkVMPGZlUgTyJSABERABERABERABESg0ASkVhUYX+ht37dqFjRs35giYbhkZGTncdCECIiACIhCrBJTvSCeQkpKCtLS0ImcjNTUVa9asCUlYRU5MKQSwb9++UohVUeZFQEpFXnRKwI8Fy9NPP40GDRqgbt26aNu2LSpVqoQLLrgAv/zyC1q2bIn58+eXQEoiM4pvvvkGjRo1QtOmTdG8eXOP4TWZTpkyJTIzplSLgAiIgAjkSoBlP+vM3Azrg/79++OKK67A6NGjcw2npDzmzZuHqlWrokyZMqhQoQJY7xcm7uXLl+Oyyy5Dx44dUaVKFRxzzDGoWLGiuf6///s/bNiwoTDBRtw9119/vck/7bBNfAwmTEpFKT/0f//737jjjjtwyy23gFr33r17MXv2bMTFxeHUU08FRyoyMzP9UsmCZOTIkX7useYwZMgQ/Prrr7j77ruxevVqY7p27Yqff/4Zo0aNQqdOnaIOyddff43jjz8+6vKlDImACIhAsARY9v/55594+OGHsXXrVmNOP/100O2nn37Cyy+/DCoVn3/+uSkv+/Xrh+Tk5GCDD7kclYA//vjDKAAMPD09nVaBzDPPPIMuXbrgs88+AxvT27dvN0rEihUrcP7552PEiBFo3bo1Pv744wKFG67CubVzyO6DDz4AZ3G888474HW45iHW0iWlohSf+Gq3Efz666+jRYsWuPXWW1G2bFmTmlatWuHTTz/FmWeeaa59D7xv8uTJ4BCqr1+MXHuyWb58eXTu3Bnt27f3uNWoUcO40Z2jPh6PKDmhwrRly5YoyY2yIQIiIAIFJxCo7K9Tp44p+3v37m065R588EF89NFH4L8JEybguuuu42mpGHYUMl3NmjUrVPxsRN9+++04cOAAvv/+e3BUwtZvHK1/4IEH8Pzzz5t2wfDhw81Mh0JFFCY35dXOiY+Px2233WYUtJtuugm8DpNkx3wypFSU4iswa9YscBSiWrVqcBwnR0ocxzG97zkcsy9+++237DNZsUaAPTLs7Yq1fCu/kUpA6RaB0iXgParLEQzWuaWbooLHzhEJdjzyTvbecxYDz30NFY169eqZNRacBcGefF+ZSLnOr51DhZEzO5566qlIyVJMpFNKRSk+Zs6rZPQzZ840PSibNm3ipcf06tULffr0QeXKlY0bG5Sc8vTf//7XXO/Zswfbtm0zhmszjKPX4eDBg5g7dy6mTZtmplZ5eZnTQ4cOgT3eHDq16zZ4z8KFC/2GExk33ceNGwfODQ0Unwk0+8Cwly5disWLF2P//v3GdfPmzWaqku9idBbyXGz2119/YdWqVWZI09xQAocdO3aAoz6Mmyy9o2Sed+7caYbMOWxO3pyiZmV4vnv3buPPaWrW3drM78SJE82ULOtmbYbNimK1O1q1YMECMB3Wj3GuW7cOixYtAp+NdeezuvTSS81ift7P9NIwDVZGtgiIgAiIwGEC7Nm3V1yD4Dg5O/CsXzD1EMvdgtSDDJt1B+tM2rwujPnggw88dQTXW+YWBmc7cBoU/Vl3cFSb56wj1q9fb+pj1rV0Y73MdLG+4TXyOLCuYztixowZYD3oLcq6nnUT4wtFO4KM82vnMO1sL3F9yZIlS7yTk+M8mDqYbQ5ySEpKMvdSEWOdzPCNQ4ADGY5z20Jcs8n3a86cOUaRCyAac05SKkrxkR955JFITEw0KeBcSA5hcj3Atddeiy+//BLc2YGNUk6PohALk1NOOcXzo+YCNA730lB5oAwNG+3nnHOOWcR09dVXm6lVtWvXBueU0o8yNG+99RbYq8HpVpwq9N1336F+/fpmwRfD5A+NcixMOGTL+akcXh08eLBZTD5gwACsXbuWIh7DAueqq64yC9EuuugiM8+zZs2aOOGEE9CjRw9wHiwNb+CP95VXXgH9jzrqKDz00ENmDQTXQXComjLFaa655hrUqlULd955J+677z7DomHDhnjhhRdMtKvdBj/TRqWOhovsyN94uocH3aF1jjLRj/e5TuaPw+1cJN6tWzc89thj6N+/v1mgx2dsBNwD5/3ymXAxIfPL+b+us6k4GGeTJk3QoUMHsLeJ7ny+fDacH8xrKhx8RjRctEc3GREQAREQgZwEOMXYutgOOXtNO9h6qCD1IMPliDLrc5bnrIdZ/3GUgY1Y+hfE2MY672F9QTs307hxY4/X1KlTzTnXWdCd04T79u1rpoaxvUEFhPVQmzZtTOejEfY6UFFg3c167uabbzYLxJmf//znP57OwlC3I4Jp57DOYx3LdHNKmVeSzWlB6mC2r7hOhYv5ORrUs2dPnHbaaWbznKOPPtrUySZQ90Blgh29J598spmi/r///c9ssMO6nh2BrkjM/0mpKP5XINcY+EN97rnnzKJsCrFwY+PxjTfeMLs/sWE5e/ZsehnDBbpckGYu3AOveQ8NCyzXCex14DkVBC7WoibNBjoLRCoJRxxxBDgyQlk2/m2hw2su/OKOU1QyGM67775LZ7BQZi8HFZAffvgB/PGcd955oKbOAoo9PEbQPdx44414++23TfrZq8H8DBw40CyeO+uss8AfIQtaVxR33XUXKM/4li1bZmRYeLKH4IwzzjCL1ClXHIbK0JtvvmmCZsOdebG9EywwmQfuvMW8XX755UaOvVzMv7lwD0888QSYdi4St70cVCIoT2WBIzXscWElQgWL83lp3FvNwkEyTcxWKulGwwKbox4s5HhtDa/5nDm8TTdWLLym+fHHH+kkIwIiIAIxTYB1GztvXnzxRbMBCju+2FnVrFkzsD5jHecLKNh6qCD14FdffYUTTzzRbPfKDUM4Gs76gPUAe/1905DfNetFK8POLXseyKaSYN1XrlxpTmmfffbZ5pz1DutgjnKzc4rrM9jjf+yxx4I7ahkh98B6mO0FdoCxLTFp0iRwlIZ1EDvennzySVcKCHU7gu2a/No5HLm55557TPy+h8LWwY8//rjZmWv69Olge4R1MdtP7Ei1cVBmzZo1ZrYG2w8cCbLM2FawcrFsS6ko5afPRiYbtGyc+y424pAiCyYWCDaZjnN46NZxHLMWw3Ec62163HkfG6Fs+FsP7jzBhjoLCjaG6c6hUrrznIbaNzVz/nA4Z3P48OF0Bkcm2CvAHgQ6MJ2chsNzKhgsyHlOwwKKNsOhTWPP2cBmQcACgT0gVKjozwVXdsEZG+MXX3wxkpKS8Nprr9G7WAx7/VmIsnFuezratWsHjh4xQm/lgYvjHMcxI0RU+OhPY6dMsferXLlyRtniTiT04z1UQnjuOI5RoHjOCo8FMxlydKN69ep0zmGobHJL3ByO2ReOk/WsHcfxe/bZIrIKTUA3ioAIRDIB1hvsEOOIOUflWUYzP1xXYesvXltTkHoo2HqQU4g524Bx0O7fvz9PjWEnGjcSMRcFOHgrIpzBkNetjN/6O45jTlm/spOMF+yZf/TRRz2Lm9krz12zeB/dKUNjO8s4uuHNjnW44zhg/c2pP8XRjnCcrHQzHY7j+NV1zA/ra/p7G7ZHClsHcxoX2z6sm2nYhmLYVLxo01DJoDJGpZWKB934brEjMT9lj7KxYKRUhMFTZuOWhR8LRPYKsGeFU1+YNL7AHMrjeTBm7NixRizQD47fwKAnh/nYw81zb2MLv3PPPRdc0GZ/VCxQuPc1G88c8aDiwFEPey+VGHvOqUA855xH2jT23PrRjT9Ouy6Du1j9888/sIa7elDGOw5eF9VQqWGPC8NhQcgRHMbBfcN///13s00fd+igv3eeqHhQ4aI7e2hsoc6eGk6hssoDe6NYMFPOlz+VhISEBHqBcZkTHURABERABApOIJc7OCrOxjCns7Ksp6EoR565bTvPvU1B6qFg60GOeFOxYTyc1ku7qIYjLTYMG7a99rW9e/mpEPj6B7o+7rjjjDNnRrDNwQtOvabNHntbN9PmyDvrcq4R4RRhynibULQjvMMryHlR6mC2w2z9zzipuNC2dTrPKcP6n9OkOPWKnZOXXHKJ2VSHnCgT60ZKRSm+AWzcswC0SeBLPGjQINx///3gwh+OUtCPIxm0gzH8wVOODWXa3oYNaV6zULJyvLaG05vsubfNQmbo0KFm7QPTxN56DgF6y9jze++915xyTQh7DdgTxKlYdOT3OGjTsBeJNg2HEN9//328n23YK8MeHtuQp0woDJUhcrVhMY1UnDhiwEKCChkVutaEfAAAEABJREFUHOvvbdu0c6SHSh6ndVGe80ytnDfTvPhzWpi9R7YIiIAIiEDxELjwwgthR4O9R59tbAWph4KtB70b2raDzMZXWJv1lL2Xu0ba80A2pzJZd66VtOd52VxbSH9O4aFiwXPLhtOlbN1sbc4m4DSoQKMuoWhHMP7CmKLUwZZBXvFypynOLrEdhFwIzu3/uaaCbYK87o0VP1+lIlbyHRb55Lx+ztPksJtvgtgo5f7LdHcch1auhj90hkMB7hhFm70ItL2NdaM2zulM3n48pzttb8NRBhZMH374oZkjyoKGH5uzjWxvWZ6zkOF8zbp164K9H5xyxSlWY8aMgZ0yRTkuWKNNw6lCTL+voSJD/1AY7ijB0RcqEAyPQ7fseWIhxNEhjlhwjmT37t3p7We4yN1Ok+IHiDiFjPlhT4UVtux5bVnznIajMhwq5jkXyNHOy1Cxysvf24+9c3xO3m46FwEREAERgNkkgxzYAGR5z3Nrgq2HWL4GWw/aKbSMoyDlOOVzM+xk45Qc+rMuph3IcD0e13PQj/UM6wae52eoOFgZO6PBsuFoh2/dzOtXX30V3CjE3mftULQjbFjetnc7x9vd+zyUdbB3uPaciukjjzxi1q5ypsNLL71k1lWybueCdSsXy7aUilJ++pzyxAVkgZJBP7pzsRRtGs7dd5wsJcMOy7FnYfXq1fQ2DX+esDecvQ48t4ZTl3jO9RvB9qDwR2SnAnGuor0vt5EKLkrjMCBHAViAc4EZF14NGDCAUXsMC147bYiNeo9H9gkXR3HBdPZlkS0u3mKlwoKWgXG0hTZ3ovJOW275oqxVpLhNLr9oyrUgdLeGu0LYbYIta+vHZ0TFgtcc7aFNYxdqWz+6cfs/O/TMa29je0gOHjxonPmOsHALNJ3NCOggAiIgAjFMgB1dNvtcz8ZzTuflCHqw9VBB6kF2XNnpy6yHGZ81LOdp7HWwNqc/cUMRynPBN9POc1/Dhj4VILpz/R6nKfHc27CDzfua53ZKLjce4QYxdOOuT7T//vtvz05PvKbhFCtO7WIdz+v8TEH4Maz82jmUCWQKUwcHCic3N06r43oK1vOcCsU1MlzA7jiOH6Pcwoh2dykVYfCEOZzGBiunCtnkcCiN06C4aJc9+dadDXGOBPCa80FZgHDrOjaO6caw2KvOngfuakF/FmKcW8o1BZxiZRvUnOrDwoH30fC7Cd7XdPPeSYI7Q9GNDVkuaGJYvOYOUhwa5jlHQDgqwYKJPR4cpWDvPxczca2ILYQ4EsOGOcOg5s+wOWLD+YocMWCeuTCOYeZl2EPANLOHxsrZfHH6FXdm+Ne//mWmlNGfBT5tmy/OwbTfiOA0LDs9ivdSeWB6KE/D3au4NR/POQLju26CDX6mnYoCnx0VKspSmWHhw3OmxVuJIRe628qO5xxitfFykT6fpVUaOPpDGSqRDJdTuli4kSPdZURABMKMgJJTLAQClf0cGWB9wHrPRsrGrz3n2kWec20ce76DrYdsfcF7WVfRzqseZBnOMplbhds6haPXrDd4H+9nme9db9EtL8MRcjt6z1F2rhOxHYuc0sxF1qx3WP9Q1nv7c+9w2XHGbdStcsHe9vHjx4PTozm118pypywu4mYnF7dw5SwF+nGGBacmkx3XCtr6ln40oWhH5NfOIUOmi/GxruQz53lB6mB7n+Vgw2RnLMOzbPku8ZrujIPvDjtC7TXX0PDcKmGUiWUjpaIUn77jOODe0dTi2XPCYUbO/2RPP+focWoNF0Z5F2hMLnsgKMvCgPMAuasDG5b0471UMrhGgA1OXrO3gr3qLIhYkNnG8CeffAI2kjn6QDNs2DCwR4ThWMNGL3s/uCiJDX2miUoLt6fjQmWm7aGHHjJDgLzHKgJsqLNHhfGxl57rRx588EFweJLulKUixMKM93Afb+ad4XEYkV/TPOmkkyiWp2EeOdXqzDPP9MjxWw50o2LDRefeI0F2pIKFJ3fHYmHBPLGH5p133gFHCDjNiYU9GTPvNmAO6958883mkkqgOfE5cNHWuHHjQIWKz4Q7QpApK7v33nsPzJv3LVT8qKyw54nPhe8DFQnuJkE57jxBN/LkNQtzpoFpYbh8zt7rcigjIwIiIALRToCdQOwk4roJ1l/sPWanDstLNoht/p9++mmzxTkbwc8++6zZ4Y91gi03g6mHCloPskHPHn6uheDuhyyrWcfQMK1MGzvVOFLC82CM4zigIsRRBc42YH3M+oV1DetNfvOJdQmn8rJeyC1Mjtzwg3Gs98iK6yCZXq7V8FbAKlasCNZLDzzwgPkeVfPmzc13nZhmTo1iJyXroeJoRzDtebVzWM8yj2TJDjd2ZrJxz/voNy6IOphrIvn+8KvcDIe7TXI7fipoZMMZAHRnxyPbDVQ6mN/+/fvDfiOMm7hwfQk7WfkeMv5YN1IqSvENYK8F90RmbwB/mNTw2ajlFCK+6DxnY943ifxRs7FOGS4c5ncK+LJbOfY4sKeCPe1sjLKHgb3aLMRYiFi5K6+8EhwdoCZOwx4H2tbf2lyQxUKIPS0ssNjzwh53urNRTi2dvQZUgDiHkyMpLPD5Y6Rhzw4LPM5JZXq4sMmGzR8xFQMqVRzFIANOHWIhbmXysvlDZvzBGpt/FpCcosX8clSCPFmAsoBmPtgLxjBZOVEhsGngVDAW6CxUrZuvzefDQo2KCRUo3s+Rp2HDhpmt8bzlmY5vv/0WjI8jPByt4nNibwjjIlf2qLDS4H0cFubUMKaboxUcIVIPCcnIiIAIxBIBdhix/rL1FstZlovsYWYHjmVBZYMdTSyPOarOuoh1IstpKxNMPcT6Lph60IbJ+o51OOtp1imsgzl1hj39LLdZn7LTyMoHa7O8Z13B/LADih1krIfZFuDIPBWXvMLi7AfWdaxz2f7g9rusrzmrwPc+ruN40O0MZJ3MfDD/jIcjMBxNoHyo2xEMk4bPh/Uy42Wc3u0ctilYZ9rnzdkY3unnvfnVwezM5PvDMGhYz7L+ZUcsRy/se0Wbfuz05OwQpoPPkG0hPgdy5MgP0ywDSKkoxbeAL6n3Qif+SPnD4BoK9hLklzQOr7KnIi85FqiMJy+ZYP2YJo6M5CbPoWX+SDkiwp4P9nrQDBkyBGycc3iW97IgpO1r2HCmQuTrXpzXVMY4qsHGum88rKQ42sFCmgUbKwU26G0Pl698oGuGzYI5kJ+3G4dt2dti08FnxneD70QgJo7jgLtsMP3e4ehcBERABETAnwDrS07F5Wg5y1t/iSyX/Oqh/OrBrFAOH1n+sx6wLqxDaRgO/ax7YWzWGRxZZ0O4oPezXvFOV373s24qTDy+4TLfzL+ve27XfG75tXNyu5fuzGNROTMcGtbLtGlY/7KOVh1MGoeNlIrDLHRWRAKcgsQfHYeYOfrA3gMGyR5/LmbiVB8WTHZqD/3C2bB3gooFe4GoCDHd559/PjgFKZzTrbSJQKEJ6EYREIGoI8AF4xydYcY4MsEed/bO81pGBEJJQEpFKGnGeFjs0ec0Hw6HclSCU3s4Z5EaPecgcsoQ55kGO7WptHFSQeIwL6dMcd4tR5F4XdrpUvwiIAIiIAKxTaAguec6vJkzZ4J1GHvX2UHG6dEFCUOyIhAMASkVwVCSTNAEuKCJ33DgHEyuVeB8Q64N4FxMLqLr2bNn0GGFgyDXQXDtAvPBqU9cuBUO6VIaREAEREAERCAYApyazPUJ3JGQC5q51o/rSIK5VzIiUBACUir8aMlBBERABERABERABERABESgIASkVBSElmRFQATCh4BSIgIiIAIiIAIiEDYEwlap4HZrXCAbNqSUEBEQgVIjwKl0XPBfaglQxCIQxQS4lWZxZk9hRxYBlrWc9htZqVZqC0qA2/KGup0dlkoFv1XArd/4cZX8IHGHIX4hkntTe8vymrsceLuV1Dl/kHxQnMPID6mUVLylFQ/3ceZzKK34wzle7h4VzunLL218rtwFKz+54va/8cYbwW+ScP/w4o5L4YtALBHgWjFuRBEOv/No5h5JdQG/4cRvZLEdFanPhHVXuO5wlVubqaTfEW6s06FDB3ADneznXGQr7JQKfufgnHPOAb9vwEWygXLID9hwa0/C4HcYuMtQ5cqVwa8bfvDBB+A3Bbjt5yOPPBLo9mJzoyJx9dVXg3tHd+nSBf/+97/BPaT5ERwujiq2iEshYH74hXtHlylTBnwGTz/9dCmkIu8ouQsVvzPB96NFixag3bRpU5x44onmxhEjRoA7O9HdGvrT2AqWH+vj82zWrJm5v5lr9+/f39yf34Hf5uB3JmjnJ1sc/meffTb4oaSCmA8//NCTFD5j8uOe4txS1+NRCif8mBU/IMWPLlJpL4UkKEoRiDoCv/76Kx544AF8/fXX4Hbf/Hga9/XPzbCcZPnH3fxGjx4ddTyKK0OsA0qzLihovl577TX06dMHeX3Pw4Z59NFHI7f3hTs/duvWDdxu/rHHHkNJdLKedNJJSExMBL/DQcXIprO0bdanebWZSuMdYfuU391ip8KsWbNCgijslIobbrgB69evxxtvvOGXQTbwWJhR4WBheN5554H7L1O745eH+dLyGwn8xPrYsWNRkr2aHF3hl6Tfeust3H777eAXIP/44w/wE+7cceG6667zy08kO3BrOuaPH7JhPoqbNbejDWbkimmx5n//+x/45U1e82vdHM7lzlT8oirdqLwyTCqn9KMZPnw4+HVRFkiU4cd+fvvtN/BrqnymL730EqiM0C8vQx5UcDMyMvDOO+8gPT3dT7wwefILJBcHxv/DDz/gzz//BD/mxEKDSjqHO+lGw/gvvfRSHHnkkeCH/ehGhd0GSXbMM5X0zz77zDqXis3tfb/44gtQ6QlHBbZUoChSESgCAW4petFFF+Gpp54CO74YFMsJlgMPP/ww2ElGc/rpp5ty5KeffgK31KZS8fnnn4Nbg/fr1w+cqsx7i9uwvGJ5XdzxFDb83NLHsji/uqCwcRbHfZxhMWHCBFx77bVBBc+6ge8M2118X2hYZ9KNfv/5z3+wa9cu3HPPPWAnFd2CCriQQtxl8sILLzR3k705CYNDXm0mprO03pFbbrnFKJD87Ydidk9YKRWEyhfi1ltvNb3f3u8BeyfZQGePJRsY/JjaQw89ZApDbvNJw8KPP4Zmbm+y970lcc4eHjbMqBnzITmOY6K1H5wpqYLXRFoCB35FklpuSbBmY3/y5MngkGFBssZeElaWbEzb+6iE8hnxmo1t+nsXnhwGpJvjZD0/ynHUiY1r9rbwHWvXrh2d8zT8gudtt90GKl033XQTeO19Q2Hz5B1GXudUwKnQUMn+9NNPwYL9kksuMaMt9j6O2AwdOhRUvvgxJOabFYL1px9Hbfj9EY56WPfSsqn8DB482KSX2xSXVjoUb3QTiIXccWoIy7JKlSqBo382z6xHWQ60b9/eOoHfNaAby/tTTz0VDz74ID766CPjz/q2JPfp7uYAABAASURBVDrMiru8NJkpwiGv9LHsz6suKEK0xXIr22B85meddVZQ4bMNwPejRo0aHnnOGqHbgAEDwHqEnbzsdGUHMK8XLVrkkQ31CRUXKnihDreo4eXVZirtd+T+++8H26/8vliR81nUAEJ5P1/matWqwbsRaMNnD8nvv/9uLjkSwIaeufA58H72DPs4F/ulXejmmy4+JPaOf/zxx8WehmiNgL0eRckbCzHHyVIS+BxYoXqH17p1a88lh/S5KNjj4J5wSt4nn3yC4cOHu1fB/7HyZS8/ewJ97ypqnnzD873etm2bmc5w1VVX+XoFvKbyw98d77MC7HliZbls2TJ4NzKsf2nYXFtBplSESiN+xSkC0UCAjTz+rjnlwo7KFiRfHKWw8hzBYKefvS4Ou7jLy6KmOb/05VUXFDXuUN7PjjuOBrOu43S4UIXNBvUAV8FgeKxP8+NFuSBMVImU5jvCDrtevXrhxRdfhO0ILyzcsBmp4MfF2BvN4VTO0ffNEKc20Y1+nCLF89wMAXEOfW7+1MgmTpwINph8ZTgMxalMnC7DOXBJSUlGhL2+XBfBHmvj4HXgPEE7bMS5+GyY0bDQZr6OPfZYcHqW1y2eU/7A+CEaDkXbRUVMH6d38eHa9DCtjJ890PZmxssGMLV+q9TQj3FZd9p045QyhstzX8MKYc2aNeAHcphv5tVXxl5zxIVcaFu3wtjMF4dZx40bB4bn29BnmJThcPd///tfXoKMyZUmkLwRCnBo1qwZbIHGHm6yhdc/Kgz2knnniJm9ps1pQJxbyrUJvA6GL58l35Xly5eDH/7jfTQFzRN7dvghQY6wMP8MIxjDEQeOMnCtUTDylGHvEu/jOTnwXeN7wffO/g6Y/kC/D74PXPRFed5vDd9vvp98r3ivdfe1g80nnwHny3711Vfge+sbjq5FQATyJ8A1FJTi2kPaBTWcRmnv4VoBx8nqtLFu1uZvlGVCXnULw+K0S9YFXLvF8pXlJsNgmVGQOoALi1kOscxkmcIwvA3LI9YBrC85LZl+vId1EePitTW8pjvTVZQ6Kre6wMZjbdbPoWyX2HALYnOKKct6rg0tyH3ByPI5WzmuLbDnvjafG59fXnUe3yk+F46wM9w5c+YgrzaBfQ9ZF1Fx8o2T18E874K8PwyThnUj3x/avA5kcntHeA/bkGxDMO32XraBeJ1bXijHtObVtqSMtznttNPA38L333/v7Vzg87BRKr799lvTSODiIN9csHHNgoDubChxyJbneRnOe+eaC28ZDtly0W23bt1AJYVzQ/lyU9bKcR4gp8VQKeGoA3uuOR2LX4ImdN7PhUlscPEevoiUt2tAfvzxRzNczOFDTrPhfcwTFQvKW8MGInuRqSRxXisXnnP+PhtN/NIl57fR2PRwgRwbfRyxYRiMn/Ic6uOaAC4KpzsNe96t++OPP44LLrjALEhmzzPjpAwNG45cX8BwuE6F08kYR6dOncBhbcpYw/UTHL6kLAscppFDjGwsWplgbRYYzdyGPvlz5xFOaeEzZcN/7dq1nmCY7lNOOcUoE3TkehpypWHFQbdgDXterOz7779vT0EG7Jlh3q0j/VkIeV9ffvnlYG8L3YLhe9lll5kF+3wHOG2A99EEmydWenwXOPJ28803g+GRPacxsQBiWHkZKtackpeXjK8fnyd/I3SnIsEF2nxOfCc47ZDu9n20vw++F1xvwt8Kh8sp37dvX7NZAt8tTp268MILQXkaNh4YjjUFzSd7VfnusTzgwm0bjmwREIHgCLDO+u6778AOB05RCe6unFKvv/66x8F2+ngc3BOWq/z9s8zKq25hfcZpqlwnxXKXdR4X/dpNVoItL9lG4Bo5Kjisn1hns15mJyX93CSZP655ZHwsl5h3cmC9yPnurFfY+KNgKOsolt1sN/jWBYyHhmUu/UPZLmG4hTFsx7DeYVldmPtzu4cdbLax2rZtW5xxxhnw/RdMXUBlgu0pKsOc1ssRa74vZGc7UH3DpaLG53vuueeC9RTrNb4H3nLBPm/eF8z7w7BZNwbbZsrtHeGzaNSoETjlmr8FKk9sf3IaYm55CbZtyTR6G/5Oec02LO3CmrBRKuwLR1C+mfFeld6wYUNf74DXfBiEbz2pRLBhyMY5tTf2frBBzAY954TSUJbDuuzVZ28or9ko5w5H06dPB0ceWEhSO2ZjmP6cC8dCmnMmec0GFRuk1nBKF919DadxvP3226bBT42cjeSBAweaBXFsnPHHwhfSNz02HKaDi58C8WJvj50DzwV11FjtlDDavI/h3HXXXWA6WMAyb2wwsveGBQB/9FaOvcKcf88f9KhRo8ARJTIkO/YqMKyCGFZIZFyvXj1wMTELAyqA49xRC8uP4bE3zfac22tWVjRsWNItWEMerHAoz+F69rbzfMyYMWatBllQAaQbCzf2rPGc8fNdGTZsGC+NCYYvRzu4MM3c4HUIJk/kf8QRR5h3gdO1uH6IPWZsvL/wwgt48sknvUIMfEpllZVmYN/ArlTsLFcWwnwulpm9w/d9pDLGxgnfHxpWnEwvC32OCPG9ZkHIBeFUGDl10YZV2HyyAmEYUioAcpARgYIQ4Lox9nTyN876K797Z86caTan4NSIO+64w4z68nfPDoR3330XnELlGwbL0/zqFv7+2RnG8ox1DJUKNupZb7P+ZJjBlJccsWe5RQWB5SXrZ3aKsaHI8FiWMg8Mj51q3h0bTDtHolkHMhzmh3KhrKNyqwsYT3G1Sxh2Qc3s2bNBdt5rDAsahpWncsLOWk7/5podtlNY57IeHj9+PNh+sbK0+S7wObENwmfIOiRQncf2GNshfK5sW3FTFdt5Zt8ZhmcN4+TMFsrwfaDyyk45rnPkKICVC/Z5B/v+8H0uSJspt3fEdmoznRzJYhuA7yjrvtzywt9dMG1LhultQlWvho1SsXr1apM/78U+xsE98CVwLfNnG/vmIsgDG0fczYLibNTYhpLjOGDhR3f+APgSs5Cl4sLF4HRng5wvMt1p+OOgO4eeaFvjOIGHf6kVWxlv2ypRLECtuz1nI5YNUr5ojNM7PVaWNnuxOXLDc2/DxmHLli2NExUDNkS5yJ2a+r333gvmjQ3n5557zshQIeI9vKDSRVkOgXJbOQ6H2UKGdv/+/SlmDF/eQM/LeOZx4MgEe2aoeVOMeWSjk+d8VrYC4LXjHObqOA4cJ8vQryCGjWwbH5+pnfLEiowLmNkD7j2awR8uw6ccRxo4OsFrGrLKjy9l2LtAeV/jOI7HyXEcvzxxizfy5zoGm2bewHfCcRzwuXHIl27FZTgqwx4S9vZ5x8Fn5f0+8pprhmjzHvZCUZ4FOEeAOBLIa46m0eZvjDZNYfNpf1OsiBiOjAiIQPAE8qprA4XCsogNbvaAstffdriwg8G7fLL3Blu3sGHEhiB7nNmryw4qlsMc+WCZa8NzHMeeespKxznsdt9994Ejl6yb2Tllhak0sXOM5QTLKLozfLrznIadH6x3Wcez99fWAaGso7LrAkaXw7CuK+52SY4I87mgIsB6mTMy8hHN15vvCw2nsrETko17thWuueYas/2sbwDB1gVUeqgQU8FlJxbD4XvIDmNb19DNGk6JYpuHdSndqHzSZv3JTmWe0wT7vIN5fwrTZsrtHWG7hR11TCPzzZkK+eUl2LYlw/Q2fD6O44C/d/Lx9ivIedgoFZzOw4SzwUvb27Cha6/5otrzvGw+WBaClOFLzWue+zb02CjnfHn62YXgPLeG05bYWLLXfPg8t+HxvDCGvbu8z1thsufWj/5FNRzG5FQoKmPUnm0hxh8nf3AMn/PyWMBbwx1A6M7eAE6Dsczt8Bj9imJYEXGeICsPNj75I2BcNkxWEPY8lLatMBgmlQm+H5x2Z0chqEzZd4E9ZOzJ4LQf7/t4r7fJja+3TEHPOVzLe9ibY58JbRaCfDeYLtswoFxpGv4+HOdwBc9Cl+nhlCmmn+c01t37d1PYfFKZZpgsZGnLiIAIBE8gr7o2UCgcQWejj4139iLTUI69odzpkOfeJti6hQtDWd5yVIG9uvxdU5mgUsIOLO8w8zrnonP6+9btdLOdHOzx5Qg33byN7SRjhxtHsKmY0L8k6qhwapewTmEHGkcVODODDIpi2FnLd4ablLCTkLMuOOpPJS9Q3RVsXcD6hutWOb2NjW1OXWOnIOPyrm+8026fMd1s+43n3nVRYZ+3Ddv7/SmONhPTS2Pj43lueWEbgf62Pel9bv3o5mvYMWj9i1K3ho1SYXcaCPTDZy8nCxxCYOOT0414bkwuB2rEzzzzjPFlY8ycuIdAPxjb4OE0JFckx5/tFc3hGIILas8M5ssvvwR7LFiQcviWbhxiph0KwylGgcJhj5N15xAiG9nWsMeIhTp7cbwLAKts2PsKa/OF5Y5MLARYmbCHhEOahQ0v2PvYI2UrGQ71suBjRcQCj2FQoWWhx3MyuPPOO8GpTlzvQrdAJje+gWSDdbPPhlPE7DOxNhUfDoGyVyHY8IpTLrffR27u7Jm06SlsPq0yHKissGHLFgERCEwgr7o28B05XblGimUlXTl9lba3sb9ruuVVt3DEkw15dkBQlr9rdjJxugp7nukWjLH1e151OzvGrJx3mLmV3yVRR3mnJ6+0l0S7hKNFbIRyeo83n1Ccs0PTdsyxBzzQzk/2ncmvzqOSwqnqVEaZNk4JYto5dYdTbenma3Kri7zlCvu8A70/xdFmsmkNJi9FaVvyN8i4ilK3ho1SQY2TmeHQC21vwyEgfh2ZbtxOktMqeJ6bYcOFc/85R48y7BGhTUONnLY1hMgXndec7kG7JAwbhWzMcpHRcccdZxYucViWc/ztVKBg0sHGb15y3qMs3nJclGuvOSWMcwp9DRv+XPBr5fKLy8rlZbPg4lx/PkMqFCxMOPe+IIoUG9hMa17x5OZnCzf6c3qX9zXduBicNs2rr74KLv6z2jvdfE1ufH3l8rv2zpN9NhzmZD59DdPFRYX5hRlu/r7pKWw+OaWPYQVTwFJORgRE4DCBvOraw1J5n9mpJmzUeTeOeZf9XfM8r7qFjS/2OrOjkLs9cSSd9bzjOOAH0jhlhmEEMt7lpa3ffet23mfdWE5zag/dvA3dva95XlJ1lE0347Tp5DlNSbdLOP2bi+QbN27M6ENubKcwA+YIDW1vY9+Z/Oo8KrFcxE8lkSNcL730Ergehm04LqL2DjPY86I870DvT6jbTMHmw8oVtm3J3yJZMJyi1K1ho1TYHzwLKWbK1/DjHGyM0p3TZjjExPNA5qGHHjI7WwwaNMh4c7cAKia8YE8IbWvYY80fMK/ZyKVdEoZKD3vqOVLBQpkLpDnlhjsgBYqf2j7dbVp5zvmKdtiQ1wUxfPHt2hIujvK9lwvROX+PvUkcKaK/b48J00JDv2ANCwU7vYlTsezoR24jFexVc5ys6TV2uJLPjBVSsHF6y3GXBQ7z0Y1hs+ef59Z4uVmNAAAQAElEQVRwgb+3cumrdFi5otiM13FyzxPTwPC5oNL+yHlNwyFkTkPjO8PrSDaFzadtbHCUKZLzr7SLQGkQyK+uBZBvsthwsUJ2nRSntHK0Pdi6hYtxudkHOxK5Ro1TSNgw5Egsw+a0XNr5lZe23mb9xA5F3mONre+5AYita6xfbnZJ1VHh0i7hwnVuhsPZHbkxKap7oPeFYVKRZHsg2LqAU/C4noLtOU6F4rpOvkeO48C3rmT4wZiCPu/8wgx1mym/+Hz9C9q2tPfbepXtH363yroX1A4bpcLOFeMuS4Eywd5i7tJAjZZDVd27dwdfMA6XWXlOIeJ2ctymjqv9LRgOlXGnADbMqZywAc97qMDwpeQ55xKyQU9tjQ03O8WKBR4b7yys6G4btnyBeU13Pgzbi093po9h8j7ez3OGR3nbCGfBzlEJrnfglByOUjBPXHREpci30Uh3hmMLcJ5zKJDp5Tmn6ZAFh62808OCmfHa9FGWhsOt7A3ivDxq/mTLBcwMj6zYw8TFS5RlPJQjV+7kQzf2rHCqkM0f02V7kOmfm/Fe+Ms4KccwuFCOcfCahZxlSMWHIzp051xd8uWiPvaq0K2ghtxtvrggjYqddxjsebBD782bNwe3JPT253kwfJknPn/KkymfAc9p8ssTdyRh2ng/lSCO5vA+7p7EKWl8dlwLRLdgDNeOMH4a9ujYe8iYbjR89tadNq/pzveW13y2fIdsXqy7zSd/B5S34fN3wmu+j+xV4ggjw+H7T3f6FzafLCOoGPL9Y5gyIiACwRNgPcMyiPUgf4fed/L3y98nf+/Wnb97urHstW7s2LDnduE2NwThCALLp2DrFpYLbMyyrGN4TA93FmSdaHvNmda86gBOh2E5zfqPU1qZTpYzXPvBTU9Yr3CKLcO39SHPaViWM288tybUdZQtIxm+LT95XtztEsYRjCEb1iesW4KR95XhGh0yZN6sH69pbD3BDmHbHmNbgffQsK5l/VaQuoDvGTtCWecwPnYw89wqJnx3WefRj4ZKC98HvmuMk240TB/dCvK8g3l/GHZB20y5vSP554WxATYvvGIbpyBtS95Dw3qVNmdn0C6sCRul4vTTTzd54IJUcxLgwEKGBRgXiHF+PLdjoxsVDmrCXOTFRiELJb7E3kFwMQ+3LGVhRQ2Xw7fcEo8FJhfjsoeE8lz0RU2TLxt7NjhFhjsGsGHEYWMOudGdQ3jU6Pgy0J+7BdGdBTXlqBRwfiLXJtCdvS0MlyMTjMc2bPmSM738obEHngvKHnzwQXBolO6UpWFhya1m2RPEHlrmm4oEf5T0525UdGN4TA+3jmW8DJfuVAgo523YMOf2bkwLlTHOk+UPjCw47/Gkk04y4vxWBHvNuYiN3MmNQ440jINCrETYQ8XzvAyVI07f4SIrKi5kxQqBW81xa0HGT6WKYdtwODTLYVEOdXJYjr1afIbWv6C2neJkbd/7OTrB94j+jpM1ouAtEwxfvm9cTEY+bFjzh87Cz4aTV55Y+PI5P/DAA6AiQeWG+SZfKtWsKJk+G1Z+Nitdxs/3gHOc2cvDdHFxGt9hzgvlYjrvcNhzxXtYIFOWSh+55Pb7YEXO8FmYUZ6/Y77vfB/Zk8hnS3fHcUB3KoaFySeVWc6dZScEmXinWeciIAL5E2BdxLKdHQe+v3uWD/x9ct0Ef68sK9jJxN82G342dG5nyfKDCsSzzz4Llk2sA7lgljLB1C0sw1j+s6HG6Zyss1keUbGwHU4Miyav8pIdQyxPWN5y0w9es03AXQ2ZRtaBrDMZDhcjcyc/5o2Gm3SwPqOfNaGuo/KqC+hXXO0Sm5+8bLZfuO082yp8HnnJ5ubHspido3yX+L6QK+tvvkfceYv3kTHZsy5jXcGp6axH2TFEJTfYuoBpZHysm/nOcD0OZxuwfuI7y7jYluC6AqaD7zrfKa674O6J3OGL7jRsT7GtVZDnzTzk9/4wDQVtM/E94PvLdHm3FwqaF8bN9hxt1r1sQ/L9z6ttSVkatvFoc8SQdmFN2CgVbDTyReMPjA2Z3DLEl4qjCmwMUxvltmIsUNiYJ0QWPixUAt3Pgo/hU/sjbCoUbKCwYHGcrMYjHwgLW774NNS+OQLCMKl1swCkO236sSHOhh+v6c4CkVoxfzxUIHht3RkufwBscHFHDfa+sBCngkLDgpQfDGLhyrzwh2DzwfC4UxF7kthwY5rYkKfWTl7s6WF62ChneijHeGmYBi46tmF52/xhsyDm0DUVIfbccMiYPzRvOaaJU62obDH9HOXhMCTjYo83G3tUbLzvye2cw9v8QiTv4Xxajn7wmdKdGjd7HZgfez+fG38YjJuFFD/OwvfA+hfU5g+V7w0rvkD3skDkHN/c1nkwz/nx5bO0MuTPd4KjUTa+/PLEnngWeHwWzDd5Me9UDtlzZ8MJxmZByPiZDr4PfO9p85rvDAsxpsc7LCrovIdyNHx3+T7n9vtgpcDw7O+ANu/h+8geEMvChsXCnfEVNJ989vwdFrXgY9wyIhCrBGxn1BdffJEDAX9X/N3y98vfqndZwc43K8zGIxujrEs5ssx6iz3O3uVIfnUL42I5zN80w+HaOk5rZd3OjiMbF22Gm1cdwM1WuDHL4sWLwbqTaWEdxTqSChHDoOGaDeaPeaOx+aSft2FdxDI3FHVUMHVBcbRLvPOT2zk/vEcerH9zk8nPnXW4Ld/t+0KuDJcNens/FQi2W9jOYJ3EDlq+Q9Y/mLqAsxX4vrAOZrxsC7Ht4t2+YceXTQ/rJKaD7zs7JXnN507DcyrEjD/Y5x3s+8MwC9Jmyu0dKWhe2DYraNuSaWWbi+1AKvnsuKRbYU3YKBXMAPffZyOHPSO8zs+wIcMGIIdi2auan7y3PxdI8yX2diupc4628EVnLwo1Wqafho1c9gbxgzFMC0ciaHsbDpkyr9TA6U6lhho7G5osWOlWWMMGYH5hkBnZ2TjYW0zDZ0E/6x6MzXt4bzCylOEwNkeYeF4U4ziOWdyVVxhU4izjvOSK6hdMnpgOKrVFjSvc7w8mn6wE2CvG306450fpKxECiqQQBFjvcM0hFQU2nAsRhLmF5Rd7mjmyzrrJOAY4BKpbWNewDqA4lRQ2QO013QIZxpdfHcCwWC8Gur+gbkxPSdZRrFsLWo8WNE/e8pz6xFkiHK32di+uc8dxwFELNlw55Sq3eHKrC7yfK9PMtk9ROhh94y/o8/a9P9A1nyefq/Xj+0TDuOhn3UNhF7Ztyc5tKtEcLSkqz7BSKjilhcNA7AGn9hkKyOEYxplnnmk+QMcGErVD9ggzndQWueiIU534o6J2TXcZERABgOuk2BPJhXXelYvYiIAIFJwAN+Ng55btxCp4CLojkglw5gGn5HKKdsnkQ7EUN4HCtC3Zkc9ZEWx7U6koahrDSqlgZrgWgHsOc36cbWzTPZoMR1c47YpDaSzQ2SvO+YfUvDlXkFo857f5TkGKJgbKiwgUhACno/H3wq2IWT4U5F7JioAI+BPg9Awq6lQuuIbOX0Iu0UyAoxRsi3DEKprzGUt54/MsaNuSu3xyxIRrlUPBKuyUCg6hsieSNtdHhCKT4RgGh3qfeOIJcM48P37HkRmujeAQFKd/9ezZM+hkS1AEop0Ad2njItCi7kwR7ZyUPxEoCAHu+MO1e5xTzrVKBblXspFNgOtX2KB0nKz1pJGdG6XeEihI25JrebiTFmfMcHdUG0ZR7LBTKpgZzjVjRvllTV7LiIAIxDYBLuLkbmERTkHJF4GwI8CNK7h+j72VYZc4JajYCHD+PZXJYotAAYc9Ae6kxY5srtMNVWLDUqkIVeYUjgiIgAiIgAiIgAgUjICkRUAECkNASkVhqOkeERABERABERABERABERABD4ESVyo8MetEBERABERABERABERABEQgKghIqYiKx6hMiEDICShAERABERABERABEQiagJSKoFFJUAREQAREQATCjYDSIwIiIALhQUBKRXg8B6VCBERABERABERABEQgWgnEQL6kVMTAQ1YWRUAERCDWCGSmpyMzMzPWsq38ioAIiECpEZBSUWroFXEICSgoEYhaAnOuvBIzL7oYsy69FLMuuwy0Z158MWacfx6SZswo1XyveOYZTB44EH/17o3ZQ4eGJC1FDTN56VL8M+Rk/FajOkY3aYrdM2eGJF0KRAREQAREIG8CUiry5iNfERABEShVArUHD0btk07C5h++x4aPP8aWX35C7cEnoc4ppyGhdu1STVvVHj3glC2D3dOmIXnJ4iDSkr9IUcI8tHs3/jnxROyePgMDFi9Fh2eeBpWy/GOVhAiIgAiIQFEJSKkoKkHdLwIiIALFSKDBeeeh8dDLEVc2wcRSrkYtc0238k2aGLfSOtRyRykanH9BSKMvSpjb//gDKWvWoHqfPkisXw+VOnZE8sJFIU2fAhOBmCCgTIpAIQhIqSgENN0iAiIgAiIQfgT2uwoFU5XmjljQXv/Bh6h14gk8lREBERABEShmAlIqihlwgODlJAIiIAIlQiB1105sG/0nVr/2KjZ+9RWSlyzxizdt715s/vZbrHzxRax95x0kTZuG9JQUPznrwIb7ll9+wTq3wZ7fmo6DW7dh66+/YtXLL5v4U9autcHksAsSZo4bfS4S69c3LnvmzsW69z8w08U6v/KKcePhwOYthseOCROwfexYk1e6y4iACIiACBSdgJSKojNUCCIgAlFJILIztWbECIxp1gILb7kNB7dsNY37cR06Yvbw4Ug/eNBk7tCePRjbpi2mn3MOMg4cwI7x4zF54AD80aAeto4aZWTsIXXXLnCh+JgWzbHi6aexZ85szP3XlVh4221WJIdNmdHNm2Lh7bchdds2rHYVC8ZF5cUKFjRMe19udtVevYzXoR07sOA/N+OoP/+A9xSxnX9NwMzzLsDkfv0w65JLsPL55428DiIgAiIgAkUnIKWi6AwVggiIgAiEFYGtP/+Medddh3J1aqPP35PQ9qGH0O3dd9H+icex/v33MfWUU0x69y9fjoObN5nzhhdfjO4ffoi2/3sUaUl7MOuii5GZkWH8eJhzxRWm57+h636028vf8bnncOzMGag1aBC9c5gNn3+ORXfcgbJVq7vxT0bbhx8G76nSrYvrfhv2rVhh5AsSprkhn0Ol1q2RmL3OpGLLlqjUvn2OO7g+pf5556LmgAEYtGYNenz6aQ5/XUQoASVbBEQgLAhIqQiLx6BEiIAIiEDRCHBqE3c/YihLH32Mlmnwl6lY0ZzzUPeMM2hhx+jRSNmwEdxpqcMLL6DLO++ifOPGxq/2CScY+9CunZ7pUnsXLsSW77837o2vvBKO45hzJy4OVTp3NufeB45S8LrOkJNRtkoVnsKJj0f9885HZmoaNn72GQoapgkkj0NGaipmXX65qxDtNlK7Z8zAqhdfNOf2wKlYG7/8HN3efx9xZctaZ9kiIAIiIAIhIJCfUhGCKBSECIiAAbtfRAAAEABJREFUCIhAcRP456STsHvWLPCDb3vnzzXRla1R3dj2ULb64esdY0Yb53quorF3wQJM6tsXfzRoYL5/YTzcQ2ZamnuECdecuIdEV8a1cv3j6EbygoXGf/OPP2Li0cd4zOrs9Q2pu5IKFKYJLI9DWnIyppx4IjZ88jGOnTUTNfr1M9KL773HE8++lSsxe9gwdHrltRxTooygDiIgAiIgAkUmIKWiyAgVgAiEAwGlIZYJcDpRyurVZrTBcRyUrVrN4Mg8lG5se8hMO3zNc65vGNuqNda9/TYaDR2KgW7D+8hRf1hxj51Qt47n3NVaDp8HOOPoRWLDBsan4QUX4OixYzym/6JFODklxXw/oiBhmsDyOKx7913sHD/eVRaaomKLFujq5qdszZrI2J9ipnpR2Zp68smof845aHTJJXmEJC8REAEREIHCEpBSUVhyuk8EREAEwoAARyaWPvSQSUlio0bGrpU9hSnZ54N0nus4B1V798LSB+9HZnoaml5/HZpedRXiExNxaOcOE4b3oUafvohLSDRO+5cuNbY9MH57bu1a2fHvnj3bhBnvhmsN13ts+PRTFDRMG3Ygmzs50T1t9x5X58lExVatcNTo0ShbuzYObtqEv3r0QIWWLdDxpZcoZkzy4sXYNXmyOS/RgyITAREQgSglIKUiSh+ssiUCIhAdBLhOgrskZWZmLZrOzMgAr82OTj/9hFkXX4INH32EcnXqID4h6wN5LW+7DYmNGmPbqF89uzgxnKUPPmigNLv+BlTp2BFlKlY21/uXr3CVi3QwbO91CJwWxe1l48uXR+v77jWyy5580rPl7M6JE7Hm9RHG/YDbeE9Zv96ct7rzTlRs0wa7XP9VbkOeu01lHDoEbms769LLwGlYBQ3TBJzLofaJJxofrgNZ++ab5jyhdh3UHnS8Oedh94yZ2DZyJDLS0pDqKk6zr7jC7IhFPxkREAER8CWg64ITkFJRcGa6QwREQARKjMD4Lp3xe40aSEtKMnFymhOv/6hXF1NPPx0bP//MuNuF1ryo7CoMfadORd3TzsCc4Vfgt6rV8EfdekjduhVd3noLHbMXMPf86ivU7N8fm775Gn/Ur4+JRx6JxIYN0enVV8DpQ7MuuggzXcMwW99zD7p9/DEOJSXhdzfuP91RkblXX+0qD63pjQNr12F048ZI27cPFZo1Q59Jk9Bo+HCsfu11jKpRHb/Xqo1ljzyCnm566w4ZYu4pSJjmhlwOza67Du0ef9xVVmpg3rXXuvHVxJgWLVzlYTuOHDUK7Z95BmVdhtPOOhujqlfD2FZtUKF5c7S8445cQpSzCIiACIhAQQlIqSgoMcmHgICCEAERCJbA8WvW4tTMzHzNsdOn5wgysX49HPH11zhh4wYMXL0SJ+3dg/4LF6KJ1+5N1Y8+GkePHYshBw6i3/wFOHbaNLP9bLP/ux6Dt283cfb6/ntPuFyP0H/+fJy4eQv6L15swjvS7f0/wVVWTtqzB0Pc0Qi721S5WrXAbWwHLF6EEzZtxglbt6Df3Lmoe+aZnvB4UpAwKZ+b4ejIidu2unldDW6jy/weNep3cBSj5a23YgDXcyTvRd9/puKEzZvQ45NPUKZSpdyCk7sIiIAIiEABCcQVUF7iIiACIiACEUagXPUaeW6hyu1VE+rU9s9VLi6cumQb5GUqV0ZC7dqgHVemTMA7uK2snZoVUMB1LGiY7i1+f9y2tkLTpqjUrl3A/DKOyh06IK5cOb975SACIiACIlA0AlIqisZPd4uACIiACIiACIhAqRJQ5CIQDgSkVITDU1AaREAEREAEREAEREAERCCCCUipyPfhSUAEREAERCBWCWz55RfM+7//A7/psXXUKOxfvRqZmZmxikP5FgEREIFcCUipyBWNPERABCKKgBIrAsVAIH3fPuydPx/LH30UU086CWOaN8evlSpifNdumHH++Vhy//1Y/8knSJo+HWnJycWQAgUpAiIgApFBQEpFZDwnpVIEREAERKAUCDRwFYdjJkzAiVu34sSdO9Fn8mR0fvV11DnlZGSmpWHT119jzhVXYGKvXvitcmX80aAhJg8ceHh047ffskY3MjKgf1kEdBQBEYhOAlIqovO5KlciIAIiIAIhJlCuenVUP+ooNB42FO0fexxHfPstuE3vkP37MWDZMvT+6Se0uPUWVGzVKmt047HHMPXkk7NGNypX8oxuLL7vvsOjG3v3hjiVCk4EREAEQkKgwIFIqSgwMt0gAiIgAiIgAocJcCtbKhJ1Tj0V/CZGlzffhBnd2LLl8OjGayM8oxubv/v28OhGlSqe0Y25112XtXaDoxurViFToxuHIetMBEQg7AlIqQj7R6QERiUBZUoERCAmCHhGN4Zefnh0Y/4C2NGNXj//7BndSF60EMvt6EaLFvi1QkWM79IFM84/D2Z04+OPs9ZuaHQjJt4dZVIEIo2AlIpIe2JKrwiIQMwQ2Pb775h2xhm5mulnn41F//0vNnz6KVI2bAzIZc/8+WaO/6zLL0ek7Vq04plnTNr/6t0bs4cODZi/gjoWNMyChh+svB3dqHvKKYdHN8aNx4nu6MbgXbvQZ8oUdH7jDdQ97VRkpqXDjG78619ZazfM6EYDw8aMbrzwArZqdCNY9JITAREoJgJSKooJrIIVAREQgaISqNCqFeqdfQ7i3R7rLT/+CJoKLVsat3pnnoXKnTph96zZmHXJJRjrui996CE/xWHViy9ix9ix2PDRR8YuappK8v6qPXrAKVsGu6dNQ/KSxSGJujjCDEnCvAIpW60aqh95JBq7oxvtHn0sa+2Gz+hGy9tuM2s3zOjG409krd3IbXTD5Zem0Q0vwjqNQgLKUhgQkFIRBg9BSRABERCBQAQquo1ENiyrH32Ux7vuaaeZxmbj4cPQ9uGHcdTvo9D1vfeBeAdLH3wQC2+73SPLk/qnn04LZavXcJWQzuY8Ug61Bg5Eg/MvCGlyiyPMkCYwj8Cc+HijSHB0o8Utt8Cs3TCjG5vhGd14M3t0Iz3DHd34DnOvvAoT3ZGe3+zoxoAB8Ixu/Por9mvtRh7E5SUCIlAQAlIqgqT16quvombNmjlMrVq1ULt27RyGblEnl51v5q2w+fVlwutA4dHd10iudo73zvIpbi7nnXdekL8OiZU2gcbDhqLVHf81yVj13LNIcnumzYV7qOMqIYPWrcPxGzcgoU5t10V/0UjAM7px+eUwoxvffIP+8+fj5H3JGLB8Obh2w4xutGkDz+jGkCEY4yquZu1G585ZazfuvRfruXbDfYcO7dlTYqgeeOABv3IuUBlHN1sGWptuha2beK8Nx9p08w3P+nnb0SBXo0YN5Gaqc7czLxNOcr5p43Wg9NHd14SL3OXubzU9Pb3EfmMlEZGUiiAoP/XUU7jhhhuwc+fOHGbHjh3Yvn17DkM3yflz8WXCa7Ly5Ud3XyO57TneO8unuLl8/fXXuOqqq5CamhrEr0QipU2g4aWXwqZh9SuvmFNOednq9kYnTZmCjZ9/gUO7dxt3HlLWr8f6jz7Cyuefx7oPP8S20X9i+9ix9Mph9i5ahI1ffYWVL7xg5HzXbuyePRtr3nrL+G/9+WekuApMjgDci+1jxmDzt9+a+w9s3gKma+OXX2LvwoWub86//WvWYMsvv2DdBx8iacaMnJ4+Vwe3bgPzt+rll00aU9au9ZHIuixImFl3RM/RjG60bAnP6MYbb+AYO7qRlJS1doOjG+6IViZHN77/3jO6MapqVfzRoAEmc3Tj2muznrH7Pu1buRKh3JnqXleRefjhh/3KuUBlHN1sGWhtuvnWJXSz/tamW2HlbBjedqDwvP3teTjL7dq1C7mZJPf98DbhJOedLnseKH3Wz9sOF7mP3PJ32LBh0VPYuDmRUuFCyO/vxx9/9Ij069cP3maAW9h6m/79++fwpyzdvGV4Tjf6eRu60c/b0M1bhud085bhOd3o523oRj9v4+1vz8NVjr0LHvDuSaTmI9w5B0qfi9v8vf322zh06JA51yG8CVRo3hyIc0wid8+caWwqDnOvvgYz3FGnOcOHIWXNGuO++bvvMKFbNyQvWYIylStj54QJ+OfEk7D+/feNPw8Ht2zFVLehOb5DB3BdxsGtW7D4v3didJMmpiFPmeVuh8tf3btjwycfG0Vh0d13G//p55yD9P37KWIMF5NPv+B8zBk6FKtffcXEPfvyoRjfsSP48TgKpbqNm1mXXeb2nDfHiqefxp45szH3X1di4W230dvPUGZ086ZYePttSN22DatdxWJsm7ZY+eKLHtmChum5MUZOyrpKg1m74faYtnv0URzhO7rhKnctb78dFTm6sXgxlnPthju6wfU7fqMbbgOJI2SHCjG68cUXX3iI2/KIdrjWTUwbTRHS59dOKI3wPNCzTwZ07gJr+nXuBF/T38s/FuRsHr1tXya89va353T3NdaPdjZyfOP+5ux5NNhx0ZCJkspDA7fHZty4cRjnZca4PXDeZuzYsTn8KUs3bxme041+3oZu9PM2dPOW4TndvGV4Tjf6eRu60c/bePvb83CVY7rss33UrfAiNR/hzjlQ+oYPH27Ry44QAk5cHMpUqgL+2782S3mo3L49BgRY4LzcbbQnuuUZ12Q0ufJKdHWVx0aXXgI4Duy/WZdfiq0//YQGF16EPhMnmu1Qy9WuBWSke6ZX7XTdKV+xZSu0ue8+HDdrFsrWrm1GJda8+Sa9jDl22jTUHnS8OV/+v//hCLc3vP6555jrXf9MNfacK67Aho8/RsOLLsbRbjna8bnncOzMGag1aJDx9z5s+PxzLLrjDpStWh19/p5s1pbwnirdurjut2HfihVGvCBhmht0MAQ8oxuuAtHiP/9BFzO6MQ4nbtmMwW7vdZ9//kFnO7qRkYnN37ujG1ddbdZumNGN+l6jG+5IGEeTghnd6O4qqLY8os06YIxPHUt3XyO5MX7tDjLKj8uoUaPM8+bh7RtvxJjHHvWYcY89Dl8z1svfyvrK8Dpa5GwevW3mz9d4+9tzXxleWz/aQ3r2JPaoM9GnVETdI1KGREAERCA4AukH9hnBhDp1jc1DfIUKQPYIBq9p0pOTsXfePIzv0hkLbr0NGz77DK1dpaDtI4/QG7td5WD773+Y8yZXXWlsHrgwmHJNr72Ol+jgjlS0uvsutLrnHnPNxmgtd7SWF7smT6blMfEVK5rzqm5lWqVzZ3R49jl0c3u3W993r5kGtcVtmFKgsavkOI7DUziuokRZc+F14CgFL+sMORllq1ThKRh3/fPOR2ZqGja6+eHUqoKEaQLRIV8CZnSjd280tqMbX3+dtXZj/z4McJW5XhzduMMd3WjbFsnu6MaKp57GVFc58R7dmH7uuVh8771m+t2uqVORoI/85ctdAiIQCQSkVETCU1IaRSAKCShLoSXAaSdsUDPUKl270srVtH3oIcRXqojkBQux6rlnMeviizGhZw/snj7d3LN3/nxj8/Egq0gAABAASURBVFC+SRNaxlQ/6igzKmAXfFds3RqVO3ZyRwfuwBj3fEyrVtj+52gjm5mWZmzfQ9UjjjBOCXXroNGllxqlgEqMcXQPHEFxrVz/OJef6abA5h9/xMSjj/EYu5YkdVcSChImw5IpGgEqgNytrK6rQJjRjREjcIw7qn/Cpo2e0Y0ub72Juqdn7Ua25YcfMNcd3Zh05JF4YuVKfOjqkcOXLcdcrt1wRze2jByJYEY3ipZq3S0CIhBKAlIqgqCZkd2LYu0gbgkrkb///htLly4NqzTFQmK4GOy3337DgQMHYiG7ymMpE9jxR9bIApNRa9BAWrmaWiecgAHLVqDjSy+h4SWXoJw7spG+Zy9mDRuKjEOHkNCgvude77URHkf35OC2bRjftQv4jQyOfPT++WcMXL4c9c452/XN/S+hXj0/TyoYHsfMTM9poBPHHb1IbNjAeDW84AIcPXaMx/RftAgnp6SgwzNPoyBhmsB0yI1Akd3t6Eajyy6DWbvhjm70c0fKTs4e3RhRrz6+cR/7toSy2LdkCTi6Me2UU8y3V7h2Y1ynTjCjG+6I2LoPPwRHN7w3HShyAosYAKcarcler1TEoHR7AQhsdWV/c4376rjHyPqzac7Mp7yLrFwBUioi7YkVIr1pbo9hpCpEhchu2NzCwoLsaYdNopSQqCRwYOMmLHk4a+pS9T590PSaa/PM59g2bbBz/Dg0v/FGdP/4Y7A3udHQoaBiwR2VavTpi7gK5U0Yu9xOCXOSfdgzdy5WPvec+ZgeRwziEhLR44svUKltWyNxaOdOYxfkYOJzw+E9+306QAL9fqgUUZY7T8UnJsLbcAeqDZ9+ioKGyfBkSpaA4yqIHN1YWD4RP7hR/9CwkasgjjXv4+Ddu9F36lRwdKPemWeC//jxx3lXXwOOboyqVg1/1G+Av/v3x9xrrgF3MTOjGytWhHRnKsabn+FmFtG2NWh+eQ4Hf27GmuYmxDbQ3dOI+SueNJd+9qVUlP4zUApEQAREICCBjNRUcAejdLf33Qqk791r3FJ37TRTfLggmmsj9s6dg2pHHmmUBDbWKJ/hdiik7toFZGRVYWl79piRCE5N4px2z/avjgOwgdemNcq7owBspLd96GHXzcGS++8HG+8ML2XtWkw74wwcSkpCfOXKdDLh7V+1ypzvcXuf7fSnAxvWgztI0SMtORkZ2SN2tJkmb2Uhvnx5cG0FZZc9+SRsfrkQfM3rI+iMA5s2gbtZ8aLVnXeaHYl2TZyIVe5oS/rBgyYd3Pp21qWXoWz16ihomAxXJnwIcK1MtV69YEY3uLjfZ3Sj98iRaPnfO1CpXTvscxVRrrMxoxutWsEzunHOOVgcpqMb4UNaKRGB0BGQUhE6lgopigkoayJQGgTWf/wJfq9RA4vdRrSNn416uv1eoyb+OqIXVjz1FKofcwy6vvse+kyejArNmllRJLk9vX/UrgOOJnANxeT+A7DNbYw58WVQsWVLTO7fH+O7dgM/gLbXHYHo8eVXnnv5obQj3GuuqZh01NEY5cY36di+aDxsGFq7ikbj4cPdRt1/Ua5WTfzVswcmdOuOBTfdhN6/jgQXY++eNRN/NKgHbm/L8LeN+h1MA7eU/b1ObaRu3+6Jiyet3cZfN3fUhArL7/Xq4s9GjTD36qtd5aE1vXFg7TqMbtwYafv2mTz2mTQJjdw0rH7tdTdt1fF7rdpY9sgj6Pn5Z+C8ft5UkDApLxP+BKgwc3Sjzskno8XNN6PLiBFZoxsbN8IzuvH2W+DohhMfhy0//QTv0Y3f69Y7PLrhjrh5Rjei7CNk4f8klcJoJCClIhqfqvIkAiIQFQSaXDEcp2Zm5m7S08w6ht4//ojGw4fBcZwc+a7hKhunpB3CkAMpOHlvMnhe94wz0H/JYhz522/gl5ZpHzdnDo6dPh1VfRZ413N7eul+0t49GLhyBY5fsxZtHngAcWXKGNP+iSdw4pYtGLxjJ46ZNNE07hgn7znl4CGcmp6Bqj164MStW03cTAPNqYfSkFC7do608qLRJZeYnYRO3LwF/RcvRv+FC3GkqwSd4N5/kjvKMuTQIZTJ3kWqXK1a6PbuuxiweBFO2LQZJ2zdgn5z56LumWcyKI8pSJiem3QSkQQ8oxuXXop27uhGT1cp7ucqy1y7MXDlSnB0o9Vdd6JS+/ZZoxvPPAPP6EbFSjBrN9x33jO68c8/CKe1GxH5UJTomCIgpSKIx+04WRW142TZQdwSViJ9+vRBmzZtwipNsZAYfrzvpJNOQmJiYixkV3mMIAJsfDG5juMgsX492Gu6BTJxZcuibLVqgbyMG/1sY984FPHAqUtlKlUyofDjfFRAaFOZMY4+B6Y/PiHBxzXnZUHDzHm3roqTQFxcVlPE2qGOi6MbFZo3Rx07uvH660YBPsF3dOOss5BjdOOoozCqWjVkjW70g1m7wdGNX34x30LJzB7dGDBgAJo1axbqZEd5eEXPXl03iJNck/X2uCcR9GfT7DiR2a7MDbXNV27+cncJOE7WQ3ecLNt1iqi/+Ph4FFdhHVEgSjixjuOgjNuj6ziR+d6UMC5FJwIiEKMEHCerjHScLLskMVAhNWs3OLrB6XO+oxu//gp+i6VS+w7Yt2wZVnB049RTMZZrN8zoRkfMvuACLL3vXqzjzlQc3UhKKsksxGxccW7Oy7hGf+FDgM8kfFKjlISEgAIRAREQAREQAREoPAHP6IY72tzi3/9GF45ujBkDjm5wKl7fadPQhWs3zjobjttxt+Wnn2HWbnB0o3r1w6MbV18N7pa2haMby5fDjm4UPmW6UwTCl4CUivB9NkqZCIhAdBNQ7kRABCKQAKfiVTviCPDjje3M6MaXZj2PWbuxahV6Z49uVO7QEftcRWLFM89iGkc3WrfGyAoVMK5TR0w3azfuxroPPsSuYhjdSD9wAJ7d3SKQsZIcmQSkVETmc1OqRUAEREAEREAESoRAcJGY0Y1mzVAne3Sj82uv4WgzurEBdnSj67vvop4d3fh5JOZdcw0meUY36uLv/v3AXc9WPPssijK6wW2hlz/5JLh1NLd0Di4HkhKBohGQUhEEP/vhOGsHcUtYiUycOBFLliwJqzTFQmJ27dqFkSNH4oDbYxQL+VUeRUAERKAwBGzdGqkfkBvjKg6rV6/OM+ue0Y1LLoFndGPObHhGN377Da3uvhtmdGPFCqx89jn/0Y2zz8aiu+/KGt2YMgWHkpLyjLOtO4qy/NFH8Vvlylj79tvIzMjIUz7SPPlF7ZFuojNdE2l/Ns3e3+uJtDwESm+pKxWBEiW30BJggR1tL25oCRVPaGQu9sXDVqGKgAhEDwGWlZGcGypDLOsLkwfP6MbgweDaDTO6MXo0TtiYPboxfTo4ulH/7HPglCmDrb/8inn/dy0mHX00Rpm1G+7oxnHHeUY3tv78s5lyxbUb5Vz/Tu5oCdM196qrwI9kbncVIF5Hg0l3M0E1yTbQ3cuI+YvENAcDV0pFMJQkIwIiIAIiIAIiIAIlSMCMbvTsCX5rpe3DD6Pnl1+iH0c3kvdhINduuKMb/MBj5U6dsM8d3Vj1/POYetppGGvXbnTogO1//IHyLVqYVCcvWoIpgwZh6slDkLx0qXHTQQRCSUBKRShpKiwREAEREAERKFUCijzaCTiOgwpcu+GObjS/6SbY0Y3j16/PWrvhjm50e+891D/3XCAuDqlbt2QhyWDfPrD1t18xrm1bpL3yCipm+egoAiEhIKUiJBgViAiIgAiIgAiIgAiULgE7utHw4ovR9Lr/w+ZvvkG6O7LBVDnlytHymMyU/ejpudJJiROIwgilVATxUO2H4xzHCUI6/ET69u2Ltm6vRPilLLpTVL16dQwZMgTly5eP7owqdyIgAiJQBAK2juWHWosQTKndOmjQIDRv3rzU4g8U8d6FCzGx1xEerzqnnoo2DzyAI//8A4OTknBqZibK3n4HJngkIu+knpvkIa6JxIasTbPjRGa70sUe8M/mK6CnHHMScJzIfPgssB0nMtOe8wkEfRUWgo7juCPP+olB/0RABEQgDwKOE9n1U7jVsQe3bsPm775DpxdfxKC1a40C0funn9D67rtRe9DxKFu1ah5PI3K8+Naohg2v56XnEV7PQ6kRAREQgRgioKyKgAiEmkBCndrgAu5655yD8o0bhzp4hScCuRKQUpErGnmIgAiIgAiIgAiIgAigkAiu3l0Zw5KqRKx5bb+mLxfk0UupKAgtyYqACIiACIiACIiACARF4O2UCvjwQHn8UqZWxBmme3xqzsXtQWU6hoWkVATx8O1HbawdxC0FESl22QkTJmDx4sXFHo8iyElg165d+Pnnn3HgwIGcHroSAREQARHwEODH43hhbZ5Hkvnzzz+xatWqSEpyiaW1rJOJtg0boFOzpiE3Z9WvgHcrrUTnpk1CHjbTW6NK5WLjZD9+F+kffvQFJKXCl4iuRSBEBGxhYe0QBatgSo2AIhYBERABfwIs42n8feRS3AS4WBuwTfTiji104UdeioPLu5SK4DhJSgREQAREQAREIBIIKI0iIAKlQkBKRalgV6QiIAIiIAIiIAIiIAIiED0ECqpURE/OlRMRKGYCjpM1MOs4WXYxR6fgRUAEREAESoGA4zhwHKcUYlaUWdOIIo995KU4uHdNSkUQnPhhG4pZm+eRZI477ji0a9cukpIcFWnlF7VPPfVUJCYmlkJ+FKUIiIAIRAaB+Ph4k1Brm4sIOhx//PFo3rx5BKU4OpK6LKE2nqs1AJkRqNBZpcJx7Fl0PBMpFdHxHJULERABERCBSCSgNIuACIhAlBCQUhElD1LZEAEREAEREAEREAERKB4CCjV/AlIq8mckCREQAREQAREQAREQAREQgTwISKnIA46vV6TuQ82P9oV32n1JR8c1mZN9dORGuRABERCB4iHAsrJ4Qi6ZUFnOR3oeSoZUiGPJzER8ZkaIA1VwRSEgpSIIeiwwKBaphcbEiROxZMkSZkGmBAnwi9ojR45ESkpKCcaqqESgGAkoaBEoBgK2jk1PTy+G0Is/yNGjR2PVqlXFH5FiyEGgdep23LxjPJwIVCysKhSp7cocD8LrQkqFFwydioAIiIAIiIAIiECkE1D6RaA0CEipKA3qilMEREAEREAEREAEREAEooiAlIoCP0zdIAIiIAIiIAIiIAIiIAIi4E1ASoU3jSg950f7HCe6PrASCY/KcRyIPUrvn2IWARGICAKOE9n1Ez/ax7I+ImBHUSIz4CANbMZG3vsTeSkO7sXh0whOMoalbGFh7UhD0bdvX7Rt2zbSkh3x6a1evTqGDBmCxMTEiM+LMiACIiACxUXA1q1snBdXHMUZ7sCBA9GsWbPijEJhByCwIqEWXqzVD5kRqJRapcJx7FmADEagk5SKCHxoSrIIiIAIiIAIiIAIiIAIlCCBfKOSUpEvIgmIgAiIgAiIgAiIgAiIgAjkRUBKRV505CcCJUVA8Yj5S2okAAAQAElEQVSACIiACIiACIhABBOQUhHEw7MfJ7F2ELeElQg/KGQ/LhRWCYvyxPB9SUtLA+0oz6qyJwIxQ0AZDT0BW0ZaO/QxFG+ILOdVxxYv40Ch86N3ZTPSAnnJrZQISKkIArwt6KwdxC1hJTJp0iQsXbo0rNIUC4nZtWsXfvvtNxw4cCAWsqs8ioAIiEChCNgGubULFUgp3jR27FisXr26FFMQm1G3St2Bm3b+ldsXtcMaSkZ26iK1XZmdfD9LSoUfEjmIgAiIgAiIgAiIgAiIgAgUhICUigLQSk1Nxc8//+wxCxcu9Lt7xYoVHn8ru3jxYj+5ZcuW+ckFGk1YsmSJn9zy5cv9wlu0aJGf3MqVK/3kFixY4Ce3Zs0aP7m5c+f6ya1bty5Lzus4e/ZsP7kNGzZ4SWSdzpw5009u48aNWZ5exxkzZnjkJkyY4OWT83Tq1KkeOct527ZtOYXcqylTpvjJ7dixw/XJ+ff333/7yXGkIacUMHHiRD+53bt3+4qBaWeY9Bg9erS5Z+/evbzMYcaNG2f8bB5o79u3L4cML8aMGeMnl5KSQq8c5s8///STO3jwYA4ZXowaNcpP7tChQ/SSEQEREIFSIZCcnJyjXGJ96psQ1rssJ73NqlWrfMUwf/78HGFRvrTquk2bNvmlb/r06X7p27Jli59ccdZ1rB/9Isx2mAgHP7vn3sa/pgPG+8hQPtl18/0bvmsKbtk+Noepku4/in/lzsk5ZHhPxXT/OuyaHZNw+t75Jppbdow39yRm+Ndh1+/4y/gxHGvKZvpPmbppe1YYVoZ2GWSa8L0Pv7gXzKO3cZ1y/GW4V97+PB/pusXCn5SKAjxl7qHdo0cPWNOwYUO/u+vWrevxt3INGjTwk6tXr56fXP369f3k6GbDsTbv9RVkHNbf2nXq1PEVQ6NGjfzirVWrlp9ckyZN/ORq1qzpJ9e0aVM/OX6fwVewefPmBZZr166dbzCe65YtW/qFV7lyZY+/PWndurWfXKVKlay3x27Tpo2fXMWKFT3+9oTf+7B8rV2hQgXr7bGZdrtveYcOHUzYgb5XYf1sWLQTEhI84diTjh07mjDob025cuWst8fu1KmTn1yZMmU8/vaka9eufnJ8v62/7OggoFyIQCQRYNlnyzfarE990896l37epnbt2r5iaNy4sV8ZF+q6rkaNGn7xBqrrqlWr5ifXokULv/RVrVrVTy5QXVelShU/uVatWvmFl19dx/rRL6Bsh3Zug7qHe+5t/Gs6oIOPDOUDfZlpXMXW+KVyhxxmf1xZ9+6cf2Mq+csdjCuTU8i9+rNSGywul9XGGVmpvQn3kBPv+uT8+71SO+PnHXfWB/Nyyv1aOSsMbzl/lQLo7t7GPHob1ynHn+NeefvzvJvrFgt/UioK8JTZ6GLj3ZpABQB/xNbf2oEKADaArb+16eabHN5r/a3NOHzlmBbrb20rx0al/bhQILlAjedq1arBhmPtQI1nKhDW39rBypUvX943G2AhbcMJpBTZG6jgWDlrJwb4yFwgOVZcNhxrs7Kx4Vg7UKOdlZf1t3bZsv4FI9POyo/s85Oz/tbmPTZd1mblav2tzffR+lubCqf1t3awcvYdsWHJFgEREIGSIOA4bIYBLHNtuUXb1mHeaQhUhwUrF+q6ztZhrANsORuoTrRy3vnwrusaNGhg6ttQ12H51XWsH73T5H3OrkZ2h3ob/5oOqO3e5C3Dc38VAFhVriaWJNTNYdICKAEry9XKIcN7AsmtSKiNhYl1ccApg8XZ4aY7/k3aZa4cw/A2mQHl6vjFm46s99LNoueP+fM1Hs/sE97lK8PrbG9jUcacRNnB/wlEWQZDkR3b2LJ2KMIsyTCOOeYYsCe+JONUXAArl5NOOgmBKgrxEQEREAERyCJg61ZrZ7lGzrF///7gqH3kpDg6UrrKVUBerXks9EXt8Hme0a9UhA9rpUQEREAEREAEREAEREAEopKAlIqofKzKlAhEHgGlWAREQAREQAREIHIJSKmI3GenlIuACIiACIhASRNQfCIgAiIQkICUioBYcjraj5NYO6dv+F9xq9D09PTwT2iUpZAfcuJ2rpH63kTZ41B2REAEwpSALSOtHabJzDVZ3G4+lurYmYfKgCY90PZIuVIKvUd8ZgbKZ6SGPuCoCbHkMyKlIgjmtqCzdhC3hJXI5MmTwe9ihFWiYiAxSUlJ+OOPP3DgwIEYyK2yKAIiIAKFI8AOGN5pbZ5Hkhk/fjwCfQMjkvJQkLR2LJOOp/dVRNktdRG3uS7u31sRvx0sh90ZJbunUYvUHfi/nZPguMpFQdIfDrL8lgXTEantSqY9kJFSEYiK3EQgHwLyFgEREAEREIFYJJDgZOLTqrsxqFzWR+n+t68Shuyqjhpba6PNtpq4ZndljNhfHsvT4mMRT0znWUpFTD9+ZV4EREAEopqAMicCIhBiApz2tCo9HndU3I+LE1NM6A4y3f8OlqeXwVspFfB/e6rg8t1VkFHKU6RM4nQoMQJSKkoMtSISAREQAREQAREQgfAnkOIqA/MPxeOHAwl4bl8FXO+OPpy4sxpauyMRiVvqoNX2Whjsjk784vpTnWCOaFO5aBqXjq+q7cbfNXchLugZUQxBJtIJxEV6BpT+/AmUK1cOZcqUyV9QEiElwA858YumjqNSNaRgFZgIiEBUEXCcrDLScbLsSMscy/lIrGP3pLu8m7cHep2AX+p0wnB3ZOG4HdXRaGttVHIVhy47auHspKquUlER89LKoFF8BoaVP4CPqu7G1Jo7sLPOVlxe4QAc93+ma2o4mXilyl4sq70D5yQeKPbHeMiJwz6nnBuP45rI+ou8FAfHV0pFEJzYOKSYtXkeSeaoo45Cq1atIinJUZHWatWq4YQTTkBiYmJU5EeZEAEREIHiIGDrVmsXRxzFGeZxxx2HJk2aFGcUhQqbi4A37EvFhE178e6Sbbhn2npcOGYFen2/ADU+nInzN1SDc/cHcP71ECZWb4lN6XHoXCYNt1bch+/ckYZ5Nbcjue5WrK+zDRPcUYd3q+7BPZX24cLyB3FE2TQsS4/Hy/srmLQ9UCkZa2pvx3UVUlDGVS6MYzEfVperiRE1+yAzApVRq1Q4jj0rZlglFLyUihICrWhEQAREIB8C8hYBERCBAhE4lJGB5bsPYNS63Xh14RbcMmUtzvh9GTp+PQ8V3p+Bxp/NQf9fFuPGv9fix7VJOJiegf4NquCxXo3wWO29yLzrLGRe1xePL/kBv9VIwqtV9+I/Fffj9MSD6Fg2HeVzafOmZjq42h3ZuDAxBRtrb8MDrrJRKS6zQGmXcPQRkFIRfc9UORIBERABERABESg2AiUb8L5D6Zi7Yz++W70Lz8zdhGsnrsYJI5egxedzUeG9GWjz1TycPGopHpixAX9vSUalsnE4p3l1jOjTDBNObYeNF3fDvuE9Me+cTvjuhNZ4undjXNu+DnqUTwO2bwAKsSXr7EPx+MIdzfi02h7Ui88oWSCKLWwJSKkI20ejhImACIiACIiACMQCgR0HDuGfrcn4dPkOPDxzA4aOW4m+Py1C/U9mo/IHM9HtuwU458/leHnBVix2RyaaVSqHq9rVwqcDWmL6WR2w6/Lu2H5ZD0w5owM+cd0e7tkIQ9vUQt96lVGvQtmQI+xdLg1tyuijuiEHG+EBhp1SEY48OS+R6bI2zyPJHDx4EIcOHYqkJEdFWvkhp5SUFETqexMVD0GZEAERCHsCtoy0dtgn2CeB/MBpWlqaj2vOS+ZtXfJBjN24B28v3oa7pq3D+aOX4whXWajmKg21P56No39chGETVuGT5Tux/UAautesgLu61sePJ7bGgnM6Y/+wnlhzUVeMO6Ud3jquOe7q1gDntaiBHjUromq5MjkjjIGr+Mx0VE4v/gXhxYEyWieKSakI4m1hYUAxa/M8ksw///yDFStWRFKSoyKtSUlJGD16NFjhREWGcmZCVyIgAiIQEgLsgGFA1uZ5JJm//voLa9euRWp6BpYkpWDk2iQzonDz5LU4ddRSdPhqPjhNqennczFo5BKz7uHXdbvNNxyOb1QFTx3ZCH+c3BarLuiCA8N7Ysn5nfHLSW3w8jFNcVOnuji1STW0r56IxDJqsnm/Fy1Sd+LqXZPhFGL6lnc4pXFulYpIbVfmxkxvaG5k5C4CIiACIiACEU9AGQglgb2p6Zi9cz++WbUTT83ZhKv/WoXbt1TEsX8nGcWh/dfzcervy/DwrA2Yui0Z1RPK4PwW1fHmsc0w6bT22HxJN+xxRxxmn90JXx/fCk/0aoyr29XBoIZV0LRyAuKibDegULJXWOFPQEpF+D8jpVAEREAEREAERKCECGxNOYTJW5Lx0bLteHDGBlw2biWO+XEhan88C1U/nIke3y7ABaNX4LVF27Biz0E0KJOBSxsm4ItBLTHz7I7YfXkPbLu0O/4+vQM+6t8CD/ZsiMta18LRdSuhTvnQr28oISyKprgJREH4Uiqi4CEqCyIgAtFF4PMVO3DUDwtzNdwyklMrvly5E+kZdiA9uhgUd25unbIWHb6eh6afzcHgX5cUd3QKP4wIZGRmYs3egxi9YQ/eXLwVd05bh3P/XI5u385HlQ9moN4ns9Hnp0XuKMRqfLlyF3YdTEPv2pVwf/eG+PnE1lh0bifsH94Tqy/sgtGntMN/aqbgmqblcU7zGuhWowIql4sPo9wqKSJQcgSkVJQc61KLiR9fK1s2pntHSoV9fHw8ypcvj0j9oFOpQFOkhkCXGhVxXfs6aFKxHKZu22dM5xrljdulrWqiWeVyeH/pNvMhq7Zfz8f8nSnmvmg93DBpDZ6csymk2TuxUVXUSSyLdftSsd41IQ1cgRWIgOM4Rt5xsmxzUcQDv8ewaNcB/Lw2CS/N34Ib/16DU35birZfzkPiezPQ/Iu5OMFVJu/4Zz3+XL8HcW7UJzeuiueObIIxQ9pijaswcBvWhed1ws+D2+CFo5vgxo51MKRJNbStVh7l4g83n1jOq44t4gMrxO2pTjz2xCW4d7oPzz1G0l/kpTg4uod/FcHJx6RUXFwWJmtHGoTevXujZcuWkZbsiE9v1apVMWjQICQksNCL+OwoAyVIoEP1RLMdZPdaFTyxnu42ZrhF5A0d6+LFo5ti/Gntjd/KPQdw+u/LomSXMZOlHAf2Kn+xcidWuj3LOTyKeDHYVSrObla9iKHo9lAQsHWrtYMNc3dqGmbu2Iev3Pfj8dkbcdWEVeCH3jj6VNFVHDp+M8/8Nh53FdJZO/ajVmIZXNKqBt4/rjkmn94e2y7thqShPTD9rI74clArPN6rMa5sV9t8HK5xpQRX0Qiu6de3b180btw42GTHlNyhTAdbd+/B1qTdITfT9sfj0biO2FJM4SftTS62Z2XfLMexZ8UWVYkGnNVaLtEoFZkIiIAIiEBRCXStUQE13UYSw1mdfBBLd0fm1opMf15mwqa92HEwLS8R+UUxgc37Ire+uQAAEABJREFUD2Hi5r34YOl23D9jPS4ZuwJH/rAQtT6aieofzsIR3y3Exa7bW4u3Y3VyKtpXTTQjCl8d3wqzXWVhr6s0bLqkGya6SvgH/Vvg/h4NcbE72ndknUru76dsFJMr5qwFGXwFJxO7kpOxdP36iDPs0CiDzCBzKjESkFJBCjIiIAIiEGEEDqRlIPlQ1senEuIdNKpYzi8HC3cdwDtLtuHuaevx0vytmL5tn58MHbgo9a3F28xuNtO2ZfXOzdm5HyMWbaU3uL/+e244ryzYYqYhbcyeLjTL7SVm+Az72bmbcSgjw8jbQzDxMwzuosN57Zzi9L7bePzS7XlmL/Qbbvznjl5ugpuxfR+YBppN+1ONmz0EEw9lp2xNNt8IYDw/rtmFtEw1GMilNA3XBB2qWgdo3xvbOx6H26euw1l/LEPnb+aj4vsz0ODT2Tju58W4dtJqfLNql/vOZ6BP3Up4qGdD/Dq4DZae19msb1h5YRf8MaQtXu/bDLd1qY+z3FGoLjUroGLZ+NLMXszHnVx3KzLqbcGhelsjzjDdn1TbE/PPsCAApFQUhFbJyCoWERABEciTwI4Dh8wc8YPpWY3iJ3s1ztF4YkON6xA6fTPPVQI2m7A+X7Edvd0eXi5QNg7uYfXegzj2p0VmUerjszdh3q4UnP77crPbzXl/rsDLrhLhimFR0gHcPX0Dbpq8Fne5CsrMHfvpjH+27sOtU9bh5ilrTGNw18EsJSfY+F9buAXH/LgIO1PT0KJyghtPCq6YsApfrNxpRl4+WLYdqdl5XOf2Qr/lKjY0G93eayYg2Hi4KLe/2zBlXO+5SstGVym5x83Pbf+sYzAyxUwgxVWAF7gKLhW55+ZtxvWT1uCkX5eg1ZfzUP69GVh77QtwbnkFG488C+M27kFCfBw43e/lo5uAH3pbd1FX8MNvC87tjB9ObI3njmqC6zvUxeDGVdHKHZkomz1FuZizoeBFQATyISClIh9A9LYfJ7E23SLJ8KvOqak5e/YiKf2Rmtb09HTs27cvaue6R+pzCV26SzakoeNXocXnc820j9ofz8Y7buO4TJyDP93eWX4gyzs1L8zfgtfcXv4aCfH45/T2eKxXI4w7tR06Vi+P512/f9we+7SMTFwwZgUmbUlG6yqJmHN2R7P95YwzO+AHtxd/+Z7D06m4qHmKG46Nw5aF17avg7eObWadPXYw8VP4pQXbUDOhDB7o3tDs1f9+vxamh5l9y71qVzJbcnaqUZ6iONPtef779A7GrWetisYtmHhsPids3oszmlbDX6e1M2tSZp3VESc3qmrC0aHoBJIOppmRsM9X7MCjszYa5bDfz4vQyB1p4IhDZ1fBPfOP5aBSMc8dBatfoSyGt66JD/s3R4P370Hmv49H53duxrQzO+LzgS3xqPvOXtG2No6rXxkN3VE4xwnfuef79+/HoUOHoH8lS4CTIgOPvZZsOgoTW1Z3UGHuDO97pFQE8XxsBWrtIG4JK5Fp06Zh5cqVYZWmWEjM7t27MXbsWBw4cLhxFgv5Vh6Lh8AzRzbGlDPaY+oZHXBNu9omEjaYf1qTZM69D68uzJq2dHzDqqieve6inNv7e17zrIXJX6/e5Y4yJGNa9nSoWzrX9WyD2cBtwF3SsqZ3cOa8ktc0Esc53MCrXNa/Ggkmfgaamp6Bje6oQ5PP5uAiV8Fhg/NWNy1PuXmlf34mmHimbks2u2cxrH+5jdS47LRTIeuQrbDQTyZvAqz/NuxLBde4cAravdPXm2fW6/sFqPHhTNT4aJYZCbts3Eq8t3QHKNupegXc0rkevjuhFeae3Qn7hvXE+ou7YcJp7fGeq0De070BLnTftXIblwP796DYvqidd9aK7Dtp0iSsW6dRryKDLGAA21z5sa7JOenSdYiAP6tU8HcVAckNOon+tUHQt0pQBERABESgpAjUdpWDOuXLooU7qvBan6ZoXy3RRP3Kwi1mz31z4R72p6WDC7fdU3y5cqcZ3eAIB80zc7OmQu1JTQfXTFCGpn21rNEAntP4XtMtWBNs/Azvf0c0REK8YxZif7FyJzgdqd/Pi/HT2t30ztMEG8/s7KlaDKxBRS3MJYfcDJXU5bsPYNT63eDUtFumrAW/idLp63mo8P4MNHaVP+6udMPfa93RrCQccJXCfg0q49EjGmHUSW2w/PzOSBl+BJZf0BmjTm6LV933lErFGU2rgyNO5cuoyZEbe7mLQDQQKOovPBoYKA8iIAIiEFEEHMdBS1e5YKK55GD2zqw1DryuUCYedV3lg+eXtaqJhed28pitl3Y3c9NHuI29BhUOL+w+6DYOKV8Yk5qRmeO2YOPnTX3rVcKS8zrjhaOa4IIWNcxUKObnlilrQMWHMr6GPeXfuyMtwcZT3yufPuvIfYOOiWsqY/N2poAMn5m7CddNXI0Tf12Cll/MQ/n3pqPNV/Nw8m9Lcf+MDWZqXEVXEeCi5xF9mmHCqe2w4eKu4Pcb5p3TyR2BaI1nejfBdR3q4IRGVY3CWybu8ChWTABVJkVABDwEpFR4UOhEBKKZgPIWbQS4577NEz/yxXMuvP5zw24McnuPeT1t+z4zEpBYJg7WjFi8zayrGODKuIMEFMOUrTlnJq/b778Gy8ryBm61SJvmzw17aOUwwcTPGzp9PR/frEoC14R8NrAlNl/SDWc3qw4qFhuz0xDnKlCUPZStvHy8fLvpSadbMPEM9MrnwqQU3uYxaYzIcxU9JzsOpJnpbZ8u34FHZm7EsPEr0fenRaj/yWxUen8mun47H2f/uRwvzt8KLsJvWrEcrmxbE58MaIlpZ3bArsu7Y/tlPfDPGR3wqftcHnFHIoa2qYW+9SrDW0mLHmLKiQiIQCgISKkIBcUwD6NChQooV65cmKcy+pLHL2pXrFgRcdqZJPoebjHniIteF7sN4G1u49BGtX7fIdCNPc10a10lgZYxI9dlrat4eNZGPD9vi9mLv1mlBFf+AG79Zx241//WlEN42u2Z5pSWxm4jsmq5MritSz1z/4sLNoO915zfy21jn8+eJmU8sw/VEsqgVfboyIJdKWYDAk6TGbGIM5uzhP5wFQxuBXt/j4bIL37ewTb9E3M2mgW+vKbikOKOmjSoUBbtsqdk9cz+ACC3lGUeRm9MRv/6lSkedD5v71LfyD86a5P5enZKWgbY4H5zSVbaN+0/hGnbksHdpIxgaR6CiJvPidv8cqckbul797T1OH/0chzx3QJU+2Aman88C0f/uMhVJlbhI1ex2JaShu41K+C/XevjhxNaYcE5nc2I1Tp31IEL+N86rjnu6tYA57ujRT1rVQTfjSCSETIRx3FMWI6TZZuLCDqwnFcdW/IPjJMZK7rRRuJbY9PsOPbMzUgU/EmpCOIh2kahtYO4JaxEjjjiCLRo0SKs0hQLialatSoGDBiAhITDjb9YyLfyWHQCX63ahS7fLjDfVKhUNg40t01dhy7fLMDfW5JNBDd2rIvLW9dEQpyDia5bXbch+fWqnbizW320qZqISae3w6WtauLHtbvNXv8NP52N95Zux6cDWuC8FjVMGPyC8Bt9m5ne555ug7Sq2yDlrlG967CqNiI5Dq8c0wQtXMXivukbUPOjWWah7vUd6nhkuEj313W7g46fox/tq5fHab8vMw3hmm4e1u9LxbfHt/KEeVvneji2XiUsdBUZ5uGoOhVwWtNqxj/YfHL3K+5SFe/WeM0/nwMuLH7I7cFvVqmcCScpNR1H/rDIrO0wDmFw4CJ2ftDwt3VJeHnBVtw8eS1OG7UUHbLXNzT9fC4GjlyC/7juVCqpoA1sWAVPHdkIf5zcFisv6IKU4T2x9PzO+OWkNnj5mKa4uVNdl111tK+eaEauwiCbJglx2R0v1jaOEXQ45phj0KhRowhKcXQktZabjQGuKa1muRt1of8iMc3BZNYtYoMRk4wIiIAIiEBJEbiqXW2kXnEE9gzriT1Ds8w+9zz1X0eAOzoxHZXKxoNbsCYN7WEakH8OaYftl3Z3G+BZvficpvJh/xbg4tmdl3XHXjeched2xoUtc+7sxLg4Pz5paHezMw9l+tTLCoPxeBtuLcvwkt20LD6vM3Ze3sM0Yjde3M1MmWEj1oYfTPxrL+oKfoeA8/TnnNURG92e8zlnd0LvOpU80TZ2R1zGn9oezP+Oy3qYKTpcT2EFgomHstz5iXnb7fLa4nJa4ja2J5/eAVsu6Ybdbj4OXtETXAhP2ZIy/HghF8x/4yqD/ADgNRNX4/iRi9HMVRgqvDcD7b6ahyGjluHhWRswdWuyO4IQj/Ob18CbriI48bR22ORy5zsy22X2jauIPdW7sdmad5CrXDSrnIB4V+EsqbwoHhEQARGQUqF3IAwJKEkiIALBEuCHwtiA7FyjPMrFBy7SOXWJayryCpMN9Srl+IWIvKSy/BiWbYCXdXuZ61Uo6zZ4yyChgPEzXQwxznHArWzzmnbDvOWXPobHtDHM3Ix3Phle7fJlzXa6zEdu9xTFnVO2JrsjSR8v246HZm4wHxY85seF4MhSFXdkqLs7InXB6BV4deFWcOclfgTwuva18fmglphxVgej8GxzlaC/z+iAjwe0xIM9G+Ky1rVwTN3KqOtyL0radK8IiIAIhJJAXCgDU1giIAIiIAKRTeDBGRvw69qsNRqcfnP1X6twIC2XneAjO6shST0Xra/ZexBjNu7BW4u34c5p63Dun8vR/dv5qPLBDNT7ZDb6/LQIV/21Gl+s2IVdB9PQq3ZF3NOtIX4+sTUWndsJ3E1pjTtqM/qUdnjz2OZm7cO57ohE95oVjcITkoQqEBEQAREoZgJSKoIAzEVxFLM2zyPJ7Nu3DwcPHoykJEdFWtPT07Fnz56I/aBTVDwEZaLABJpXTjBrLh5xe8Tv694AtRPLIi0z57axBQ40Cm7gYvefXWXrpflbcNPfa3DKqKVo99V8JL43A82/mIvjRy7B7f+sxe/r94AV6+DGVfHckU0wekhbrL6wi1EcFp7XCT8PbmO+6H1TpzoY0qQa2lYrn+sITxRgCyoLtm61dlA3hZFQcnIyUlP9d0wLoyQiGtOS5mbKf+851zEC/qK1RGXZFwH4SzeJtqCzdummpuCxz5gxA6tWrSr4jbqjSAR2796NCRMmSKErEkXdXNIEuHXoAz0a4h5XoeAuTo/2agSu3yjpdIRbfN+u3onTf1+Gx+dswozt+1EroQwublkd7x3XHH+f3h7bLu2GpKE9MfOsjvjy+FZ4oldjXNmuNgY0qIImlRLAKV7hlqdwSU9G9gdErB0u6Qo2HZMnT8b69euDFZdciAhsc8OZ4JpIHEe1SkWktitd7AH/pFQExFIQR8mKgAiIgAhEO4GhrWthz9Ae2HRJN0xylYgP+rcwW9pe0qomjqpTCTXdEZ1oZ6D8iYAIiEBeBKRU5EVHfiIgAtFDQDkRgSIQqFg2XiM2ReCnW0VABKKfgJSK6H/GyqEIiIAIiIAIRAwBJYzhCDkAABAASURBVFQERCAyCUipiMznVqBU82uf+gBbgZCFRDg+Ph6VK1dGXJx+ZiEBqkBEQASikoDjOCZfjpNlm4sIOrCcVx1b8g+snBtlZddE4ltj0+w49szNSOT9+aVYrR0/JP4OtlFobX+J8Hbp2bMnmjdvHt6JjMLUVa1aFf369YMqmyh8uMqSCIhAyAjYutXaIQu4hAI66qij0LBhwxKKTdFYAvyMZz/3IhKb5ZGYZhd1vn9SKvJFJAERKAUCilIEipnA5yt24KgfFuZq+IG2y8etxLNzN2P2zv3FnBoFLwIiIAIiEOkEpFRE+hNU+kVABESgEASOqFUR17Wvg7rly2Lqtn3G8KvcdLu6XW0MbFAF6/el4vap69Dj2wW4YsIqHEwv2uaNN0xagyfnbCpEasP3FqVMBERABEQgi4CUiiwOQR03btyIDz74QKYEGdgHI+4l/95Z9rKjk0CrqongNymOqlPRk8HTm1Qzble0rY3/HdEIY05ph1ePaYqEOAfvL92OS8auRGH3VeeXp79YuRMr9+pDnB7gOslBYNasWapfS7B+Zb1qH8AHo8fgg9Gjo9mEVd5Gzphh0UeVLaUiiMfpXYkOGzYMw2RKhMHzzz/veTpLly4tkTj1bIcZzpH6ESrPC6OTkBG4rkMd3NCprgnv29W7MHLdbnNe0MOETXux42BaQW+TfAwQUB2bVe4OK+G2xaeffup5u/5auADDXnhBpoQYeMBH2YmUiiAeqHeBF4R47IkoxyIgAlFN4LKWXBKZlcXXF23NOnGPLBt/WZuEh2duwJ3T1uHNxVsxdWsyDmUcnia1OzUNb7j3nDt6uXsHMGP7Pry3ZJsxm/anGjce1iUfxKsLt+AuN5xHZm7Ej2t2YcO+w/6UkYlOAurEiM7nqlzFHgEpFQV85iNHjkSkmSeffBJvvPFGxKX78ccf9zydvn37Rlz6v/jiCzzwwAP47rvvIi7t9erV87DXSfQQKGxOWlRJ8Nw6b2eK5/zfk9fitN+X4Vt3BCMxLg4vzN+Co35chGN/WoytKYeM3NLdB/DBsu1ITc801+uSU/GWq1TQbNyfJUPloc1X83Dj32uR4IazMCkFF4xZgeafz8U7rqy5UYeYIBBp9SvT+7///Q/vvPNOxJXzN998s+edOueccyIu/R9//LGpY3/++eeIS7sF7zjRtQ+UlAr7ZIOw2dA6+eSTEWmG290dd9xxEZducraPheeRZo4//ngcccQRGDx4cESzt89AduwSqFQ2HnHZ2acCwPURvJy7K0vBaFM1EQ/2bIhZZ3VEpTJx4MLvV9xRB8r0ql0Jf5/eAZ1qlOclzmxW3VzTrWetrPUcy/ccwMFspeP8FjXx2cCWuK97A6RlZoKLu/enpZt7dYhuAl26dInIsrJ3797o379/KNNeYmHZNyrS6lemd+DAgaaO5XmkmZNOOsmijyrb1hNRlSllRgREQAREIHQEDqRlwE5oql2+LOKye9deOqoJ7uhaD4+4CgWnQqW4cr1qZykKM7YFvw1tv/pV8GTvRmZReIfqia6CkYGeNbPCOZiRifleoyOhy5VCEgEREAERCCWB2FMqQkkvQsKqXLkyEhMTIyS10ZPMMmXKoFq1aojUDzpFz5NQTopKYNuBrGlKDKd7zfK0jGleOQHVy5XB8AmrUPWDmej0zXxM2bbP+HGUwZwEeehbtzLGb96Lll/MQzU3rP/7e63nzuxBDM+1TqKLgONkTQFxnCw70nLHD52qji35p1auXDlTxzpO5L03jhN5aQ7mCUupCIKSbRRaO4hbwkqke/fuaNasWVilKRYSU6VKFXAtSELC4fnosZDvUOVR4YQPgV+8dnwa0KCKSdiKPQfQ4su5uGvaepSLjzNTmtZf3A3cltYI5HPgblDfr95lpG6ZshZ9flqEn9Yk4T+d6mDH5d0xcnAr46dD9BOIi8tqilg70nLM6U8NGjSItGRHfHpr1qxp6ljHibwGuuNkpdlxsuyIfxjZGcj6JWdfyBIBERABERABbwJzd+zH47M3GaceNSvg5uztZT9ctgM7DmRtEfvlwJaeNRNbshdomxu8DnbK1KGMTOP68fLtGLV+t5nq9MqCrB2lhrWphRs61kWFMvHYkpIVthHWIZwJKG0iIAIiYAhIqTAYdBABERCB2CKQdDANi5NSsC1bMWDu1+87ZNwW7jqAkWuTcP+M9Tjyh4VYty8V3VyFgguoy2b3KldPKMNbjPl7S7Kxf1uXhHGb9prz1XtTMWvHPs/2sj1rVTDu3FJ2q6t4jN6YjP71K6NsnIMq5bKqogW7UsAtaPempuPJuVmKDG+auGUvNrpp4LmMCIiACIhAYQgU/z1ZJXnxx6MYREAEREAEwojAJyt2oMPX88FtYG2yrv97jXHr9M08nPr7Mny8bCcGNayC1/s0xfQzO6B11UQriuva1zajFjUS4nH2n8tR66OZeHjWRvx6Uht0qVEeK/ceQM/vFmLS5mRzz22d6+HYepWw0FUcGn46G0fVqYDTmlYzi76/HtQKfepWwoTNe1H749no+u0CdHbDeKJXI1QrF4//Tl2PS8etNOHoIAIiIAIiEJ4EpFQE8Vy4qwnFrM3zSDK7d+9GSkrW1o+RlO5ISmugtKalpWHnzp3Qh50C0ZFbaRO4vkNdZFzZK0+z8sIu+HlwG1zTvo5p/HunOSE+Ds8d1QTbL+uBnZd1x+oLu5p1FYMbVcXsszvh0L+ywu6fvQajcaUEjD+1PfYN64kd7j2fDGhppjkxTMr8dVp7HBjeE+vccBjvE70a446u9bHz8h4mjWNOaUdRmSgkYOtWa0daFpOSknDgwIFIS3bEpzc1NdXUsZGYEfuuWzsS8xAozVIqAlHxcbMP3do+3mF/OWfOHKxZsybs0xltCdyzZw/+/vtvHDx4MNqypvyIQA4C1RLKgN+yyOGYy0U5VxmpUi4+oC/96lYoG9AvRI4KJgwJ2I4Xa4dhEvNM0rRp07Bx48Y8ZeQZegI7duwwdWwkvjeR2p7M7ynG5ScgfxEQAREQAREQAREQAREoOQKKKRIJSKmIxKemNIuACIiACIiACIiACIhAGBGQUhFGD6OkkqJ4REAEREAEREAEREAERCCUBKRUhJJmmIbFr32WL18+TFMXvckqW7YsatSogfj4wPPHozfnylmICCgYEYgJAo6T9QEwx8myIy3T1atXh+rYkn9q5cqVM3Ws40Tee+M4kZfmYJ6wlIogKMVl78tu7SBuCSuRrl27omnTpmGVplhITOXKlXHMMceABV8s5Fd5FAEREIHCELB1q7ULE0Zp3nPEEUegfv36pZmEmIy7Zs2apo51nMhroDtOVpodJ8uOlgcopSJanqTyIQIiIAIiIAIiIAIiIAKlRCDslYpS4qJoRUAEREAEREAEREAEREAEgiQgpSJIUBITARHIk4A8RUAEREAEREAEYpiAlIogHr79SIm1g7glrET4tc99+/aFVZpiITGHDh3C9u3b9UXtWHjYyqMIRAyB8EuorVutHX4pzDtFO3fuREpKSt5C8g05gdTUVFPHRuJ7Y9Ns7ZDDKaUApVQEAd4+dGsHcUtYicydOxfr1q0LqzTFQmL27t2LKVOm4ODBg7GQXeVRBERABApFwH4R2dqFCqQUb5oxYwY2bdpUiimIzaj5RW3WsZHYNgsqzRH4WKVUROBDU5JFQAREQAREQAREQAREIJwISKkIp6ehtJQUAcUjAiIgAiIgAiIgAiIQQgJSKkIIU0GJgAiIgAiEkoDCEgEREAERiBQCUioi5UkVIZ382mfFihWLEIJuLQyBsmXLolatWoiPjy/M7bpHBERABGKCgONkfQDMcbLsSMs0P8JWoUKFSEt2aNNbCqElJCSYOtZxIu+9cZzIS3Mwj1hKRRCU4uKyMFk7iFvCSqRz585o3LhxWKUpFhJTuXJlHHXUUShXrlwsZFd5FAEREIFCEbB1q7ULFUgp3tSjRw/Uq1evFFMQm1HXqFHD1LGOE3kNdMfJSrPjZNnR8gSzWsvRkpvozIdyJQIiIAIiIAIiIAIiIAJhTUBKRVg/HiVOBEQgcggopSIgAiIgAiIQuwSkVMTus1fORUAEREAERCD2CCjHIiACxUJASkUQWO1HSqwdxC1hJcKvfSYnJ4dVmmIhMYcOHcKWLVuQnp4eC9lVHkVABESgUARs3WrtQgVSijdt374d+/fvL8UUxGbU/LAs69hIfG9smq0dLU8w1EpFtHDJkQ/70K2dwzMCLubPn4/169dHQEqjK4l79+7FtGnTkJqaGl0ZU25EQAREIIQEMjIyTGjWNhcRdJg1axY2b94cQSmOjqSyw5R1bCS2zSIxzcG8NVIqgqEkGRGIOgLKkAiIgAiIgAiIgAiEjoCUitCxVEgiIAIiIAIiEFoCCk0EREAEIoSAlIoIeVBKpgiIgAiIgAiIgAiIQHgSUKoAKRUx8BbUrFkTlSpVioGchlcW+dG7OnXqID4+PrwSptSIgAiIQBgRcBzHpMZxsmxzEUGH2rVro2LFihGU4uhIamJiIljHOk7kvTeOE3lpDuatkVIRBCX7lU9rB3FLWIl07NgRjRo1Cqs0FSwxkSlNRa53796gchGZOVCqRUAERKD4Cdi61drFH2NoY+jWrRvq1q0b2kAVWr4EqlevDtaxjhN5DXTHyUqz42TZ+WY2QgSkVETIg1IyRUAERCDsCSiBIiACIiACMUtASkXMPnplXAREQAREQAREIBYJKM8iUBwEpFQUB1WFKQIiIAIiIAIiIAIiIAIxREBKRRAP236kxNp53xJ+vtu3bwc/xBZ+KYvuFPGjd5s2bUJ6enp0Z1S5EwEREIEiELB1q7WLEFSp3Lp161bs27evVOKO5UgPHDgA1rGR+N7YNFs7Wp6jlIognqR96NYO4pawElm4cCE2bNgQVmmKhcQkJydjxowZ+qJ2uD5spUsERCAsCGRkZJh0WNtcRNBhzpw52LJlSwSlODqSumvXLlPHRmLbLBLTHMxbI6UiGEqSEQEREAEREAERKBUCilQERCAyCEipiIznpFSKgAiIgAiIgAiIgAiIQLgS0MfvwvbJKGEiIAIiIAIiIAIiIAIiECEENFIRIQ+qKMmsVasWKleuXJQgdG8hCPCjd/Xr1w/NF7ULEb9uEQEREIFIIOA4WR8Ac5wsOxLS7J1GfviuYsWK3k46LwECiYmJYB3rOJH33jhO5KU5mEcqpSIISvYrn9YO4pawEunQoQMaNmwYVmmKhcTwi9o9e/bUF7Vj4WErjyKQTUBWwQnYutXaBQ+hdO/o0qULqFiUbipiL/bq1auDdazjRF4D3XGy0uw4WXa0PD0pFQV4kmlpaVi1apXH7Ny50+/u3bt3e/ytLHco8BVMSkoKSo732nCszXt9w2NarL+1mRZfuR07dvjFu2fPHl8xcBtaG461A21Lu23bNr/wuOuRb4Dccs+GY+1AW/BxBw3rn9eOVd5yVj4lJcVgY1ukAAAQAElEQVQ3WmzevNkvfYHkuC2dDcfa3K7ON8CNGzf6hXfw4EFfMbPblg3H2txi1ldw/fr1fuEdOnTIVwzr1q3zk+P76Cu4du1aP7lAW9quWbPGTy5Sd17xZaBrERCByCKQnpG17faO5B344O8PPOaNGW/g5fkv5zBvTn/T429l35jpL/dWQLkROcJi2G9NeytAeMHJjZj1ul94b0972y+8gHJTCy/32pzX/OJ95593/OLNT+6PBX/k+qIEqusC1WGsp20dZ+1AcmFY1+XI++rVq/3qxEA7NNk8ets5AnIveJ+3P88ZvusV9X9SKgrwiNmIW758Oaxhw9v3dioB1t/abMj7ylEJsP7W5r2+crzX+ls7kBzTYv2tHUj5oBJg/a0dSPmgEmD9rR1I+QgkF0j5oBJgw7F2IOWDSoD1ZwPZl4e9ZoFn5awdSElhgWf9rR1IWWCBZ/2tHUiOjXvrb+1ASgXTbv2tHaigZePe+ls7kFLBAsn6W5vvo+Vh7UBygZQKFnI2HGtLqbAUZYuACJQkgbTMNBOdk+Ygfnu8x/y54k88OOvBHOafVf94/K3sqGWjcsjwnmmrpvvJ/bzsFz+5WWtn+cl9v/QHP7n5a+b7yX275Ds/uYVrF/rJfbH4Sz+5peuW+sl9svhTP7lV61f5yX288GM/uTUb1vrJvbfgfT851omWW9yeOOT2ryh1XaA6LFBdF2wdFkguUB0WqK5buXKlp82WV123YsUKPzkqB758bBjetq8M7/P25znD95WLxuvc36hozG0R88T5eyeccAKsadOmjV+IzZo18/hbuVatWvnJtWjRwk+ObvD5x3ttONZu3ry5jxTAtFh/azdt2tRPrl27dn7xNm7c2E+OU6ZsONYONIWqY8eOfuHVr1/fL7zOnTv7yQUaLu7atatH7uijj/YLxzp0797dI2fTx7Uj1t/aHBq1/tbmkKn1t3avXr38wqtWrZr19thHHnmkn1yVKlU8/vaEabfxWZvToay/tfv06eMXXoUKFay3xz722GP95Pg+egSyT4477jg/Oa7tyPb2WP379/eTK1OmjMdfJyIgAiJQ0gQyKmeg2unVPGbo+UPxwWUf5DBnnXeWx9/KXnHBFTlkeM/p553uJ3fNBdf4yQ05Z4if3PUXXu8nd8K5J/jJ3XThTX5yA88Z6Cd360W3+skdd85xfnL/vei/fnJHn300bD6tffdFd/vJ9T67l5/cAxff7yfX4+weHrmKR+e+DiRQXRdobWagui7Q+pJAdV358uX9XrFg67p+/fr51WGB6roBAwb4yQWq6wYNGuQnFxcX55c+W597275CvM/bn+cM31cuGq/9iUVjLouYJ2qdDMLaPI8kw9GEQKMMkZSHSEwrRybYKxSo9yQS86M0Rx8B5UgEwoGAp27NDIfUFCINm917kl2jvxIlwKnMrGM970+Jxl60yGyarV200MLnbikVQTwL+9CtHcQtYSWyePFicLpQWCUqBhLD6V2zZs0ClYsYyK6yKAIiIAKFI2CVCWsXLpTSu2ueA2wt1ugVeAACnOLNOjYS22aRmOYAj8DPSUqFHxI5iIAIiIAIiIAIiIAIiIAIFISAlIqC0JKsCIiACIiACIiACIiACIiAHwEpFX5I5CACIhCOBJSmghPYf2g/Gj3XGIM+GJTnzZPWTkKX17ug5YstUe+Z+pi9ebZHftj3w9Dg2Qb4bN5nHrdwPLl11K3o8GoHNHm+CQZ/NDgck6g0iYAIiEBUE5BSEdWPNytzderUQaAdirJ8dSwuAgkJCeajg/Hx8cUVhcIVgTwJfDbvU+zYux5/rx6DGRtn5CrbpGoTnNbmdGxMWond+zaDygiF52+djy/mfIBdyZvwxMQn6BS25sSWJ6JOxTrYtmcd1u/ZELbpjIGEFTyLTsFvCas76mcClcMqRTGRGO4exV0pHSfyXiDHibw0B/NSSakIghK3B6OYtXkeSYbbyDZo0CCSkhwVaeW2etz6tly5clGRH2Ui8gi8Pn2EJ9Ejpr/hOfc9aVy1Me457m5fZ7Sv1R4tanU07oNa5D3aYYRK8TC41WCc3e6sUkyBoi4sAcfJbmBFaoukk5vz2q7RX4kSqFatGljHOk72+1OisRctMsfJSrPjZNlFC62wd4f+vkj9CYeehEIUAREQgSgiMHXDVCzYdHh04tN5n2D3gT0FymF8XDxmXjMDC25agecGP1egeyUsAiIgAiIQWwSkVMTW81ZuS4iAohGB0ibw+rTX0bRmW5zY5nSTlIy0/fhozgfmPNjDnyv/xAfuPd8v/h5fL/w6x23cEnHsqrF4deqreHLik/h8/ufYnLwZb814G7eMusWsy1i7ey3enfUOXv7nJTw+8XEkp+7DnoN78cX8L/DClBewdMfSHGHai4XbFuKdmW/j7tF3m3unb5xuvXLYU9ZPwdsz3zHx/7jkR6Rlpufw14UIiIAIiEDJEZBSUXKsFZMIiIAIlAiBnSk78cWCL3HtEdfiul7XeeJ8bfrrnvNgTqhM/PvXG3HP77fizRlveW7Z5yoHfd89FkM+HIhRK0ZhddJqDP9uGJo/Wx8v/vMCXp/yPG77/TYs3r4Y94y+F3f89m887CoI9Gv4bAOjgLw45UV0faUtrvzxSk+46RnpuGHkDej5Wmc8Oekp4/7pvM9w7Fu9wIXYxsE9rElag/7v98eAd47Ge7Pew8a9G9147jHpdL0L8idZERABERCBEBGQUhEESPbIUczaPI8ks3nzZuzevTuSkhwVaT148CDWrVuH9HT1nkbFA42gTLw/+32T2qFdh2Jwy8FoUK2FuV61fRHGr55gzoM5vDLkFVzR/XCj397z8rSXMXP9JDSu0Ro/XvQjXj/1dVx1xNXGu3aF2nj+lNdx69G3gounp1w5xbjz8MLkF7DkxiX47sLvcMvR/6ETPpn9IdIy0sw5Ry/emfYqKiRWA+97bNBjGD98PFrX6YzXpjyHf9b/Y2Qv+PoC/LNmPE5qeyb+umICXjz5Rcy6dhYGtDrZhKNDZBHw1K2ZkZVuT2o3ugnf67nSSbEQ8A80JSXF1LGe98dfJGxdbJqtHbYJLWDCpFQEAcw+dGsHcUtYiSxduhSbNm0KqzTFQmL27duHOXPmIDU1NRayqzyGCQGWUyOmj8CFnS5A9fLV4TgOrul1jSd1r09/zXMezEnFchX9xFbtXGXc6lWqZ2we6leuTwsrdq0wIyQnt85q4FcqV8m483DLMbegQeUGPEWL6i2NjcxD4MgKL151FQraJ7Q8ATXK1+ApysWXw7kdzzXnnILFtSJzNvxjrv/V41+Ic7KqsTJxZdCxdtaicuOpQ+QQcNvkJrHWNhcRdFjgvoPbIii9UZLUpKQkU8eyzIu0LEVimoNh7P4SghGTTDQTUN5EQASihwDXQazbuQxT1k/BeV+eZ8yvS3/1ZPCHRd9iS/IWz3VhTgY0H2BuW7x9sWfxN0cR6Ni/WX9aAU2HWh087lYZsA7cxnZT0hpz+cOCr9DixRYe8/TEp437noN74P0NDaugGE8dREAEREAESpWAlIpSxa/IRUAERCBoAkEJvj79dTSv1R4Xdb4IXep1MWZgi4Ho1eTYrPszDuGd2e9knRfyeBa3bo1LQGp6Kpq/2BT1n6mPUUt+whkdL8CLJ72Ya6jcTSo3zwplK6BaxTrG+/wul2Lh9Qs9Ztsd27DrnhSMOHUE6lfKGhGhYEZmBi0ZERABERCBMCAQFwZpUBJEQAREQARCQGD9nvX4dcmP+G+f/+K+4+7LYZ4+Iau3n9G8Of0tFKVBPmndJMBVTqZdNQ0Tr5iEMUPHuI3+ffj83M/NlCvGURgzKPtbGNM2TkNCfAISyyR6zIjpI/D8lOcxsPkgwIkH/y3ctpCWx9i1GR4HnYhAqRBQpCIQmwSkVMTAc69Xrx6qVq0aAzkNrywmJCSgcePGKFOmTHglTKmJOgIH0w5i3pZ5uH/s/UBmulmvsCtllyefVCCqJlZFlQp1jduW3avxwewPzO5MB9IOgLs5cSqT8XQPa5LWYMf+HWaa1IY9G1wXGJmVu1aa84pluc4iA91GdMNFX1+ES769FAPeH4CTPz4Zj018DFRuKMhGvne463avw9Z9W7Ft3zas27OOIsYs27EMew8m4/5+96Ne1WZYtX0Rbv39VrNFLeWfnvQ07hp1KxpXaYyqiVXwnz53mPsenfCoiSvlUAo+nfcp3pr5tnHnblDTNkwDd5MyDjqEN4FI//5Xw0w4VcIbcTSmrkKFCmAd6ziR9wI5TuSlOZh3KOKUimAyFWqZuLgsTNYOdfjFHV6bNm1Qv/7hKQPFHZ/CzyJQsWJFdO3aFWXLls1y0FEEionAh3M+Qu83euLLBV+hTNlKOPHD4/HJ3I89sY1bPQ7dX+uEPQeSjH/ZcpXN1q3dX+uCVbtWgSMDfd7sBadMBcS55orvhmLEjBE44/Mz8O3CrDBnbpphlAYG2qRqE1RIrIGMtP1YunUuFm2eidkbpmDcit/wyOh70H1Ed2zauwk/uqMmx793nAk33k3XLb/9B09MfAKvTX8N/x55A+jG+I53FZJflv2MNjXb4O9//Y3zulyGn5b8bLaobfpsA7w3+z28d84nOK/jeYwe3BXqldPeQnxcPFq/0Aw1nqyGh8Y9hKZVmxr/Awd34bi3e2NHyg5zrUN4E3Cc7AZWVlUb3okNlLoOri5fK5CH3IqTQNWqVU0d6zjZ709xRhbisB0nK82Ok2WHOPhSCy5Sf8KlBkwRi4AIBEVAQiVI4KqeVyLl/lQk370Xe12Tct8B3HDkjZ4UDGw+0PVPA93pv+euPdh3zz7XLRXta7cHF1enPJCO/a6bdb/n2HvMtq4p96VmhXlvitkOlruWHPdeP+w/uBfPDXkNIy8fgz+GTzT2i6e+YbavTU7Zju8Xf4+z258NGy7TxvifG/wcHuj3gJuWrPTa+C7sdKFJL3eR+vCsD7HspqXY9N9d2HV3MhZevxDW3wi5h3/1uNK473DzsuX27SZtk6+cjLW3bcXWO/diz72pqJO9RsMV118pEOBoF0fJSiFqRSkCIlAKBKRUlAJ0RSkCIiACkUrgYPpBrE9ajYRyFXBJl0vBnaD6Nulj7Kt7Xo3T2p5mssZpVeakCIdqidXMmoq8guAC7yoJlY0I7doVa6NyQiWUjdcIoYGS76H4BDiSdOqnp+K+sfdj0bZFxReRQhYBEQgLAlIqwuIxKBEiIAIiEBkEuHj6wYGP4GDqfhzxZk88PvFxfDbvM7wx/Q1c+PWFeOOfl9GmThdc6ioc0L+YJkClcMSpI/DMhEfQ47UO6PVWL7w/+32zPiemwSjzIlAYAhFwj5SKIB4Sh/spZm2eR5LZuHEjdu06vGgzktIeyWk9ePAg1qxZg/T09EjOhtIuAn4E/tv3v1h280rc1PtGcEH4yGUjMXPTTLSp1RbfX+KeXzMTHDHwu1EOMUeA62/+umqayff8jdNxl2L48AAAEABJREFU3Q//Qq2na2Ho90MxYc1fxt1Tt2aay8g7rHeTvNc1+itRAvv37zd1rOf9KdHYixaZTbO1ixZa+NwtpSKIZ2EfurWDuCWsRJYvX44tW7aEVZrCLDHFkpx9+/Zh3rx5SE1NLZbwFagIlCaBRlUamXUbT53wFD46+yO8cdobeLj/QxjcarBZQF2aaQsm7v/99T/0fqu3TAkw+L9f/g+Na7TOfiwZQNoBfDn3Ewx+/ziUf8jB7i57gPKud6QqFYscYJubfv2VKIHdu3ebOjYS22aRmOZgHq6UimAoSUYEREAERCAMCIQuCW1rtEW/pv1kSohBzkXzbtMjM0uD6NGoDxI3JwApoXu2CkkERKB0CLi/7NKJWLGKgAiIgAiIQGkR4Pa0T5/4NGSKn0GlhEqYwQ8mZj/sHo2Pxogz3sX2u5Ix6V8TkbDFVSqy/WRFCQFlIyYJSKmIyceuTIuACIiACIhA8RMYs2oMnhj3oInojn73Y8FNKzDpiokY2m0oKparaNx1EAERiA4CUioi7zkWOMUNGjRA9erVC3yfbigagcTERDRt2hRlypQpWkC6WwREQAQikMDKXSvx7aJv8fNlf2L//Rl4qP9DaFG9hX9OnGwna2dfRozVOBNOlYhJbdQklB+YZR3rOJH34jhO5KU5mBdHSkUQlOyXtK0dxC1hJdKqVSvUrVs3rNIUC4mpUKECOnfuDH1ROxaedqA8yk0EYpsAFYhXhryCQS0GwXFyb0Q5TrZfthVx1NoBmbUiLtURn+AqVaqYOtZxIu/FcZysNDtOlh3xDyM7A1IqskHIEgEREIFYIfD5/M9x1NtH5WrO+OwM3PzbzfhywZdIzyidLZGX7liKbiO6odVLrVD/mfoYvXJ0rDwe5bOkCSg+ERCBkBCQUhESjApEBERABCKHQJe6XXDdEdehcZXGmLPhH2M618lyu6zLpWhWrRnem/Uuhn59Adq90h7zt84v8czVqlALl3S+BBt2rUDSvs3YmbKzxNOgCEVABERABIInUNxKRfApkaQIiIAIiECJEOhQu4NZKNujQQ9PfKe3Pc24Xd/7Brx48ouYMDzrw2Trdy3DaZ+ehpLeV71G+Rq4vc/tSEzQejDPQ9KJCIiACIQxASkVQTwcW5laO4hbwkpkw4YN2LlTvXwl/VAOHDiAlStXIi0traSjLkR8ukUEchLoWq8rKleobRw3714NTkcyFzqIQIgJeOrWzBAHXFLBrXUj2uMa/ZUoAX5glnWs5/0p0diLFplNs7WLFlr43C2lIohnYR+6tYO4JaxEVqxYga1bt4ZVmmIhMfv378fChQtx6NChWMiu8hhlBA6kHUBy6r6sXMUngl/Qzro4fFy4bSHemfk27h59N17+5yVM3zj9sKd7xjLzl6W/4OEJD+POP+/EmzPexNQNU3EoPfBvYtu+bfh+8fd4etLTeG3aa1idtNoNRX85CETjRWZ2pqydfRkx1hIH2B4xqY2ahO7Zs8fUsSxnIi1TkZjmYBhLqQiGkmREQAREIIYIbN+/HTeMvAGZafvdXMfhyeOfyPFNAS7epn/P1zrjyUlPuTLAp/M+w7Fv9cKto2411zz8+7d/49zPTsW3C79FYtlEvDDlBfR7+0gc+96x2LovZ0fHK/+8jJYvtcSl316GVUmrMGHNBHCh9oGDuxiUjAiIgAhEFIFYTKyUilh86sqzCIiACPgQGPr95WjxYgvUfLImGj9dG5/MegeIS8Avl/+Bm476dw5pKgfvTHsVFRKrYcqVU/DYoMcwfvh4tK7TGa9NeQ7/rP/HyM/bMs/YbWq2wYP9HsSsa2ahTNlKZmH4K+79xtM9/LTkJ9z+20045I6MTHDDee2U1/D5uZ/jx4t+cn31JwIiIAIiEAkEpFREwlNSGn0I6FIERCDUBJ458RmjIEy7ehqG9bw2K/iMg2CDP+vi8PHVbIXghJYngAuq6VMuvhzO7XguT/H1wq+NzQXfNx1zBx4Z8IhZ6L3/0H70btjb+E3fMM3YPLw89RVa6Fi/O3rU72HOeejpdc5rGREQAREQgfAlIKUiiGfjOE4QUuEr0qhRI9SoUSN8ExilKeMXtVu0aAF9UTtKH3CUZat2hTqoU7GO+eIxRwpa1Opocjhi6stYk7TGnPNAxWBT9vUPC74yoxsc4aB5euLTFMGeg1mrVptXa2GUjuE/DEeVx6ug02ud8c+GqUaGU6jMiXuYvXmWewQaVG5gbB1ijICtYq0dadlvmglok7ISf2oVK1YE61jHibwXx3EiL83BPGApFUFQcpyshx8XF5m4+KOrU6dOEDmVSCgJVKhQAR06dEDZsmVDGazCEoFiJ+A4DlrWaJEVT2Y6Zm+enXXuHiuUrYBqrvLhnuL8Lpdi4fULPWbbHduw654UjDh1BFbsXIGWL7XAg6PvREJ8AiZfORkbbl2P09qcyltzmPqV6pvrjMwMY+sQWwQcJ6uORbYVcblv46Y4wpUKNwcR91elShVTxzpO5L04jpOVZsfJsiMOfi4JjsvFXc4iIAIiIAIxTKBWhVqe3C/ascicr05ajT9X/olBLQaZ62kbpxmFIbFMIqwZMX0Enp/yPD6c+yH27t9m5L447wt0qtPJnG/Zt8XY3gdOo+L1wm0LaXlMWka651wnIiACIiAC4U1ASkWxPx9FIAIiIALhRWBXyi4s3r4Y3MLVpmz9nvXGjdOb6Na6ZmtaxoxcOtLYD49/GM9Nfg7397sf9ao2w6rti3Dr77dic/Jms5sTt4K9a9StaFylMaonHu66/Xvd3+b+X5f9ismrx5pzKiizNs0Ct5e9puc1qOuGtylpFZ6f/DzSMtKwZPsSXP7dZUaWhynrp2DDng08lREBERABEQhDAlIqwvChKEkiIAKlQCCGovx64Tfo/no3vDXjLZQtV9mY2/+4w3XrAqsA3Nj7JlzQdSgQXw7T1v6Fuk/XMwuw7+p7N7ib09//+hvndbkMPy35Gc2frY+mzzbAe7Pfw3vnfILzOp6H6464Dtcd9R9ULF8LF31xFmo9VRuPTHgE31/yG9rW7Yq1u1bhmDePwKR1k0AF5p8rp+DENqe7Mg+j8mMV0dP1i3MOV1HcVerRvx6NoaekrIqACIhAZBE4XGJHVrpLNLX2IyXWLtHIQxDZunXrsGPHjhCEpCAKQiAlJQXLly9HWlpaQW6TrAgUO4Grel6JlPsOYO/de7Hnrj3GJLvnKfel4vgWx5v4K5WriPfPfB9Jd+7Bon+vwp+X/4Htd2zHsU37Gv/6levjw7M+xLKblmLTf3dh193JZm3FhZ0uNP4JZRLw3ODn3Hu2YeMdO7H65tWgIjK41WDMvnY2Uu5PRcoD6ejfrL+Rr1upLn646AfsdMNa+5/1bpp247sLv8O627Pu33n3frw65FUjq0N0EfDUrZnB5SvspFa7KdrtGv2VKIHk5GRTx3renxKNvWiR2TRbu2ihhc/dUiqCeBb2oVs7iFvCSmTVqlXYtm1bWKUpFhJDpWLx4sU4dOhQLGRXeYxSAlQOmlVrhs51O4PbxgbKZrXEamZNRSA/ulUvXx1UUngejKldsTbsKEWtCrXA+8uXLQ/HcYK5XTKRRsAqE9aOtPQvc99L9duV+FPbu3cvWMdGYtssEtMczAP2USqCuUUyIiACIiACIiACIiACIiACInCYgJSKwyx0JgKRQ0ApFQEREAEREAEREIEwIiClogAPY/PmzejcubPHdOnSBd26dcth6OYtw3O6dSsFua5du5q0XXHFFTjzzDPNOdPCNHkbK9fNK43hIMf02Mdzzz33mPTTjSYc0mcZ5savb9++uPHGG9GrVy/zzuQmZ8OxdlHkbBjedqDwvP3tubfce++9Z9HLFgERKAIB3Ro8gXXz1+G6jtcdNp3c884+hm7eMjynW2HleL+vCRSerwyvs+VuvOYGXD/kelzHNNDd12TLGX/K0PjK8LqE5R4Y/IDn4Vx55ZURVcey3ho4cKCpY1l38Zo22wfehu6+Jlg573DsuW9YvLZ+3jbdfY23/2+//eZhH00nUiqCeJqXX345EhISjOT8+fNhzbx58zBnzpwchm7W39p0Kw25uXPnmrStWLECS5YsMedMi02Xta2cdxrDQc4A9zqEW/ry47do0SKsXr0aCxYsMO9MSXC2afK2A8Xr7W/PveUs9ltuuQXlypWzl7JFQAREIOQEzht2nifM9QvXw2MWuOfzfQzdvGV4TrfCyvF+XxMoPF8ZXmfLrV61GmuXrMV6poHuviZbzvhThsZXhtclLOeBnn0SSXUs6y2up1jt1rE8p/Guw2xe6O5rcpGDr5wNw9v2leG1t789p7uvsX60s5Hj5ptvtqdRYUupCOIxXnvttXjttdfQsmXLHKZVq1Zo06ZNDkO3cJFr3bp1jrQxrYHSJ7k2EJeW8H0Phg8fjmeffRZly5YN4lciEREQAREoHIHH7n0Mtzx4C5o0b5LTtHCvW/oYukWDnG8eeM28+eaX7r4mxHLNWjaL2raCb3uM1751HdtGdPc1wcrxfl/jGxavfWVuu+02PPpodG2TLaUiyDKQU4i4Pai3WbZsmRkB4CiANXTzluE53ay/telGP29DN+tvbbp5y/Ccbtbf2nSjn7dZunRpUOkrdjl3lETpW45I4/zuu+8G+euQmAiIgAgUnkBimUQ8+8CzWLNyTU6zwr1e7mPoFg1yvnngNfPmm1+6+5oQy61avipq2wrebSJ7Hqgutn7edrByth3mbXuHY8+9/Xn+9NNPF/5HE6Z3SqkI0wejZImACIhASRNQfCIgAiIgAiJQWAJSKgpLTveJgAiIgAiIgAiIQMkTUIwiEJYEpFSE5WNRokRABERABERABERABEQgcghIqfB9VroWAREQAREQAREQAREQAREoEAEpFQXCJWEREIFwIaB0iIAIiIAIiIAIhA8BKRXh8yyUEhEQAREQARGINgLKjwiIQIwQkFIRIw9a2RQBERABERABERABERCBwASK7iqlougMFYIIiIAIiIAIiIAIiIAIxDQBKRUx/fiV+ZIioHhEQAREQAREQAREIJoJSKmI5qervImACIiACBSEgGRFQAREQAQKSUBKRSHB6TYREAEREAEREAEREIHSIKA4w5GAlIpwfCpKkwiIgAiIgAiIgAiIgAhEEAEpFRH0sEoqqYpHBERABERABERABERABApCQEpFQWhJVgREQATCh4BSIgIiIAIiIAJhQ0BKRdg8CiVEBERABERABEQg+ggoRyIQGwSkVMTGc1YuRUAEREAEREAEREAERKDYCES8UlFsZBSwCIiACIiACIiACIiACIhAUASkVASFSUIiIAJFJKDbRUAEREAEREAEopiAlIoofrjKmgiIgAiIgAgUjICkRUAERKBwBKRUFI6b7hIBERABERABERABERCB0iEQhrFKqQjDh6IkiYAIiIAIiIAIiIAIiEAkEZBSEUlPS2ktKQKKRwREQAREQMudiEYAABAASURBVAREQAREoAAEpFQUAJZERUAEREAEwomA0iICIiACIhAuBKRUhMuTUDpEQAREQAREQAREIBoJKE8xQUBKRUw8ZmVSBERABERABERABERABIqPgJSK4mNbUiErHhEQAREQAREQAREQAREoVQJSKkoVvyIXARGIHQLKqQiIgAiIgAhELwEpFdH7bJUzERABERABERCBghKQvAiIQKEISKkoFDbdJAIiIAIiIAIiIAIiIAIiYAmUtFJh45UtAiIgAiIgAiIgAiIgAiIQJQSkVETJg1Q2RCC0BBSaCIiACIiACIiACARPQEpF8KwkKQIiIAIiIALhRUCpEQEREIEwISClIkwehJIhAiIgAiIgAiIgAiIQnQRiIVdSKmLhKSuPIiACIiACIiACIiACIlCMBKRUFCNcBV1SBBSPCIiACIiACIiACIhAaRKQUlGa9BW3CIiACMQSAeVVBERABEQgaglIqYjaR6uMiYAIiEDJEPjhhx/w0ksv+UWWmZmJjIwMj7v3ucexmE8OHTqESy65BLt27QpJTOnp6TnCKY085UiALkSgGAgoSBEoDAEpFYWhpntEQAREoIgErrjiClx22WV+5rbbbsO7776LTZs2FTGG4G//6KOPcNJJJ6FXr16YNm1a8De6kmPGjME999wD5se9NH8bNmzAueeei759++LYY481YY8fPx4XXnghFi9ebGSCOezevRtnnXUWjjvuOFx//fXB3OInU7ZsWVxwwQU49dRTcfDgQT//YBx+/vlnk5/u3bujT58+Jj/PPvssqGCceeaZmD9/Pj7//HOcfPLJhiGZBBNuKGTeeustk54jjjjCpKOgYV511VV+76B9L++44w6Tr7S0tIIGW+zyKSkpOPvss9GvXz/861//Cml8W7duxTfffAMqxSENWIGJQJQTkFJR4g9YEYqACIgATEN80KBB+Pjjj01D+9577zVugwcPxujRo9GmTRuw4VoSrE5yFQqa6dOnF6hHn43+oUOH4tNPP0WlSpVMUvfv328aep07d8akSZOMoZJ0880346uvvgL9jWAQB4b58MMPY/369ViyZEkQdwQWOf3009GzZ0888MADgQVycd23bx+uvPJKM9IxYMAA/PPPP5gyZQqoZOzZswfM408//QQ2cE844QQwHjLcu3dvLiGG3vmMM84wvGfMmIGkpKQCR3DXXXeB7xzfwxUrVuDuu+/GQw89ZN7Fjh07GmWua9eu5hkUOPBivCEhIQF8N6gALFq0KKQx3XjjjUaJ/O2330IargITgWgnIKUi2p+w8icCIlA4AsV8V8uWLU0PM6Np3Lgx2rZti3bt2oGN008++cT0iHPU4ssvv6RIsZratWvjtNNOM3EUpHf2scceMyMRXbp0Mffy8Nlnn2Hjxo34z3/+w0tjGjRogF9++QXlypUz18Ee4uPjTcO9W7duwd6Sq9ydd96JV199FWvWrMlVxtfj3//+N9555x288cYbpnFt01+mTBnT8OazsvfUrFnTKBX2uqTsOnXq4JRTTil0dC1atDAjLAygefPmaN++PejGd5EK4xNPPIGFCxfiuuuuo0jYmLi4OHTq1AkcoSnIOxtMBjhSc+aZZ8L7vQ7mPsmIQKwTkFIR62+A8i8CIhCWBNgzzoR99913tIrdOI5j4nCcLNtc5HFgQ+7DDz/EDTfckEOKDVA6sNFH2xoqFoVt/DpOcGmycQWyGT+nJ33xxReBvP3c/vjjD6NQcMoWjZ+A6/C///0P7DF3T82f4xQ9nSagAh4cp/ji7d+/v0kNR2nMSZgdHMcBTSiTxaly/N01bNgwlMEqLBGIegJSKqL+ESuDIiACkUiAU2+Ybs7jp+1tuPiYax84j3/lypXeXuacc/059YjTdDg9xDgGODCOqVOnYtu2bQF883ZiI5PTfo4++ugcgh06dDDTgW655RYwfG9PLphu1KiRcdqyZYvpAee0HePgHphWTmVh2O5lwD9OhaJMQM98HNnzzMZiPmLG+/XXXzf2pZdeauxAh8qVK5t1I4H8Dhw4AE6F4rQt78XcweZ78+bNZo2EXYOSnJxspl8FM8WJayD4jlDxox0ofcG6/frrr0b0xBNPNLb3gVPA/vzzT4wbN86sL/H2K8588p2fPXu2X5ze8ef1G+B7yalenMrGd5is58yZY27nupvVq1ebZ7dz507jZg9kmdvvjsz//vtvcKSO7++8efMKvYbHxidbBCKNQD5KRaRlR+kVAREQgcgnwIW/nHJz1FFH+S1C5UgA57hzihQbRJzHz2k6Nte///47Bg4ciFWrVoEKAxvSb775pvU2NhtVHGE45phjwB75++67z2/EwQjmceBaAi7s9u0l5mhE3bp1wQXErVu3xvDhw/Hee++BDbRzzjkHnK7DBhgXT3Na0/nnn++J5fLLL0ePHj0CTudhfo4//ngzesAREipbXMvhuTmIE65TYYOPykt+4jNnzjQiTKM5yeXw8ssvm6lr3t4LFiwAla0PPvgAnEZz5JFHgnmmCSbfVo4srrnmGrO+gQoZFR2OuHBtind8vudPPfWUmWrG0SIulvf1D+aaCgmVhSeffNLkhVPdvO/j8+U6laVLl4KGC9iff/55I2LT361bN+T1fK1csPkcP368Yf3oo4+C7zmnn1llwEScfaBfXr8Bpn3AgAEmX9y1jIyZVi5M573cGIDv9tixY7NDhFGAc/vdcWcxPuNZs2aB7z6n+jE8/o49AehEBGKAgJSKGHjIymIMEFAWI5oAez+5IJhzuXv37m3WEXBNwoQJE8C5+jZz7EXlwlw2qF944QXTYOeaCzaMbAPm8ccfBxu15513nlnIevHFF4MLT9nDb8Phjj/sYZ44caJpsI4YMcIs1qU/G5O08zPsLeY6EF+5evXqYeTIkWZePnewev/9983OUByhoCJCea5JGOf2blPJ4LU1XBh70UUX2cscNkdT2IDnYmvm8bnnngPn/DOuHIJ5XFCpYP5Wuz3ReYiZERa79sKbf6B7uAahSpUqObyoxP31119geqn8ccSC6Qw235RjA3rIkCHgO8BpONy2lyypWDH/OSL0uWjSpAm4yJr3Uony8c71kmnme0hDZYjvENfb8B1jmPZGjoBxjQXz9H//93+4+uqr8e2334LKKXcSY/qDeb6UCzaf7PnnaAnXGXGdCxUA7tDEERw+U5s22uST12+AmwZQMaUsRxYYDllT4eXaIioW9LMmv98d88xpcNyhjMoMF5BTebT3yxaBWCEgpSJWnrTyKQJhRoA97tzlyNtwGoJvMtk48JbhOadW+Mqx0UE/b8OGqK8ceza9ZXi+Y8cOXzGw15F+3oY9kn6CIXBgr+jbb78NNk44QsEGIRs93F3JO3g2fpYvX252prHunG7UqlUr2Gkq7O3n6AUbOZRhwz01NRV2yhDzxYYUG46cvkMZGm7NSTtYw97+qlWrBhRnzzN7rzm9hD3d3OWKoypsaNnpPLyRuzvR9ja5NeK5cJgNeCvLnma6Pf3009YpX5sjJxQK9P7Q3ZrExESwwctrjurQLojh6IzNG58N7yUv2jTWj+fWBMo3G7ncEpdKoJVjeN5hWXdrc6ctPl+y53a+1j0Ym/J8D2moxPD3w5EGLuBmQ9mGwZ3KOBJjedKdIygcOeDz5jVNKPPJOCtWrIhhw4YxaGOqV69uFHBz4XXI7zdAUcubu57xWXN0gQqLtx/PafL73fG3NnnyZNx6661mihpHYPjMuBkD75cRgVghIKUiVp608ikCYUagVq1aaNq0aQ7DRoNvMoOVY4+qb3gVKlTwDc5Mv/GVK1++vJ8cpzH4yrGx6ScYYgeOQrz22mtGCWBDyjt4ziPnNXus2VNsDUcH2GinHxu0XAfA3mM28tizT3cqFrRtzzWnRfHa1/hOZ/L1t9fsIQ7EbdmyZWaqD3du4pQQ9ihzVIRTd5jGH3/80QaBYOPy3OBzwpESNp45hYu96t6GOz35iIPvV7Vq1cyaD18/72um3TaYV+czqsGF3+vWrfO+HWyEWweGxXP2dtOmKUi+OcJDxYL30TA877DoZs39998PNmbZyA3UoLdyBbH5/DgSw/eIU8K4VoSjYtyxzDccjmawE8Aq6aHMJ99bKpHeLGz8vvHk9xuw99HmFEDaeZn8fndUYvhb4+gZpyxS2eHUKXLLK1z5hTUBJa4QBKRUFAKabhEBESg6AfbCstfV23j3nNsY2Lj3luF5oAYTG9b08zZsRNpwrF2/fn14y/A8kPLBnlf6eZtAjWgbbiht2yPvOw2jRo0aJhruOsRGujWctsIGJT25QJojFZwmw8Y2p0nR3Rq7Laq9LqzNnt5A32NgfFyc7BsuR0aoIHKkxNfP+9p7UbO3e6BzytL8P3tnASVHsQXQmo+7uwW34O7B3d0hWHCXBA1ugQRycAluwTVISHAnBAlOEgguwYLD31uf2t+ZzO7MbmZnd2bvOVvb3eV1q+ec96req8Z3gP5kQ0MCHTsPKBaF6svGoZjxnHaAuC8U2BXIf2/zhVzK5ZvoEJcNjCP7nO6bUhf+E/hb4IzObkOqY2yurOKzK0EdKHD0B8EeR2jisiGNgZX6bHz2PuXJxnFPvVyzIcusKe9tsd9Ato30m8rG5d+nPA397tjBZKcDpYv3H3+KU045JfoS5dflswRqmYBKRS3PrmOTgASqkkAyK2KVnBN2GASr/Jj8cM8pM1xTQFBGuGWXAGdZvnCN2QrpqTz33bt3D+yEcM+KMtcUsgJcimvsilKIeUx+Hurp27dvfnQ0J0L4zq7iI7CSP5sZ5/Lsc7rPz0c8pmyYWmFehV9INuCfQp5sQAniBB/6no0vdI8fCo7ImFdxUlChPOxQMC+lKCnZ8k0Zd7ZcsXu+Ys5XptmtwG+A96FYmWLpjA/fG/Kh7LILwmo85obEZcPgwYMDuxUsBBBfznF26tQpOoTnKyz57wVjLvYbQAmgf6WGYr87HL8J/LY4NAEln10zTKJKbaOd5HOYNU5ApaLGJ9jhSUACbZdAEvjzfSdYlWVHBYEJJ19GgLCI7wXX3r17h2wZ7N0xiUEJIS/luBKwjeeK4oEgiLMy5hoIQQjZpJE/mQsV8zcgPwEBCqGe+/yAoyxH2mbjEbQQwjnFKMVjXjZixIhoLkUcjt0IsPQLYZa4FBBYk18IcazIM94ePXrwWFKgv+xKYTZVrAAKELsU7IDhC0LfsmVQqFBc+Dhcik/HvWbnJsWNGjUqZYsmf6WMm7K8I8xPKkwcilFWuCaO9HQ999xz4+lP+NPQT9IaC6m/vEPZfLTRtWvX+K7hwIxgT/pJJ50UcNRPjvfE4XCNskvbPBNKnV/6XWyc7D4wbkz/qJuAYoxDePad5Z0gLcss/zfA74s2yUe7XLMhpWHiR3yx3x27LxyWQP/IT6B9HMu5N0igvRBQqaiVmXYcEpBAVRFYd911Ax/ZQnhFccD/gJX2NIhevXoFTHgQ6jC7QBmdADC1AAAQAElEQVTAzIdjVDFzwcEUQYtTehCScGBlRZXTeDjClZNo2K3g5CBMj/C/oD5Mwjg9h1NqMJPCbwPhM/lcHHTQQdHhNPWjoSvtYlufL/yTH8UBW3zqx86cNviAHIoMTujkIXTu3Dng24HQzi4KR3ty1C196dixY8AXg3wElB76duSRRwaUIgTbfv36hVJs4ilPwC4fQa9UMzb8dDAh69ChQ8DHokuXLoF5oR98SA+FAuWKujE34jhd+GL6wnzgbM/xpMSx48HqNXmLjZtxUS/mb/CFGavemCGxC4Tp0SKLLBL5oMDBmza4ItRzmhgKAoI/+XA0pt1CAV8A+kgaSgK7MzDlimKL0M4RtbSbTJRw6sfECkE6+S+wO8JBA2mM1FfOcWJShD8OSgX+Qrz7xxxzTMAZmnHCCLO7Un4D9BtWMKMuxkN/CZzcxTtCWrdu3QLvb7HfHX4u7H6xa0a/OEkKJZLfCHUaJNBeCKhUtJeZdpwSkECbIJA6geDICUmslLJSjMCL0JrSEc5Y2edoWQRDhEXSMD/Bl4KdAAQiHIURYknD3AThm3oxw0DIQwBGmMcshJOBUj52OxDa8R3gyFBOnWLlF8Ese4IP+QsFVqER/NkxyKajvOBoTv84UhWBfOmll46naSFoZ/MitCMAo0wh5CE0YrpFX/koHmZN5EcoRjFCyIYRwjx9xW+E9FIDHwvkSN5S85MPcx4Eak6toiz29QjQmGlhBkQeAv3jWxqsbrNyjvCL8oPpFHHsGCRWxcaNUIvvCSvmlOWkNBQKFAtMd4hj14bxo3QOHz48EMf7wglGcGNnijhOVENppI+FAkoTxw2zso4iAnd2HLjSZ5yUUeRQXLPlUaooi7KE0ss7h+KYzVPOcVIvwj87WbzvKK0oQbwbcMffgx2oUn4DWWaMnV0W6ifwm6I+2LGbhJJAfGO/O0zl+vfvH+gDShbKJD4W+cyoxyCBWiagUlHLs+vYJCCBqibAzgKCGsIszrHZwbB6ymk4CDvZeO5ZUWZnI5uGMExaNrDCSh0IYqzeczwtK648Z/M1dM+uAgpBNh0FKLWL4oG5Fiu21JvNl71H+MQhnjjMjbjHT4FxEJcCz6xM40Sf4kq9IvAhYLN7U2qZbD5OYUKQxuSJMcI/m96c+1LH3Zy6K1UG53sUx8baK/c48dngPaFNfHRonx0/nlPgXSnlN5DyF7gWjGLe+c2kd5xMnPbEb4bdDd4NmBBvkEB7I6BS0d5m3PFKQAISKBMBdh5QTDD9KVOVLVIN5lTs2ODAi09FizRipRKQgATaOYHWUSraOXSHLwEJSKAWCLAajEkPduf4V7TVMWEShA8JpyO11T7aLwlIQALVTkClotpn0P5LoAUJWLUEihHADAXb+mHDhhXL2irpnMiDMzh+GK3SARuVgAQk0E4IqFS0k4l2mBKQgARaigAnTHHyUUvVPzb14ouCE/XY1FEFZe2iBCQggVYnoFLR6lNgByQgAQlIQAISkIAEap9AbY9QpaK259fRSUACEpCABCQgAQlIoMUJqFS0OGIbqBQB25GABCQgAQlIQAISaB0CKhWtw91WJSABCbRXAo5bAhKQgARqkIBKRQ1OqkOSgAQkIAEJSEACY0fA0hJoGgGViqbxMrcEJCABCUhAAhKQgAQkkEdApSIPSKUebUcCEpCABCQgAQlIQAK1QkClolZm0nFIQAItQcA6JSABCUhAAhIogYBKRQmQzCIBCUhAAhKQQFsmYN8kIIHWJqBS0dozYPsSkIAEJCABCUhAAhKocgIlKRVVPka7LwEJSEACEpCABCQgAQm0IAGVihaEa9USqDABm5OABCQgAQlIQAKtQkClolWw26gEJCABCbRfAo5cAhKQQO0RUKmovTl1RBKQgAQkIAEJSEACY0vA8k0ioFLRJFxmloAEJCABCUhAAhKQgATyCahU5BPxuVIEbEcCEmgHBP7666/RRvn333+P9uyDBCQgAQnUBgGVitqYR0chAQlUGYHXX3897LbbbmGfffYJ++23X7yef/75cRS333576Ny5c+jSpUtM23PPPUO/fv1i2qhRo+rLUZa0mNDAv0cffTS8++67DaSWEt30PPfdd1/YaqutwhJLLBFWWmmlsN5664UePXoEFIzNNtssvPHGG+Hmm28O66+/flhmmWVC//79m95IM0p8//33YfPNNw+rrrpq2H///ZtRQwh77bVX2HnnnQuGo446Ko7rzz//bFbdLVnol19+CVtssUVYbbXVwh577FHWpr788svAO/vPP/+UtV4rk4AEqouASkV1zZe9lYAEaoTA3HPPHQ477LAwcODAcPHFF4cpppgibLnllnF0CL0Iv9dcc01M22CDDaLwTeKEE04YBfannnoqDBo0KOy+++5EFwxDhgwJa6+9dhSA8zM8+OCD+VFj/fzzzz8HlJwdd9wxrL766uH5558Pzz33XEDJ+OGHH8IiiywS7r333oCAS7822WST8NJLL4Uff/xxrNsupYJJJ500nHzyyeGTTz4J77zzTilFxsjTtWvXsO6664brr78+fPDBB6Fbt26he/fu4dhjjw0LL7xwVFYWW2yx2MYYhVsxYoIJJohjRwHgvShnVw488MD4Tj700EPlrNa6mkrA/BJoZQIqFa08ATYvAQm0TwITTzxxWHTRRcMBBxwQAUw00URh9tlnj/fTTTdd2HjjjeMKPxHTTjttmHrqqbkN//nPf8JGG20UEFxvuummuBMQEwr8m3feecP2228/hlLx8ccfhzPPPLNAibGLOvjgg8OVV14ZLr300ihcjz/++LHCcccdNwreKBIxou7fNNNME1Aq6m4r9jfOOONExWbxxRdvdptzzTVX3GGhgjnnnDMsuOCCgbgFFlgg7LrrrpHrW2+9Ffbdd1+ytJnAe9OxY8ew9NJLh3LvKLBzww4U73ObGbAdkYAEKk5ApaI05OaSgAQk0CIE2JHI5XKhb9++Y9Q/cuTIGJef9ttvv4VvvvkmCrMxQwP/EOZvvPHGesUlZXvmmWfSbdmujzzySFQotttuu0AoVPGpp54aWDFPablcLt1W9JrLtVy7nTp1imNhlybetLF/uVwu5HK5svYKJffOO+8Ms8wyS1nrtTIJSKC6CKhUVNd82VsJ1AwBhNBevXqF8obK1PfAAw+UbR4QxFZYYYXA6vbbb79dXy8KBSvKrPbfcccdo60uP/zww2GdddaJcZ999lkYPHhwNOfBlv+VV14J3333Xaznq6++ivW+9tpr8Zl/+FdgqvP7779HsyNMj/744w+S6gN13n333dGPg3z1CY3cYMJF8k477cSlYJhsssnCyiuvXDDt119/jaZQmCVlnbm/+OKLOIaXX365vlwy4ckK7p9//nn01UgMf/rpp2h+Bcf6gg3c0DZtEvD7aCBbSdHJrIz5yS+ACRg+LgMGDIj+Jdn0lhznhx9+GE3lGhsb7fOevfjii/G9yvYNszZMvTBlw3QN1umdQsEdOnRonLtvv/02WyzwXlEf/jP0IZvIu4pyy24b84iPEXVl83gvAQlUFwGViuqaL3srgZohcMMNN4RDDjmkKsPVV19d1nlIvhTZHQl8DzArWXPNNcOIESOib0JqFOGPMgjM22yzTVhyySXD8ccfHzbddNPo0zDbbLMFFApMn3CWToL+m2++GQ466KCAwzJKzNZbbx0ITz75ZKwaJeaEE06I/hAoG9jIY+JDuZihkX8oMyQXMy268MILw/zzz0/W+kD9KFb4kGBGs9xyywWETsK2224bqJNxpgK77LJLHPOGG24Yo1I+OOC8jtKEXweKzswzzxxuu+22mK/QPwRfTLEwjcLsDCG3UL5icbBDWTjrrLMCYzn99NNHK3L55ZeHpZZaKjrNo9jhwJ4c81P/yz3OgQMHRtannXZaQBHF/CwpA6lzjJ/dsp49e0ZflyOPPDKsssoqASUj5aHv+MgwrgsuuCAeKkBfcUynXnyAcLh//PHHU5GoCGKix+8cRQRTN8zjyIDSu1zdHL/66qthhhlmCPfff3+cYxz4STdIQALVSUClojrnzV5LQAI1RAAFgeFklQp2ChCwOUUpm4YQiKNxhw4dwlRTTRVQCFj9R3DGSRoTo/nmmy/6XrAqjsBIeQKOxCgKCICLL7544J6wxhprkByuuuqqcMYZZwRMWVBEEHqp+8QTT4zpDf1jJXvYsGExGQE93jTwDx+EySeffLTUyy67LI4DhQMhFOdtdoMw30JQT3xSIfqMwpSeyYcAjUP7E088Ec1w4NenT5+w1lprxTGlvPlXBHp8T44++ujAGBh3fp6GnmEPcwJzhYKGYnLrrbfW+8dQFkd1fCwYEyd97b333gHFEEXwuuuuC/R/QN3uRTnHyco/uyVHHHFENEtDAeCEJnZwUIDoF4HdgrvuuivQbxQx8rzwwgvRB4Z0wiGHHBKuvfZabgNKF3lgPf3000ffHxSLmPjvP3YcUHBRaFFWOEwAJigkKA6MGTM4TuDi3cN5HuXx3+JeJCCBNkSgKV1RqWgKLfNKQAJlI4BAhUBTjYGV37KBqKtojjnmiA60rCK///77ASEdwQwBHeEMoRMhtC5rYDUYQYz7FMjHSjsr0Qh77BoQR3q6cl8ssMq+0EILRefjlJcVZpQThO8Ul3/lRCr6SDx959qUgNDJyUyUmWeeebgETJziTd2/lFZ3W/9XaFwIueONN1489jVlpL5sXSmeK7s5HGuLUIvjeqF2yNdQYEX/iiuuCASUGOpjR4XdHepM5Y477ri4e4HykuLYQWG+YJ7iCrXf3HHS5iSTTBJ22223VH1UQjmBqz6i7obdEw4FYJen7jHQHsfO8i7ynALx3HM8MHPN7gIKC3EpjXsCSgfvcVKIieO9Yi4wD0OhePbZZ8Phhx8eTdR4tziqlxPRyGuQgASqk4BKRXXOm72uCQLtexAIEJhTVGNgJ6Dcs5cEMAQyVrRRDmgDga1Tp05h6L9266Tnr2iTj5N3CgmlpJUSUAYQBL/++ut4chEr64R77rknMF4E5obqQaFJAjP9bCgf8bfcckvg9CnuU0AIT/fUxT1KFVdCLle6Y/Gss84aUCwoR6C+bF3EERjriiuuGFBq8WshrhyBHQF2YtjdQbnDX4PVeUzS8uvntC9Mv3C6Jy2XK9848VPgRKosC9og5HL/b4d3BrMzfB9451AiMY1ryJcGZtTRWBg0aFBMZqeIdyiFGWecMZpYYb6GQnXeeeeF5ZdfPio7KMtwiwX9JwEJVCUBlYqqnDY7LQEJ1BqBpChgAoX5UdZsKaVhQoLDLEJ+/vjTkbP58cWecU5GUWH1mJVthDx8EVJACcAcZqaZZmq0qmQ2xEp0Yxkxn8FhO5snl/u/kJvisyY6KS57pd/Z53Sfy5VWF/Wz2o4ytsMOOwQ+KpjqGOPahAhW8VGUKYJjcy6Xi0pOISfpNAZW6slfKKQ8+Wm5XOPjxME/v0yhZxQezJQw8VOpqgAACZFJREFU5eI4YJTI1P9C+Ut5z1IeTPHSe8SVNvDZwacC9ihdPXv2jP4Up5xySii3r1Kh/hsnAQm0HAGVipZja80SkIAESiaAaQiOrfgT4DPBqm4qjILBdwbOOeec6ESb4pt7pS6EasojTGNTjzCM/wSCcEojnUB6Wk3nuVDgA2iY0tBHFJ9CedihwGl3yimnLJTcYBx9y+8Tik6DBUpIoK8oZyg5OMIfeuihJZQqnoXx8WFCcuI/grKGooaPA3HZwKld7FbgrEx8OcfJ7hYO4fkKSz5HzL44lQoOSXHkmf4w5wj73Dcl4NRNfnaBuKbAbhjt4PhNQJnBeRtlA38UTKJSXq8SaA4By7QuAZWK1uVv6xKQgATqCaQdiS222KI+jhuETgR+7lMe7lMYOXJkSIJgiktX0nDOTc9c+QDa0KFDA6vgKDGs1hOPnwv5syvGn376aeAkoymmmIIsDQZ2H9ilQBnC6ZZjabOZMZ/iNCuE2BRPW9xzGhVXQopD2eGZgM8Jgn8SkKkbwZ0TqhDiyUOgLByygjNxOLensikf8dxjtoWTNs7ifLSPuGIh9TffrIo2unbtGk/Xwk8BwZ66TjrppMCRqpzoxTMBx3KE7nPPPZfHGMo5zsMOOywe6dqnT59YN/8wtRowYMBoJzt99NFHAV65XI4sAbYI9ygAzD2MSUi84MtzNqS09J5xEADvcO/evSOLlBc/E5jx3uG0zbykNPqAY3l69ioBCVQfAZWK6puzVuqxzUpAAi1NICkMCGT5beFzwVGsHTt2rE9iJZlTnFi1xw4eZ1tO1iEDQhorwZiZ4KiMIpGEYWzaca7mxCJW6NkJoQz28jgcI4iycowjLYoANvesopOnscAJQnx/pEOHDgFhvUuXLvE7JBxji0M0CgV9og6cmzkSFpMrVsNxKOaEIY4nJY4dD/pA3s6dOwcUH5SV7t27xyNNDzjggIDdPzz69esXqJdTiFAyOOUKwRgzHszJMD3CQRlnd3jhP0CgDEwQ8CeaaKKAIJ4UAdotFPAFoI+koSSw4wE3rqz0D6gT2s8+++z4McNc7n+COo7PmLQhSOOUjqkYjsnMVRoj9ZVrnDjWM05MmZhLTptibMccc0zAlwkFB0Z8EwSFkfcKB2xOeSIvO1McKctHDHnvOBGMk6GYF9IZD/0loIyhDJDWrVu3gJ8EO2F8dBFFmHppm3cNkyzmGT8XnOp5X0nDlwMlkjzUaZCABKqTgEpFdc6bvZaABGqQACfkIOBiEpM/PAROBLVsPE7cOMWymswqMTbqO++8c8ySy+UC3wFgFZlVZ3Yk0m7DXHPNFT8Uh5CP6QmmV7FQ3T+OYOVY1osuuiigcDz22GOjnQZVl6XRP/qOIM9H6Di5Cvt6BGgUH8yAUmGOYWWVnH4jwCL8ouxgOkUcOxsck0t+lBWUgEsuuSQg3CIs8y2K9957L/BRPITaNFbKomChUKBYYL9P3JAhQwLKGrxYbYcZZWCCEI5QCyeUAtpsKKA0YZ6G0saqO+2z48CVPlM/33pAgM7WgVJFWZQlfA0wTUJoz+Yp1ziZQ+pF+GdHB6UNB3KUIPxn4I6ZG8oEebgnDQGfK99GGT58eHx/ll122cDuC89wZOy8o9RPQMmgPtLYTaIO4jH7wo/i6aefDuTBN4d+kIapXP/+/eO3V1CyUCZRfvOZkdcgAQlUDwGViuqZK3sqAQm0AwJpFTx/qKwEs7KbH9/cZwQ4lBhO/ylUBwIuJykVSisljrII0ux0sEvA6nUp5RrLQ5+SAoSZFff4Z+Ry/9sRaKxsW0mbdtppQ4cOHRrtTrnHifkcvGiUk7ZoH3M1nlNAuUIhTM+8b7wj6bm5V+adU6hQMlIdfF+FnS/a4N2ASUprT1fHKoFaI6BSUWsz6ngkIAEJSEACEpCABCRQYQI1qlRUmKLNSUACEpCABCQgAQlIoB0TUKlox5Pv0CXQ6gTsgAQkIAEJSEACNUFApaImptFBSEACEpCABFqOgDVLQAISKEZApaIYIdMlIAEJSEACEpCABCTQ9gm0ag9VKloVv41LQAISkIAEJCABCUig+gmoVFT/HDqCShGwHQlIQAISkIAEJCCBggRUKgpiMVICEpCABKqVgP2WgAQkIIHKE1CpqDxzW5SABCQgAQlIQALtnYDjrzECKhU1NqEORwISkIAEJCABCUhAApUmoFJRaeKVas92JCABCUhAAhKQgAQkUCECKhUVAm0zEpCABAoRME4CEpCABCRQCwRUKmphFh2DBCQgAQlIQAItScC6JSCBIgRUKooAMlkCEpCABCQgAQlIQAISaJxA21AqGu+jqRKQgAQkIAEJSEACEpBAGyagUtGGJ8euSaCtEbA/EpCABCQgAQlIoBABlYpCVIyTgAQkIAEJVC8Bey4BCUig4gRUKiqO3AYlIAEJSEACEpCABCRQWwRUKmprPh2NBCQgAQlIQAISkIAEKk5ApaLiyG2wUgRsRwISkIAEJCABCUigMgRUKirD2VYkIAEJSKAwAWMlIAEJSKAGCKhU1MAkOgQJSEACEpCABCTQsgSsXQKNE1CpaJyPqRKQgAQkIAEJSEACEpBAEQIqFUUAVSrZdiQgAQlIQAISkIAEJFCtBFQqqnXm7LcEJNAaBGxTAhKQgAQkIIECBFQqCkAxSgISkIAEJCCBaiZg3yUggUoTUKmoNHHbk4AEJCABCUhAAhKQQI0RaJZSUWMMHI4EJCABCUhAAhKQgAQkMBYEVCrGAp5FJdDGCdg9CUhAAhKQgAQkUBECKhUVwWwjEpCABCQggYYIGC8BCUig+gmoVFT/HDoCCUhAAhKQgAQkIIGWJmD9jRJQqWgUj4kSkIAEJCABCUhAAhKQQDECKhXFCJleKQK2IwEJSEACEpCABCRQpQRUKqp04uy2BCQggdYhYKsSkIAEJCCBMQmoVIzJxBgJSEACEpCABCRQ3QTsvQQqTEClosLAbU4CEpCABCQgAQlIQAK1RkClonkzaikJSEACEpCABCQgAQlI4F8CKhX/gvAiAQnUIgHHJAEJSEACEpBAJQioVFSCsm1IQAISkIAEJNAwAVMkIIGqJ6BSUfVT6AAkIAEJSEACEpCABCTQ8gQaa0GlojE6pklAAhKQgAQkIAEJSEACRQmoVBRFZAYJVIqA7UhAAhKQgAQkIIHqJKBSUZ3zZq8lIAEJSKC1CNiuBCQgAQmMQUClYgwkRkhAAhKQgAQkIAEJVDsB+19ZAioVleVtaxKQgAQkIAEJSEACEqg5Av8FAAD//3xPUwgAAAAGSURBVAMA7rTqMbfaEPUAAAAASUVORK5CYII=)" + ], + "metadata": { + "id": "SJ5y5VS4rL4Z" + }, + "id": "SJ5y5VS4rL4Z" + }, + { + "cell_type": "markdown", + "source": [ + "The chief drawback of this feature is that it must be specified at **save** time, meaning that you must have some insight into the various ways in which the checkpoint will be loaded with resharding in order to optimize.\n", + "\n", + "If such patterns are not known in advance, or are variable depending on the use case, it is still valuable to specify a `chunk_byte_size` 10MB or greater." + ], + "metadata": { + "id": "fRdXdLxcs6oo" + }, + "id": "fRdXdLxcs6oo" + }, + { + "cell_type": "markdown", + "source": [ + "The `StorageOptions` config can be used to limit the size of chunks. This\n", + "config can apply to all arrays globally, or can be used with `scoped_storage_options_creator` to apply differently to individual weights." + ], + "metadata": { + "id": "txIvTnaaFTVX" + }, + "id": "txIvTnaaFTVX" + }, + { + "cell_type": "code", + "source": [ + "import inspect\n", + "\n", + "doc = inspect.getdoc(ocp.options.ArrayOptions.Saving.StorageOptions)\n", + "print(doc)" + ], + "metadata": { + "id": "RpJKQFMKqS1r" + }, + "execution_count": null, + "outputs": [], + "id": "RpJKQFMKqS1r" + }, + { + "cell_type": "markdown", + "source": [ + "In the following example, we limit the chunk byte size to 16 bytes. For a 128-element array sharded across 16 devices, the shard shape (called `write_shape` below) is `(8,)`. Limited to 16 bytes, the shape becomes `(4,)`, as you see below with `chunk_shape`." + ], + "metadata": { + "id": "uSGcefQ87a_p" + }, + "id": "uSGcefQ87a_p" + }, + { + "cell_type": "code", + "source": [ + "arr = jnp.ones(\n", + " (128,), # 128 * 4 bytes\n", + " device=jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(jax.devices(), ('x',)),\n", + " jax.sharding.PartitionSpec('x'),\n", + " )\n", + ")" + ], + "metadata": { + "id": "2-uTR8bg6lpy" + }, + "execution_count": null, + "outputs": [], + "id": "2-uTR8bg6lpy" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "with ctx:\n", + " path = root_directory / 'ckpt_not_subchunked'\n", + " ocp.save(path, {'a': arr}, overwrite=True)\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "b0XeIJdB2F3I" + }, + "execution_count": null, + "outputs": [], + "id": "b0XeIJdB2F3I" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.storage_options.chunk_byte_size = 16\n", + "with ctx:\n", + " path = root_directory / 'ckpt_subchunked'\n", + " ocp.save(path, {'a': arr}, overwrite=True)\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "VYfd4rIz2sgg" + }, + "execution_count": null, + "outputs": [], + "id": "VYfd4rIz2sgg" + }, + { + "cell_type": "markdown", + "source": [ + "Let's imagine our model has the following (32, 32) array, with shard shapes of (4, 16):" + ], + "metadata": { + "id": "FJSJiUBZ615K" + }, + "id": "FJSJiUBZ615K" + }, + { + "cell_type": "code", + "source": [ + "arr = jnp.ones(\n", + " (32, 32),\n", + " device=jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(np.array(jax.devices()).reshape(8, 2), ('x', 'y')),\n", + " jax.sharding.PartitionSpec('x', 'y')\n", + " )\n", + ")" + ], + "metadata": { + "id": "1d1_DCd96-GW" + }, + "execution_count": null, + "outputs": [], + "id": "1d1_DCd96-GW" + }, + { + "cell_type": "code", + "source": [ + "arr.sharding.shard_shape(arr.shape)" + ], + "metadata": { + "id": "IgsoAWlD7MtM" + }, + "execution_count": null, + "outputs": [], + "id": "IgsoAWlD7MtM" + }, + { + "cell_type": "markdown", + "source": [ + "Now let's suppose we want to load it with the following sharding, giving a shard shape of (32, 2):" + ], + "metadata": { + "id": "QZ1fc_-37Sce" + }, + "id": "QZ1fc_-37Sce" + }, + { + "cell_type": "code", + "source": [ + "sharding = jax.sharding.NamedSharding(\n", + " jax.sharding.Mesh(np.array(jax.devices()).reshape(1, 16), ('x', 'y')),\n", + " jax.sharding.PartitionSpec(None, 'y')\n", + ")\n", + "sharding.shard_shape(arr.shape)" + ], + "metadata": { + "id": "H4wqwXav7c8Y" + }, + "execution_count": null, + "outputs": [], + "id": "H4wqwXav7c8Y" + }, + { + "cell_type": "markdown", + "source": [ + "Without subchunking, in order to read a single shard of shape (32, 2), we would have to read all 16 columns in the origin chunk, discarding 14, in order to obtain the 2 we actually want." + ], + "metadata": { + "id": "waWYFMUi7CRg" + }, + "id": "waWYFMUi7CRg" + }, + { + "cell_type": "markdown", + "source": [ + "To fix this, we can ensure that our checkpoint is saved with sub-chunking along the column dimension. Now the `chunk_shape` becomes `(4, 1)` while `write_shape` (the original shard shape) remains the same." + ], + "metadata": { + "id": "X3udRKEP5R12" + }, + "id": "X3udRKEP5R12" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array.saving.storage_options.chunk_byte_size = 16\n", + "ctx.array.saving.storage_options.shard_axes = (1,)\n", + "\n", + "with ctx:\n", + " path = root_directory / 'ckpt_subchunked_with_shard_axes'\n", + " ocp.save(path, {'a': arr}, overwrite=True) # 32 * 32 * 4 bytes\n", + "ocp.metadata(path).metadata['a'].storage_metadata" + ], + "metadata": { + "id": "W2oOq6-A5fa2" + }, + "execution_count": null, + "outputs": [], + "id": "W2oOq6-A5fa2" + }, + { + "cell_type": "markdown", + "source": [ + "A large-scale performance evaluation of this feature on Llama 3.1 70B model shows positive results for various resharding patterns, as shown below." + ], + "metadata": { + "id": "cv-ojAhH87w8" + }, + "id": "cv-ojAhH87w8" + }, + { + "cell_type": "markdown", + "source": [ + "| Model | Origin Topology | Sharding (data, fsdp, tensor) | Target Topology | Sharding (data, fsdp, tensor) | Subchunked | Not Subchunked | Speedup |\n", + "| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n", + "| Llama 3.1 70B | v5p-256 | (4, 8, 4) | v5p-256 | (1, 128, 1) | 13.41 ± 1.44 | 45.23 ± 2.69 | **3.37×** |\n", + "| Llama 3.1 70B | v5p-256 | (4, 8, 4) | v5p-64 | (1, 32, 1) | 14.40 ± 0.51 | 34.40 ± 2.84 | **2.39×** |\n", + "| Llama 3.1 70B (baseline) | v5p-256 | (4, 8, 4) | v5p-256 | (4, 8, 4) | 15.99 ± 1.50 | - | - |\n", + "| Llama 3.1 8B | v5p-32 | (8, 2, 1) | v5p-32 | (1, 16, 1) | 3.59 ± 0.17 | 20.40 ± 2.56 | **5.68×** |\n", + "| Llama 3.1 8B | v5p-32 | (8, 2, 1) | v5p-8 | (1, 4, 1) | 10.88 ± 0.49 | 21.07 ± 3.12 | **1.94×** |\n", + "| Llama 3.1 8B (baseline) | v5p-32 | (8, 2, 1) | v5p-32 | (8, 2, 1) | 26.20 ± 3.44 | - | - |" + ], + "metadata": { + "id": "UpSdyFebKswg" + }, + "id": "UpSdyFebKswg" + }, + { + "cell_type": "markdown", + "source": [ + "### Memory Usage" + ], + "metadata": { + "id": "F8kDkkDGdr2r" + }, + "id": "F8kDkkDGdr2r" + }, + { + "cell_type": "markdown", + "source": [ + "#### Loading" + ], + "metadata": { + "id": "hZ6q8zs8PBkt" + }, + "id": "hZ6q8zs8PBkt" + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e770d3fb" + }, + "source": [ + "When loading, a checkpoint may consume substantial amounts of RAM before being transferred to accelerators. Depending on the ratio of available accelerator to host memory, as well as the size of the checkpoint, you may find that it is necessary to set a cap on RAM usage.\n", + "\n", + "The feature can also be secondarily useful purely for rate-limiting traffic to storage.\n", + "\n", + "Note that is not possible to set a cap on accelerator memory usage, as this is the ultimate destination and must fit the entire model. RAM is merely a transient resource for loading.\n", + "\n", + "To lower concurrent RAM usage, you can enable `read_concurrent_bytes` to ensure no more than the specified limit can be in use at once before transferring loaded weights to the accelerators." + ], + "id": "e770d3fb" + }, + { + "cell_type": "code", + "metadata": { + "id": "a4b1eb45" + }, + "source": [ + "ctx = ocp.Context()\n", + "with ctx:\n", + " path = root_directory / 'ckpt_memory_test'\n", + " arrs = [jnp.ones((1024, 1024)) for _ in range(150)] # 10 x ~4MB arrays\n", + " ocp.save(path, arrs, overwrite=True)" + ], + "execution_count": null, + "outputs": [], + "id": "a4b1eb45" + }, + { + "cell_type": "code", + "metadata": { + "id": "2b9c4e43" + }, + "source": [ + "def load_unlimited():\n", + " ctx = ocp.Context()\n", + " with ctx:\n", + " ocp.load(path)\n", + "\n", + "def load_limited():\n", + " ctx = ocp.Context()\n", + " ctx.memory_options.read_concurrent_bytes = 1024 * 1024 * 5\n", + " with ctx:\n", + " ocp.load(path)\n", + "\n", + "load_unlimited()\n", + "load_limited()" + ], + "execution_count": null, + "outputs": [], + "id": "2b9c4e43" + }, + { + "cell_type": "markdown", + "source": [ + "#### Saving" + ], + "metadata": { + "id": "ol0nsGU2PEfJ" + }, + "id": "ol0nsGU2PEfJ" + }, + { + "cell_type": "markdown", + "source": [ + "Memory-limited saving is more complex than loading because saving is typically assumed to be asynchronous, while loading blocks experiment start-up. This means that limiting the transfer from accelerators to storage at saving time in order to limit RAM consumption necessarily introduces storage I/O wait time into the blocking path of save, transforming an asynchronous save into a synchronous one.\n", + "\n", + "Orbax team is currently developing features for memory-limited saving as well as recommendations for their use - please stay tuned!" + ], + "metadata": { + "id": "HGdT-xhqPH2j" + }, + "id": "HGdT-xhqPH2j" + }, + { + "cell_type": "markdown", + "source": [ + "## Performance at Scale" + ], + "metadata": { + "id": "eJbY7B0AdLr1" + }, + "id": "eJbY7B0AdLr1" + }, + { + "cell_type": "markdown", + "source": [ + "For our purposes \"at scale\" refers to training workloads of thousands of devices, often using multiple TPU pods and partitioning the model using multiple\n", + "data-parallel replicas. This scale presents unprecedented challenges, but also offers unique opportunities for performance improvements. Below, we show a few key features that Orbax provides to unlock checkpointing performance at scale." + ], + "metadata": { + "id": "jJTB2787iXMJ" + }, + "id": "jJTB2787iXMJ" + }, + { + "cell_type": "markdown", + "source": [ + "For saving and loading benchmarks, we use `n` slices of 64 n2-standard-4 machines with\n", + "a Llama 3.1 70B model fully sharded 64 ways and replicated `n` ways for `n ∈ {2, 4, 8, 16, 32}`.\n", + "This CPU-only approach to benchmarking exercises the actual resources required for large-\n", + "scale checkpointing, namely network and distributed storage, without incurring substantial\n", + "monetary cost from accelerator usage. While device / host transfer throughput cannot\n", + "be measured with this approach, inter-slice communication between slices use DCN (data\n", + "center network) connections regardless of whether TPU or CPU hosts are used." + ], + "metadata": { + "id": "6QcU6UthhLRX" + }, + "id": "6QcU6UthhLRX" + }, + { + "cell_type": "markdown", + "source": [ + "### Load-Balanced Saving (Replica-Parallel Mode)" + ], + "metadata": { + "id": "IAGWtvZ-dXs4" + }, + "id": "IAGWtvZ-dXs4" + }, + { + "cell_type": "markdown", + "source": [ + "For models with more than one data-parallel replica, traditional checkpointing avoids duplicate work by saving only one copy of the model, typically model replica 0 (located on the set of hosts corresponding to physical slice 0, or more precisely, the set of array shards with `replica_id == 0`). Even if most model weights are fully-sharded, a fraction may still be fully replicated, in which case the same de-duplication logic applies.\n", + "\n", + "By default, Orbax improves on the traditional behavior with its *replica-parallel* feature, which slices shards on device to balance saving work\n", + "across hosts. Concretely, this means that for an array shard with `n` replicas, Orbax obtains `n` unique sub-shards that are then transferred and written from each host independently. This translates to reduced I/O and memory pressure on all hosts.\n", + "\n", + "Currently this behavior is enabled by default based on our evaluations, which show write-time speedups for many levels of replication. However, the behavior may require tuning for specific scenarios, as slicing shards too finely results in an elevated number of queries per second (QPS) for a constant data volume, potentially degrading performance for some topologies or storage backends." + ], + "metadata": { + "id": "dazMPJTsBunw" + }, + "id": "dazMPJTsBunw" + }, + { + "cell_type": "markdown", + "source": [ + "Options for tuning can be found under `ocp.Context.array_options.saving`." + ], + "metadata": { + "id": "a8NV3hsWK-q5" + }, + "id": "a8NV3hsWK-q5" + }, + { + "cell_type": "markdown", + "source": [ + "| Slices (`n` X 64-n2-standard-4) | Single-Slice | Replica-Parallel | Speedup |\n", + "| :--- | :--- | :--- | :--- |\n", + "| 2 | 49.20 ± 12.28 | 29.40 ± 3.00 | **1.67×** |\n", + "| 4 | 49.63 ± 6.86 | 25.12 ± 7.19 | **1.98×** |\n", + "| 8 | 57.46 ± 20.19 | 24.47 ± 6.97 | **2.35×** |\n", + "| 16 | 77.53 ± 18.88 | 32.51 ± 13.88 | **2.39×** |\n", + "| 32 | 131.83 ± 20.37 | 77.00 ± 19.85 | **1.71×** |" + ], + "metadata": { + "id": "pAybgNi6LJ1s" + }, + "id": "pAybgNi6LJ1s" + }, + { + "cell_type": "markdown", + "source": [ + "### Scalable Loading (Load+Broadcast Mode)" + ], + "metadata": { + "id": "EBh0ljmJdcTK" + }, + "id": "EBh0ljmJdcTK" + }, + { + "cell_type": "markdown", + "source": [ + "For large training runs where the model is partitioned into data-parallel replicas, parallel reads from all hosts introduce excessive load on the distributed storage. Since our model is replicated, however, we can reduce duplicate reads by a factor of `n` (the number of model replicas). Instead, only hosts belong corresponding to the primary model replica need to perform a read from disk. After being loaded to the primary-replica devices, it can be broadcast to all other devices.\n", + "\n", + "For example, we may have 32 slices, each consisting of 1024 TPUs (256 hosts), giving 32768 chips total. All 256 hosts of slice 0 perform the read from disk. After loading the model onto the corresponding 1024 TPUs, the model is broadcast to all 31 other slices over the DCN (data-center network).\n", + "\n", + "Note that the load+broadcast feature is tied to the logical notion of a model replica rather than any physical notion of TPU pods / slices. Replication is determined exclusively from the device mesh of the arrays (assuming SPMD) along with a `replica_axis_index` parameter. In our example, the mesh might have axes like: `('replica', 'fsdp', 'tensor'): [32, 1024, 1]` with `replica_axis_index=0`." + ], + "metadata": { + "id": "xasMShwDgg8I" + }, + "id": "xasMShwDgg8I" + }, + { + "cell_type": "markdown", + "source": [ + "| Slices (`n` X 64-n2-standard-4)| Broadcast | No Broadcast | Speedup |\n", + "| :--- | :--- | :--- | :--- |\n", + "| 2 | 35.31 ± 1.69 | 27.88 ± 2.10 | 0.79× |\n", + "| 4 | 33.33 ± 2.55 | 51.77 ± 3.22 | **1.55×** |\n", + "| 16 | 38.45 ± 1.45 | 156.03 ± 13.68 | **4.06×** |\n", + "| 32 | 46.85 ± 3.33 | 231.13 ± 10.11 | **4.93×** |" + ], + "metadata": { + "id": "KCMVqxSLLNOP" + }, + "id": "KCMVqxSLLNOP" + }, + { + "cell_type": "markdown", + "source": [ + "As we can see, load+broadcast mode offers a significant speedup relative to the baseline approach. The relative benefit improves with larger scale. Note that at the 2-slice scale, the overhead of broadcasting outweighs the benefit." + ], + "metadata": { + "id": "dCxr9f9QyrAw" + }, + "id": "dCxr9f9QyrAw" + }, + { + "cell_type": "markdown", + "source": [ + "In reality, the performance gains from load+broadcast mode are likely understated by this benchmark, since TPU hosts are equipped with higher-throughput network interfaces. We were also forced to rely on gloo, which is largely unproven performance-wise, rather than megascale as our collectives implementation for this benchmark." + ], + "metadata": { + "id": "JTcfOhAJhCd6" + }, + "id": "JTcfOhAJhCd6" + }, + { + "cell_type": "markdown", + "source": [ + "Let's take a look at an example. In this example, we have a mesh with shape `(2, 8)`, where the row dimension represents a \"slice\" where each row has a replica of our \"model\"." + ], + "metadata": { + "id": "G_SFxPguHnBh" + }, + "id": "G_SFxPguHnBh" + }, + { + "cell_type": "code", + "source": [ + "num_replicas = 2\n", + "num_devices_per_replica = 8\n", + "assert num_replicas * num_devices_per_replica == jax.device_count()\n", + "\n", + "device_array = np.asarray(jax.devices()).reshape(\n", + " (num_replicas, num_devices_per_replica)\n", + ")\n", + "mesh = jax.sharding.Mesh(device_array, ('replica', 'model'))\n", + "pspec = jax.sharding.PartitionSpec('model')\n", + "sharding = jax.sharding.NamedSharding(mesh, pspec)\n", + "arr = jnp.arange(1024, device=sharding)\n", + "abstract_arr = ocp.arrays.to_shape_dtype_struct(arr)\n", + "\n", + "jax.debug.visualize_array_sharding(arr)" + ], + "metadata": { + "id": "u8TcxqvBFuaG" + }, + "execution_count": null, + "outputs": [], + "id": "u8TcxqvBFuaG" + }, + { + "cell_type": "markdown", + "source": [ + "As we can see, each array shard is replicated onto one device from each \"slice\"." + ], + "metadata": { + "id": "IqcLnJruHgti" + }, + "id": "IqcLnJruHgti" + }, + { + "cell_type": "code", + "source": [ + "path = root_directory / 'load_and_broadcast'\n", + "ocp.save(path, [arr], overwrite=True)" + ], + "metadata": { + "id": "LSVOs0F0HEcO" + }, + "execution_count": null, + "outputs": [], + "id": "LSVOs0F0HEcO" + }, + { + "cell_type": "markdown", + "source": [ + "```{warning}\n", + "Make sure to specify **`replica_axis_index`**. This must correspond to the replicated axis of your mesh.\n", + "```" + ], + "metadata": { + "id": "dEXIZIxmIu1U" + }, + "id": "dEXIZIxmIu1U" + }, + { + "cell_type": "code", + "source": [ + "ctx = ocp.Context()\n", + "ctx.array_options.loading.use_load_and_broadcast = True\n", + "ctx.array_options.loading.load_and_broadcast_options.replica_axis_index = 0\n", + "with ctx:\n", + " loaded = ocp.load(path, [abstract_arr])\n", + "\n", + "jax.debug.visualize_array_sharding(loaded[0])" + ], + "metadata": { + "id": "fcZqsZ4hIJn6" + }, + "execution_count": null, + "outputs": [], + "id": "fcZqsZ4hIJn6" + } + ] +} diff --git a/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb b/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb index 2ce451c2bc..053649a4b8 100644 --- a/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb +++ b/docs/guides/checkpoint/v1/orbax_checkpoint_101.ipynb @@ -288,7 +288,7 @@ }, "cell_type": "code", "source": [ - "!ls {directory / 'experiment'}" + "import os; print(sorted(os.listdir(directory / 'experiment')))" ], "outputs": [], "execution_count": null diff --git a/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb b/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb index 69d5cc66e0..4480ea8065 100644 --- a/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb +++ b/docs/guides/checkpoint/v1/orbax_v0_to_v1_migration.ipynb @@ -146,7 +146,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/migration/root_dir/0" + "print(sorted([p.name for p in (root_dir / '0').iterdir()]))" ], "outputs": [ { @@ -282,7 +282,7 @@ }, "cell_type": "code", "source": [ - "!ls /tmp/migration/custom_checkpoint/my_checkpoint" + "print(sorted([p.name for p in my_checkpoint_dir.iterdir()]))" ], "outputs": [ { diff --git a/docs/guides/checkpoint/v1/training.ipynb b/docs/guides/checkpoint/v1/training.ipynb index 8f5b8262c8..b012f07c61 100644 --- a/docs/guides/checkpoint/v1/training.ipynb +++ b/docs/guides/checkpoint/v1/training.ipynb @@ -313,7 +313,7 @@ }, "outputs": [], "source": [ - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -374,7 +374,7 @@ }, "outputs": [], "source": [ - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -1128,7 +1128,7 @@ " # Update model and optimizer separately\n", " nnx.update(model, loaded_state['params'])\n", " nnx.update(optimizer, loaded_state['optimizer'])\n", - " last_step = loaded_state['optimizer']['step'].value\n", + " last_step = loaded_state['optimizer']['step'].get_value()\n", " else:\n", " last_step = 0\n", "\n", @@ -1146,7 +1146,7 @@ " model, optimizer, last_step = init_or_restore(ckptr, abs_state, ckpt_path)\n", "\n", " # Main training loop\n", - " with mesh:\n", + " with jax.set_mesh(mesh):\n", " for step, batch in enumerate(train_ds, start=last_step + 1):\n", " if step >= train_steps or (stop_fn and stop_fn(step)):\n", " break\n", @@ -1184,7 +1184,7 @@ "train(stop_fn=simulated_failure)\n", "latest = training.Checkpointer(root_directory).latest\n", "assert latest is not None and latest.step == 50\n", - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] }, { @@ -1230,7 +1230,7 @@ "assert latest is not None and latest.step == 80\n", "assert (root_directory / '80').exists()\n", "train(ckpt_path=root_directory / '80')\n", - "!ls {root_directory}" + "print(sorted([p.name for p in root_directory.iterdir()]))" ] } ], diff --git a/docs/index.rst b/docs/index.rst index 0794543563..9d60f4de99 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -106,6 +106,7 @@ API that is **easy to use**, **highly performant**, and **maximimally compatible guides/checkpoint/v1/async_checkpointing guides/checkpoint/v1/checkpoint_format + guides/checkpoint/v1/maximizing_performance .. toctree:: :hidden: