Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions deepspeed/checkpoint/affine_geometry.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,35 @@ def contains(self, other: 'LogicalBox') -> bool:

def __repr__(self) -> str:
return f'Box({self.origin}+{self.extent})'


def boxes_disjoint(boxes: Sequence[LogicalBox], max_checks: Optional[int] = None) -> bool:
"""Whether no two boxes share an element.

Sweeps along the axis whose box origins are most varied, so a box is compared only with the boxes
still open where it starts. For the grid and run layouts the builders emit that is a handful of
boxes, not all of them, but a layout that keeps many boxes open still costs a comparison per pair.

With ``max_checks``, the sweep gives up and returns False once it has compared more pairs than
that, so False means "not shown disjoint". The sweep compares at most n*(n-1)/2 pairs, so giving
up implies n**2 exceeds ``max_checks``, and a caller's pairwise fallback under the same budget
refuses at once.
"""
boxes = [box for box in boxes if not box.is_empty]
if len(boxes) < 2:
return True
if not boxes[0].origin:
return False
axis = max(range(len(boxes[0].origin)), key=lambda a: len({box.origin[a] for box in boxes}))
open_boxes: List[LogicalBox] = []
checks = 0
for box in sorted(boxes, key=lambda b: b.origin[axis]):
checks += len(open_boxes)
if max_checks is not None and checks > max_checks:
return False
start = box.origin[axis]
open_boxes = [other for other in open_boxes if other.origin[axis] + other.extent[axis] > start]
if any(not box.intersection(other).is_empty for other in open_boxes):
return False
open_boxes.append(box)
return True
43 changes: 35 additions & 8 deletions deepspeed/checkpoint/affine_transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@
import collections.abc
import json
from dataclasses import dataclass
from typing import Callable, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple, Union
from typing import Callable, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Tuple, Union

from .affine import AffinePiece, ParamAffineMap
from .affine_geometry import (LogicalBox, coordinate as _coordinate, product as _product, row_major_strides as
_row_major_strides)
from .affine_geometry import (LogicalBox, boxes_disjoint, coordinate as _coordinate, product as _product,
row_major_strides as _row_major_strides)

__all__ = [
'AffineTransferError', 'InvalidMapError', 'InvalidReplicaPolicyError', 'UnsupportedLayoutError', 'PlanBudget',
Expand Down Expand Up @@ -338,12 +338,18 @@ def _shards_tile(frames: List[_Frame], side: str, limits: PlanBudget) -> None:
of them land on the same addresses of the shard, and their numels still sum to the shard size. A
shard that two pieces claim cannot be read or written by address, so a plan built on such a map is
wrong in whichever direction it is used.

A rank whose pieces sweep as disjoint and fill the shard passes without pairwise comparisons; any
other rank takes the pairwise walk, which names the pieces at fault.
"""
by_rank: Dict[int, List[_Frame]] = {}
for frame in frames:
by_rank.setdefault(frame.rank, []).append(frame)

for rank, held in sorted(by_rank.items()):
if boxes_disjoint([frame.local for frame in held], limits.max_pair_checks) and sum(
frame.local.volume for frame in held) == held[0].shard_numel:
continue
if len(held)**2 > limits.max_pair_checks:
raise AffineTransferError(f'{side}: rank {rank} holds {len(held)} pieces, over the budget for '
'address checks')
Expand All @@ -360,7 +366,26 @@ def _shards_tile(frames: List[_Frame], side: str, limits: PlanBudget) -> None:


def _validate_holders(frames: List[_Frame], side: str, limits: PlanBudget) -> None:
"""A piece's declared holders must cover exactly its logical box on every named rank."""
"""A piece's declared holders must cover exactly its logical box on every named rank.

Most maps pass without comparing pieces pairwise: when no two distinct boxes overlap, a rank covers
a box exactly when it holds that very box, so the holders of each box can be read off a table.
Anything else, such as replicas split at different places, takes the pairwise walk, which also
names the piece at fault.
"""
holders: Dict[LogicalBox, Set[int]] = {}
repeated = False
for frame in frames:
ranks = holders.setdefault(frame.box, set())
repeated = repeated or frame.rank in ranks
ranks.add(frame.rank)
if not repeated and boxes_disjoint(list(holders), limits.max_pair_checks):
if all(holders[frame.box] == frame.locations for frame in frames):
return
_validate_holders_pairwise(frames, side, limits)


def _validate_holders_pairwise(frames: List[_Frame], side: str, limits: PlanBudget) -> None:
if len(frames)**2 > limits.max_pair_checks:
raise AffineTransferError(f'{side}: holder checks exceed max_pair_checks {limits.max_pair_checks}')
by_rank: Dict[int, List[_Frame]] = {}
Expand Down Expand Up @@ -403,18 +428,20 @@ def _sources(frames: List[_Frame], limits: PlanBudget) -> List[Tuple[LogicalBox,
by_box.setdefault(frame.box, []).append(frame)

regions = sorted(by_box, key=lambda box: (box.origin, box.extent))
if not boxes_disjoint(regions, limits.max_pair_checks):
_name_overlapping_regions(regions, limits)
return [(box, tuple(sorted(by_box[box], key=lambda frame: frame.rank))) for box in regions]


def _name_overlapping_regions(regions: List[LogicalBox], limits: PlanBudget) -> None:
if len(regions)**2 > limits.max_pair_checks:
raise AffineTransferError(f'{len(regions)} source regions would take {len(regions)**2} checks, over '
f'max_pair_checks {limits.max_pair_checks}')
grouped = []
for index, box in enumerate(regions):
holders = by_box[box]
for other in regions[index + 1:]:
if not box.intersection(other).is_empty:
raise UnsupportedLayoutError(f'source: replicas of {box!r} and {other!r} have different '
'piece boundaries')
grouped.append((box, tuple(sorted(holders, key=lambda frame: frame.rank))))
return grouped


def _covers_exactly(region: LogicalBox, parts: Sequence[LogicalBox]) -> bool:
Expand Down
137 changes: 137 additions & 0 deletions tests/unit/checkpoint/test_affine_transfer_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

from deepspeed.checkpoint.affine import (AffinePiece, ParamAffineMap, block_gather_map, contiguous_split_map,
replicated_map, segmented_map, sub_param_map)
from deepspeed.checkpoint.affine_geometry import LogicalBox, boxes_disjoint
from deepspeed.checkpoint.affine_transfer import (AffineTransferError, InvalidMapError, PlanBudget, TransferSegment,
UnsupportedLayoutError, plan_transfer)

Expand Down Expand Up @@ -666,6 +667,142 @@ def test_merge_comparisons_have_their_own_budget():
plan_transfer(target, source, PlanBudget(max_merge_checks=10))


def test_many_consistent_pieces_need_no_pairwise_holder_checks():
rows, ranks = 1024, 8
source = block_gather_map((rows, 2), {rank: list(range(rank, rows, ranks)) for rank in range(ranks)}, 1, 0)
target = contiguous_split_map((rows, 2), [rows // ranks] * ranks, 0)
assert rows**2 > PlanBudget().max_pair_checks
full = torch.arange(rows * 2, dtype=torch.float64).reshape(rows, 2)
copied = _apply_plan(plan_transfer(target, source), target, _shards(source, lambda m, r: m.extract(full, r)))
for rank, shard in copied.items():
torch.testing.assert_close(shard, target.extract(full, rank))


def test_many_pieces_on_one_rank_need_no_pairwise_address_checks():
rows, ranks = 2048, 2
source = block_gather_map((rows, 2), {rank: list(range(rank, rows, ranks)) for rank in range(ranks)}, 1, 0)
target = contiguous_split_map((rows, 2), [rows // ranks] * ranks, 0)
assert (rows // ranks)**2 > PlanBudget().max_pair_checks
full = torch.arange(rows * 2, dtype=torch.float64).reshape(rows, 2)
# Merging this many segments is a separate quadratic step with its own budget.
plan = plan_transfer(target, source, PlanBudget(max_merge_checks=1 << 30))
copied = _apply_plan(plan, target, _shards(source, lambda m, r: m.extract(full, r)))
for rank, shard in copied.items():
torch.testing.assert_close(shard, target.extract(full, rank))


def test_sweep_comparisons_count_against_the_pair_budget():
"""A disjoint layout can keep every box open for the whole sweep, so the sweep is budgeted too.

h full-height one-column boxes plus 2h one-element boxes beside them tile a (2h, h+1) parameter.
Sweeping rows, each one-element box meets all h columns: 2h**2 comparisons for 3h pieces.
"""
h = 32
rows, cols = 2 * h, h + 1
pieces = [AffinePiece((rows, 1), column, (cols, 1), column * rows, (1, 1), [0]) for column in range(h)]
pieces += [AffinePiece((1, 1), row * cols + h, (cols, 1), h * rows + row, (1, 1), [0]) for row in range(rows)]
source = ParamAffineMap((rows, cols), {0: (rows * cols, )}, {0: pieces})
target = replicated_map((rows, cols), 1)
assert boxes_disjoint([LogicalBox((0, column), (rows, 1))
for column in range(h)] + [LogicalBox((row, h), (1, 1)) for row in range(rows)])

full = torch.arange(rows * cols, dtype=torch.float64).reshape(rows, cols)
copied = _apply_plan(plan_transfer(target, source), target, _shards(source, lambda m, r: m.extract(full, r)))
torch.testing.assert_close(copied[0], target.extract(full, 0))
with pytest.raises(AffineTransferError, match='max_pair_checks'):
plan_transfer(target, source, PlanBudget(max_pair_checks=2 * h * h - 1))


def _random_holder_map(rng, rows, cols, ranks):
"""Row blocks on every rank, sometimes split at a replica's own points, with holders sometimes wrong."""
shared = sorted(rng.sample(range(1, rows), rng.randint(0, rows - 1)))
blocks_of = {}
for rank in range(ranks):
cuts = shared if rng.random() < 0.7 else sorted(rng.sample(range(1, rows), rng.randint(0, rows - 1)))
bounds = list(zip([0] + cuts, cuts + [rows]))
held = [block for block in bounds if rank == 0 or rng.random() < 0.6]
if rank and rng.random() < 0.1 and held:
low, high = held[0]
held.append((low, min(rows, high + 1)))
blocks_of[rank] = held
pieces, shapes = {}, {}
for rank, blocks in blocks_of.items():
offset, pieces[rank] = 0, []
for low, high in blocks:
# A replica may hold this block as several smaller pieces, so cover it by union.
holders = [
other for other, theirs in blocks_of.items() if all(
any(a <= row < b for a, b in theirs) for row in range(low, high))
]
if rng.random() < 0.15:
holders = sorted(set(holders) ^ {rng.randrange(ranks)}) or [rank]
shape = (high - low, cols)
pieces[rank].append(AffinePiece(shape, low * cols, (cols, 1), offset, (cols, 1), holders))
offset += _product(shape)
shapes[rank] = (offset, )
return ParamAffineMap((rows, cols), shapes, pieces)


def _holders_are_exact(map_):
"""Element-by-element reading of the holder contract, independent of the planner.

No rank may hold an element twice, and every element of a piece must be held by
exactly the ranks the piece names.
"""
full = torch.arange(_product(map_.logical_shape))
elements_of = {}
held_by = {}
for rank, pieces in map_.pieces_by_rank.items():
for piece in pieces:
elements = full.as_strided(piece.shape, piece.source_strides, piece.source_offset).flatten().tolist()
elements_of[(rank, id(piece))] = (elements, set(piece.locations))
for element in elements:
ranks = held_by.setdefault(element, [])
if rank in ranks:
return False
ranks.append(rank)
return all(
set(held_by[element]) == locations for elements, locations in elements_of.values() for element in elements)


@pytest.mark.parametrize('seed', range(200))
def test_holder_check_matches_an_element_level_oracle(seed):
"""Replicas split at different points must still plan; a wrong holder set must be refused."""
rng = random.Random(seed)
rows, cols, ranks = rng.randint(1, 6), rng.randint(1, 3), rng.randint(1, 4)
mapped = _random_holder_map(rng, rows, cols, ranks)
other = replicated_map((rows, cols), 1)
target, source = (mapped, other) if seed % 2 else (other, mapped)
budget = PlanBudget(max_pair_checks=1 << 30)
if not _holders_are_exact(mapped):
with pytest.raises(InvalidMapError):
plan_transfer(target, source, budget)
return
try:
plan = plan_transfer(target, source, budget)
except UnsupportedLayoutError:
# A valid source whose replicas are cut at different points is a documented planner
# limitation, not a holder error; a target with such replicas must still plan.
assert source is mapped
return
full = torch.arange(rows * cols, dtype=torch.float64).reshape(rows, cols)
copied = _apply_plan(plan, target, _shards(source, lambda m, r: m.extract(full, r)))
for rank in target.shard_shapes:
torch.testing.assert_close(copied[rank], target.extract(full, rank))


@pytest.mark.parametrize('seed', range(100))
def test_boxes_disjoint_agrees_with_all_pairs(seed):
rng = random.Random(seed)
ndim = rng.randint(1, 3)
boxes = []
for _ in range(rng.randint(0, 8)):
origin = tuple(rng.randint(0, 4) for _ in range(ndim))
boxes.append(LogicalBox(origin, tuple(rng.randint(0, 3) for _ in range(ndim))))
overlapping = any(not a.intersection(b).is_empty for a, b in itertools.combinations(boxes, 2))
assert boxes_disjoint(boxes) == (not overlapping)


# ---------------------------------------------------------------- never materializing


Expand Down
Loading