Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
0a6a0c5
perf(rollout): TP2 decode optimizations — GLU combo, custom-AR handsh…
delock Nov 2, 2026
4bcc86b
fix: CI compliance — arch guards, hook literals, mock-safe guard, scr…
delock Nov 2, 2026
9e7e8dd
fix: CI compliance — remove torch.distributed import, fix ignore-cuda…
delock Nov 2, 2026
9cc11f5
feat(segki): arena/reshard handshake — segki_refresh drops derived-we…
delock Nov 3, 2026
8b811a0
fix: import RolloutRequest in the arena tests
delock Nov 3, 2026
4e384a2
test: ZeRO-3 trainer -> arena-sync -> segKI rollout end-to-end
delock Nov 3, 2026
8111716
docs: mark the refresh_weights contract provisional pending final han…
delock Nov 3, 2026
bd736d3
fix: gate arena tests' graph capture on real accelerators; CI compliance
delock Nov 3, 2026
e3a4652
fix: route the arena e2e script through get_accelerator() instead of …
delock Nov 3, 2026
5f44d61
chore: drop dead accelerator graph-capture probes; use DeepSpeed lice…
delock Nov 3, 2026
1152551
refactor(comm): adopt the custom_collective from #8997, drop the loca…
delock Nov 3, 2026
cef3ec9
test: reference anchors for the post-original glue kernels
delock Nov 3, 2026
47d426d
refactor: drop the compile integration — segKI is hand-written kernels
delock Nov 3, 2026
d154866
revert: restore the TP dataloader consistency check in engine.py
delock Nov 4, 2026
1c7717e
Merge remote-tracking branch 'origin/master' into gma/qwen_ki_expr
delock Nov 4, 2026
38ee22c
refactor(interface): all Python call sites route through the portable…
delock Nov 4, 2026
7a44923
test: route kernel-reference tests through grouped_linear + fix asser…
delock Nov 4, 2026
1d3948e
refactor(reference): kernel_reference aligned to the 7 interface methods
delock Nov 4, 2026
e356342
fix: reference test alignment — all call sites through grouped_linear…
delock Nov 5, 2026
fb0351e
chore: drop accidentally committed pre-commit cache dirs
delock Nov 5, 2026
9c24dba
fix(kernels): decode_attn honors the q row stride
delock Nov 5, 2026
5712d44
fix(segment_ki): pack FLA recurrent bypass inputs; gate by parity probe
delock Nov 5, 2026
169577e
fix(cache): bridge the HF >=5.19 GDN cache protocol on the static cache
delock Nov 6, 2026
7b3ea17
sync(custom_collective): align with the validated #8997 implementation
delock Nov 6, 2026
a4edcac
fix(segment_ki): correct GDN TP layout — slice z/b/a per rank, valida…
delock Nov 6, 2026
2d834e9
merge: adopt master's _token_selector (#8939), drop _sample_from_logits
delock Nov 8, 2026
a9a6ac1
fix: merge fallout — drop duplicate get_query_offset, unused import
delock Nov 8, 2026
4fd31d6
fix(rollout): sample the prefill first token and the b>1 decode steps
delock Nov 8, 2026
39210bf
fix(cache): restore the transformers>=5.20 seq-length protocol on Dee…
delock Nov 9, 2026
300e5b8
test(cache): align seq-length expectations with the restored +1 protocol
delock Nov 9, 2026
43b624f
fix(rollout): route graph capture through the accelerator-gated variable
delock Nov 9, 2026
5b70a0b
fix(rollout): define the accelerator-gated use_graph_capture; drop is…
delock Nov 9, 2026
16b6fc6
fix(rollout): adopt master's _continuous_next_tokens with token selector
delock Nov 9, 2026
30ce8d7
fix: drop leftover @staticmethod on _continuous_next_tokens
delock Nov 9, 2026
d834039
fix: add module-level logger; defensive num_attention_heads access
delock Nov 9, 2026
e53e2dc
fix: _continuous_prefill_cache sampling param; drop greedy-only restr…
delock Nov 9, 2026
6d273de
fix: add sampling param to _continuous_prefill; update call sites
delock Nov 9, 2026
eff04b3
fix(rollout): restore master's graph-capture gates lost in the merge
delock Nov 9, 2026
e2ad943
Merge branch 'master' into gma/qwen_ki_expr
delock Nov 9, 2026
2553e95
Merge branch 'master' into gma/qwen_ki_expr
delock Nov 10, 2026
aed2898
Merge branch 'master' into gma/qwen_ki_expr
delock Nov 11, 2026
2d8f409
fix(comm): restrict the CPU shm inference all-reduce to world-wide gr…
delock Nov 11, 2026
99ad881
fix(module-inject): preserve HF precision boundaries in segKI kernels
nathon-lee Nov 11, 2026
f959d64
combo: run whole decode step in graph for all batch sizes (P0-3 port)
jinyouzhi Nov 13, 2026
5afd3f8
combo: L2-L5 (input F.rms_norm, GLU m>32 native, GDN b|a concat, cach…
jinyouzhi Nov 13, 2026
db5b6b6
combo: MRoPE position shape for segKI rotary tables
jinyouzhi Nov 13, 2026
be8c2d8
segKI: optional vLLM-aligned decode backends (FA3 attention, packed GDN)
jinyouzhi Nov 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
334 changes: 334 additions & 0 deletions csrc/comm/custom_collective.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,334 @@
// SPDX-License-Identifier: Apache-2.0
// DeepSpeed Team

// Latency-optimized inference all-reduce for small messages on P2P-capable
// GPU pairs. Fills the deepspeed.comm.inference_all_reduce hook the same way
// the CPU backend fills it with shm+AVX: the op stays accelerator-owned and
// every backend can walk the same route.
//
// Stream-ordered 5-kernel protocol (graph-capturable, correctness by kernel
// boundaries instead of a grid-wide barrier):
// k0 seq_d[0]++ replay-safe sequence number
// k1 copy input -> my IPC buf[p] (all copies complete at boundary)
// k2 system fence; my flag[p] = seq
// k3 spin until peer flag[p] >= seq
// k4 out[i] = in[i] + peer_buf[p][i]
// Double buffering (p = seq & 1) lets call N+1 publish while call N reduces.

#include <ATen/cuda/CUDAContext.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <pybind11/pytypes.h>
#include <torch/extension.h>

#define CAR_MAX_BYTES (4 * 1024 * 1024)
#define CAR_THREADS 256

namespace {

struct CarState {
bool ready = false;
int world = 0;
int dev = 0; // our CUDA device ordinal (recorded at create_buffers)
int peer_dev = 0; // peer GPU's ordinal in our enumeration (from connect)
char* my_base = nullptr; // cudaMalloc'd: [flag0|flag1|pad][buf0][buf1]
char* peer_base = nullptr;
int* seq_d = nullptr;
char* own_base = nullptr; // our cudaMalloc (never freed; process exit)
int* own_seq = nullptr;
};
CarState g_state;

constexpr size_t kFlagOff0 = 0;
constexpr size_t kFlagOff1 = 4;
constexpr size_t kBufBase = 256;
constexpr size_t kBufStride = CAR_MAX_BYTES;
constexpr size_t kTotalBytes = kBufBase + 2 * kBufStride;

#define CUDA_CHECK(expr) \
TORCH_CHECK((expr) == cudaSuccess, "CUDA: ", cudaGetErrorString(cudaGetLastError()))

__global__ void seq_bump_kernel(int* seq_d) { *seq_d = *seq_d + 1; }

__global__ void copy_kernel(const char* __restrict__ in,
char* __restrict__ my_base,
const int* __restrict__ seq_d,
size_t nbytes)
{
const int parity = (*seq_d) & 1;
char* dst = my_base + kBufBase + (size_t)parity * kBufStride;
// contiguous does not imply 16B alignment (e.g. a sliced view); the
// vectorized path is only legal when the source pointer is aligned
const bool aligned = (reinterpret_cast<uintptr_t>(in) & 15) == 0;
if (aligned) {
const size_t vec = nbytes / 16;
const uint4* in4 = (const uint4*)in;
uint4* dst4 = (uint4*)dst;
for (size_t v = (size_t)blockIdx.x * blockDim.x + threadIdx.x; v < vec;
v += (size_t)gridDim.x * blockDim.x)
dst4[v] = in4[v];
for (size_t b = vec * 16 + (size_t)blockIdx.x * blockDim.x + threadIdx.x; b < nbytes;
b += (size_t)gridDim.x * blockDim.x)
dst[b] = in[b];
} else {
for (size_t b = (size_t)blockIdx.x * blockDim.x + threadIdx.x; b < nbytes;
b += (size_t)gridDim.x * blockDim.x)
dst[b] = in[b];
}
}

__global__ void flag_kernel(char* __restrict__ my_base, const int* __restrict__ seq_d)
{
const int seq = *seq_d;
const int parity = seq & 1;
__threadfence_system();
*(volatile int*)(my_base + (parity ? kFlagOff1 : kFlagOff0)) = seq;
}

__global__ void wait_kernel(const char* __restrict__ peer_base, const int* __restrict__ seq_d)
{
const int seq = *seq_d;
const int parity = seq & 1;
volatile const int* pflag = (volatile const int*)(peer_base + (parity ? kFlagOff1 : kFlagOff0));
while (*pflag < seq) {}
__threadfence_system();
}

__global__ void reduce_bf16_kernel(const char* __restrict__ in,
char* __restrict__ out,
const char* __restrict__ peer_base,
const int* __restrict__ seq_d,
size_t numel)
{
const int parity = (*seq_d) & 1;
const char* psrc = peer_base + kBufBase + (size_t)parity * kBufStride;
const __nv_bfloat16* pin = (const __nv_bfloat16*)in;
const __nv_bfloat16* pp = (const __nv_bfloat16*)psrc;
__nv_bfloat16* pout = (__nv_bfloat16*)out;
for (size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; i < numel;
i += (size_t)gridDim.x * blockDim.x)
pout[i] = __float2bfloat16(__bfloat162float(pin[i]) + __bfloat162float(pp[i]));
}

__global__ void reduce_fp16_kernel(const char* __restrict__ in,
char* __restrict__ out,
const char* __restrict__ peer_base,
const int* __restrict__ seq_d,
size_t numel)
{
const int parity = (*seq_d) & 1;
const char* psrc = peer_base + kBufBase + (size_t)parity * kBufStride;
const __half* pin = (const __half*)in;
const __half* pp = (const __half*)psrc;
__half* pout = (__half*)out;
for (size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; i < numel;
i += (size_t)gridDim.x * blockDim.x)
pout[i] = __float2half(__half2float(pin[i]) + __half2float(pp[i]));
}

__global__ void reduce_fp32_kernel(const char* __restrict__ in,
char* __restrict__ out,
const char* __restrict__ peer_base,
const int* __restrict__ seq_d,
size_t numel)
{
const int parity = (*seq_d) & 1;
const char* psrc = peer_base + kBufBase + (size_t)parity * kBufStride;
const float* pin = (const float*)in;
const float* pp = (const float*)psrc;
float* pout = (float*)out;
for (size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; i < numel;
i += (size_t)gridDim.x * blockDim.x)
pout[i] = pin[i] + pp[i];
}

} // namespace

py::bytes car_create_buffers()
{
// Pure CUDA IPC (vLLM-style): our own cudaMalloc + GetMemHandle, no
// torch allocator wrappers in the exchange path.
TORCH_CHECK(!g_state.ready, "custom AR already initialized");
void* buf = nullptr;
int* seq = nullptr;
CUDA_CHECK(cudaMalloc(&buf, kTotalBytes));
CUDA_CHECK(cudaMalloc((void**)&seq, sizeof(int)));
CUDA_CHECK(cudaMemset(buf, 0, kTotalBytes));
CUDA_CHECK(cudaMemset(seq, 0, sizeof(int)));
cudaIpcMemHandle_t h;
CUDA_CHECK(cudaIpcGetMemHandle(&h, buf));
CUDA_CHECK(cudaGetDevice(&g_state.dev));
g_state.my_base = (char*)buf;
g_state.own_base = (char*)buf;
g_state.own_seq = seq;
g_state.seq_d = seq;
g_state.ready = true;
return py::bytes((const char*)&h, sizeof(h)); // raw handle, not utf-8
}

void car_connect(const std::string& handle_bytes, int64_t peer_dev)
{
TORCH_CHECK(g_state.ready, "call car_create_buffers first");
TORCH_CHECK(handle_bytes.size() >= sizeof(cudaIpcMemHandle_t), "short IPC handle");
cudaIpcMemHandle_t h;
memcpy(&h, handle_bytes.data(), sizeof(h));
void* mapped = nullptr;
// Open on THIS device's context so our kernels can dereference the
// mapping over P2P; the lazy flag enables peer access on first touch.
cudaError_t err = cudaIpcOpenMemHandle(&mapped, h, cudaIpcMemLazyEnablePeerAccess);
TORCH_CHECK(err == cudaSuccess, "cudaIpcOpenMemHandle failed: ", cudaGetErrorString(err));
g_state.peer_base = (char*)mapped;
g_state.peer_dev = (int)peer_dev;
g_state.world = 2;
}

bool car_api_probe()
{
// Stage 1+2 of the link probe, run before any kernel dereferences the peer
// mapping: on fabrics where cudaIpcOpenMemHandle produced a mapping that was
// never populated, the first SM-side access faults with Xid 31 and kills the
// CUDA context unrecoverably. The copy engine walks the same page tables, so
// the broken mapping fails here too -- but as a recoverable API error.
TORCH_CHECK(g_state.ready && g_state.peer_base, "custom AR not connected");
// Explicit enable: without it cudaMemcpyPeerAsync may silently fall back to a
// host-staged bounce copy, which would "pass" without proving direct P2P.
cudaError_t err = cudaDeviceEnablePeerAccess(g_state.peer_dev, 0);
if (err != cudaSuccess && err != cudaErrorPeerAccessAlreadyEnabled) {
cudaGetLastError();
return false;
}
const int64_t magic = 0x5A5A5A5A5A5A5A5ALL;
auto opts = torch::TensorOptions().dtype(torch::kInt64).device(torch::kCUDA);
auto token = torch::full({1}, magic, opts);
auto verify = torch::zeros({1}, opts);
auto stream = at::cuda::getCurrentCUDAStream();
// write direction: our token -> peer's buffer
err = cudaMemcpyPeerAsync(g_state.peer_base + kBufBase,
g_state.peer_dev,
token.data_ptr(),
g_state.dev,
sizeof(int64_t),
stream.stream());
if (err != cudaSuccess) {
cudaGetLastError();
return false;
}
// read direction: peer's buffer -> local verify
err = cudaMemcpyPeerAsync(verify.data_ptr(),
g_state.dev,
g_state.peer_base + kBufBase,
g_state.peer_dev,
sizeof(int64_t),
stream.stream());
if (err != cudaSuccess) {
cudaGetLastError();
return false;
}
if (cudaStreamSynchronize(stream.stream()) != cudaSuccess) {
cudaGetLastError();
return false;
}
return verify.cpu().item<int64_t>() == magic;
}

__global__ void handshake_kernel(char* __restrict__ my_base,
const char* __restrict__ peer_base,
int* __restrict__ seq_d,
int* __restrict__ result)
{
const int seq = *seq_d + 1; // handshake rides the next sequence number
*seq_d = seq;
const int parity = seq & 1;
char* my_buf = my_base + kBufBase + (size_t)parity * kBufStride;
const char* peer_buf = peer_base + kBufBase + (size_t)parity * kBufStride;
if (threadIdx.x == 0 && blockIdx.x == 0) {
*(int*)(my_buf) = 0x5A5A5A5A;
__threadfence_system();
*(volatile int*)(my_base + (parity ? kFlagOff1 : kFlagOff0)) = seq;
}
__syncthreads();
if (threadIdx.x == 0 && blockIdx.x == 0) {
volatile const int* pflag =
(volatile const int*)(peer_base + (parity ? kFlagOff1 : kFlagOff0));
long long spins = 0;
while (*pflag < seq && spins < 20000000LL) spins++; // bounded; PCIe reads are ~100ns each
if (*pflag < seq) {
*result = -1; // peer flag never became visible
return;
}
__threadfence_system();
*result = (*(volatile const int*)(peer_buf) == 0x5A5A5A5A) ? 1 : -2; // flag ok, data?
}
}

bool car_handshake()
{
TORCH_CHECK(g_state.ready && g_state.peer_base, "custom AR not connected");
auto result = torch::zeros({1}, torch::TensorOptions().dtype(torch::kInt).device(torch::kCUDA));
auto stream = at::cuda::getCurrentCUDAStream();
handshake_kernel<<<1, 32, 0, stream.stream()>>>(
g_state.my_base, g_state.peer_base, g_state.seq_d, (int*)result.data_ptr());
C10_CUDA_KERNEL_LAUNCH_CHECK();
// both ranks must publish before either inspects — a plain NCCL-free
// sync is impossible, so the caller synchronizes the stream first
cudaStreamSynchronize(stream.stream());
return result.cpu().item<int>() == 1;
}

bool car_ready(int64_t world)
{
return g_state.ready && g_state.world == world && g_state.peer_base != nullptr;
}

int64_t car_max_bytes() { return CAR_MAX_BYTES; }

torch::Tensor car_all_reduce(torch::Tensor t)
{
TORCH_CHECK(g_state.ready && g_state.peer_base, "custom AR not connected");
TORCH_CHECK(t.is_cuda() && t.is_contiguous(), "contiguous CUDA tensor required");
const size_t nbytes = (size_t)t.numel() * t.element_size();
TORCH_CHECK(nbytes <= CAR_MAX_BYTES, "message exceeds custom AR buffer");
auto stream = at::cuda::getCurrentCUDAStream();
const int cblocks = (int)((nbytes / 16 + CAR_THREADS - 1) / CAR_THREADS) + 1;
const int rblocks = (int)((t.numel() + CAR_THREADS - 1) / CAR_THREADS);
seq_bump_kernel<<<1, 1, 0, stream.stream()>>>(g_state.seq_d);
copy_kernel<<<cblocks, CAR_THREADS, 0, stream.stream()>>>(
(const char*)t.data_ptr(), g_state.my_base, g_state.seq_d, nbytes);
flag_kernel<<<1, 1, 0, stream.stream()>>>(g_state.my_base, g_state.seq_d);
wait_kernel<<<1, 1, 0, stream.stream()>>>(g_state.peer_base, g_state.seq_d);
if (t.scalar_type() == at::kBFloat16) {
reduce_bf16_kernel<<<rblocks, CAR_THREADS, 0, stream.stream()>>>((const char*)t.data_ptr(),
(char*)t.data_ptr(),
g_state.peer_base,
g_state.seq_d,
(size_t)t.numel());
} else if (t.scalar_type() == at::kHalf) {
reduce_fp16_kernel<<<rblocks, CAR_THREADS, 0, stream.stream()>>>((const char*)t.data_ptr(),
(char*)t.data_ptr(),
g_state.peer_base,
g_state.seq_d,
(size_t)t.numel());
} else if (t.scalar_type() == at::kFloat) {
reduce_fp32_kernel<<<rblocks, CAR_THREADS, 0, stream.stream()>>>((const char*)t.data_ptr(),
(char*)t.data_ptr(),
g_state.peer_base,
g_state.seq_d,
(size_t)t.numel());
} else {
TORCH_CHECK(false, "unsupported dtype for custom AR");
}
C10_CUDA_KERNEL_LAUNCH_CHECK();
return t;
}

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
{
m.def("create_buffers", &car_create_buffers, "allocate IPC buffers, return handle bytes");
m.def("connect", &car_connect, "map peer buffer (data pointer from CUDA IPC storage)");
m.def("api_probe", &car_api_probe, "copy-engine probe: is the peer mapping usable at all");
m.def("handshake", &car_handshake, "bounded probe: is the peer flag visible over this fabric");
m.def("ready", &car_ready, "is the custom path usable");
m.def("max_bytes", &car_max_bytes, "max message size");
m.def("all_reduce", &car_all_reduce, "one-shot P2P all-reduce (in place)");
}
Loading