Skip to content

Commit 00daaad

Browse files
committed
fix(distance): include <stdexcept> for std::runtime_error and format example Python files
- Add missing <stdexcept> headers in distance kernel files (fp16_ip, fp32_ip, fp32_l2, uint8_l2) to fix Clang/macOS/Linux builds - Add root ruff.toml configuration - Reformat Python example scripts with Ruff
1 parent 07f1f79 commit 00daaad

16 files changed

Lines changed: 330 additions & 151 deletions

File tree

‎cpp/deglib/include/deglib/distance/fp16_ip.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "deglib/distance/residual_mode.h"
55

66
#include <cmath>
7+
#include <stdexcept>
78
#include <variant>
89

910
namespace deglib::distances::fp16_ip {

‎cpp/deglib/include/deglib/distance/fp32_ip.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "deglib/distance/residual_mode.h"
44

55
#include <cmath>
6+
#include <stdexcept>
67
#include <variant>
78

89
namespace deglib::distances::fp32_ip {

‎cpp/deglib/include/deglib/distance/fp32_l2.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "deglib/distance/residual_mode.h"
44

55
#include <cmath>
6+
#include <stdexcept>
67
#include <variant>
78

89
namespace deglib::distances::fp32_l2 {

‎cpp/deglib/include/deglib/distance/uint8_l2.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "deglib/distance/residual_mode.h"
44
#include "deglib/distance/uint8.h"
55

6+
#include <stdexcept>
67
#include <variant>
78

89
namespace deglib::distances::uint8_l2 {

‎examples/dynamic_data/dataset_utils.py‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ def ivecs_read(filename: str | Path) -> np.ndarray:
1313
d = a[0]
1414
return a.reshape(-1, d + 1)[:, 1:].copy()
1515

16+
1617
def fvecs_read(filename: str | Path) -> np.ndarray:
1718
return ivecs_read(filename).view(np.float32)
1819

20+
1921
DATASET_METADATA: Dict[str, Dict[str, Any]] = {
2022
"sift1m": {
2123
"name": "SIFT1M",
@@ -84,9 +86,11 @@ def fvecs_read(filename: str | Path) -> np.ndarray:
8486
},
8587
}
8688

89+
8790
def resolve_dataset_key(key: str) -> str:
8891
return key.lower()
8992

93+
9094
def get_default_cache_dir() -> Path:
9195
"""Returns the default dataset cache directory (~/.cache/deg_datasets or DEG_CACHE_DIR)."""
9296
env_dir = os.environ.get("DEG_CACHE_DIR")
@@ -97,6 +101,7 @@ def get_default_cache_dir() -> Path:
97101
path.mkdir(parents=True, exist_ok=True)
98102
return path
99103

104+
100105
def download_file(url: str, dest_path: Path):
101106
"""Downloads a file from url to dest_path with progress indication."""
102107
print(f"Downloading {url} to {dest_path}...")
@@ -127,6 +132,7 @@ def download_file(url: str, dest_path: Path):
127132
temp_path.rename(dest_path)
128133
print(f"Download complete: {dest_path}")
129134

135+
130136
def ivecs_write(filename: str | Path, data: np.ndarray):
131137
"""Writes a 2D int32 numpy array to ivecs format."""
132138
filename = Path(filename)
@@ -137,6 +143,7 @@ def ivecs_write(filename: str | Path, data: np.ndarray):
137143
with_d = np.hstack([d_col, data.astype(np.int32)])
138144
with_d.tofile(filename)
139145

146+
140147
def fvecs_write(filename: str | Path, data: np.ndarray):
141148
"""Writes a 2D float32 numpy array to fvecs format."""
142149
filename = Path(filename)
@@ -148,6 +155,7 @@ def fvecs_write(filename: str | Path, data: np.ndarray):
148155
with_d = np.hstack([d_col, data.astype(np.float32)])
149156
with_d.tofile(filename)
150157

158+
151159
def cleanup_legacy_gt(folder: Path):
152160
"""Deletes legacy/broken groundtruth files extracted from old dataset archives."""
153161
legacy_files = [
@@ -172,15 +180,16 @@ def cleanup_legacy_gt(folder: Path):
172180
except Exception as e:
173181
print(f"Warning: Failed to remove legacy GT file {p}: {e}")
174182

183+
175184
def ensure_dataset(dataset_key: str, cache_dir: Path) -> Path:
176185
"""Ensures the dataset is downloaded and extracted in cache_dir, returning folder path."""
177186
key = resolve_dataset_key(dataset_key)
178187
if key not in DATASET_METADATA:
179188
raise ValueError(f"Unknown dataset '{dataset_key}'. Choose from: {list(DATASET_METADATA.keys())}")
180-
189+
181190
meta = DATASET_METADATA[key]
182191
archive_path = cache_dir / meta["archive"]
183-
192+
184193
# 1. Check if direct folder (e.g., D:\Data\DEG\sift1m or D:\Data\DEG\sift) exists
185194
extracted_folder = cache_dir / meta["folder"]
186195
if not extracted_folder.is_dir():
@@ -198,7 +207,7 @@ def ensure_dataset(dataset_key: str, cache_dir: Path) -> Path:
198207
archive_path = archive_matches[0]
199208
else:
200209
download_file(meta["url"], archive_path)
201-
210+
202211
print(f"Extracting {archive_path} into {cache_dir}...")
203212
with tarfile.open(archive_path, "r:gz") as tar:
204213
tar.extractall(path=cache_dir)
@@ -215,50 +224,63 @@ def ensure_dataset(dataset_key: str, cache_dir: Path) -> Path:
215224
return extracted_folder
216225

217226

218-
219-
def compute_and_save_anns_gt(base_vecs: np.ndarray, query_vecs: np.ndarray, float_space: FloatSpace, k: int, out_path: Path):
227+
def compute_and_save_anns_gt(
228+
base_vecs: np.ndarray, query_vecs: np.ndarray, float_space: FloatSpace, k: int, out_path: Path
229+
):
220230
"""Computes ANNS ground truth against the full base dataset."""
221231
n_queries = len(query_vecs)
222232
gt_matrix = np.zeros((n_queries, k), dtype=np.int32)
223233
print(f"Generating ANNS Ground Truth (Full Dataset, top-{k})...")
224234
start_t = time.perf_counter()
225-
235+
226236
for q in range(n_queries):
227237
dists = float_space.compute_distances(query_vecs[q], base_vecs)
228238
top_k = np.argpartition(dists, k)[:k]
229239
top_k = top_k[np.argsort(dists[top_k])]
230240
gt_matrix[q, :] = top_k
231241
if (q + 1) % 100 == 0 or (q + 1) == n_queries:
232242
elapsed_ms = int((time.perf_counter() - start_t) * 1000)
233-
print(f"\r Ground truth progress: {q + 1}/{n_queries} ({100.0 * (q + 1) / n_queries:.1f}%) after {elapsed_ms}ms", end="", flush=True)
243+
print(
244+
f"\r Ground truth progress: {q + 1}/{n_queries} ({100.0 * (q + 1) / n_queries:.1f}%) after {elapsed_ms}ms",
245+
end="",
246+
flush=True,
247+
)
234248
print()
235249
ivecs_write(out_path, gt_matrix)
236250
print(f"ANNS Ground Truth generated and saved to {out_path}")
237251

238-
def compute_and_save_anns_gt_half(base_vecs: np.ndarray, query_vecs: np.ndarray, float_space: FloatSpace, k: int, out_path: Path):
252+
253+
def compute_and_save_anns_gt_half(
254+
base_vecs: np.ndarray, query_vecs: np.ndarray, float_space: FloatSpace, k: int, out_path: Path
255+
):
239256
"""Computes ANNS ground truth against only the first half of the base dataset.
240257
241258
Mirrors the C++ generate_anns_groundtruth_files() with include_half=true,
242259
used for dynamic benchmarks where only half the base vectors are active.
243260
"""
244-
half_base = base_vecs[:len(base_vecs) // 2]
261+
half_base = base_vecs[: len(base_vecs) // 2]
245262
n_queries = len(query_vecs)
246263
gt_matrix = np.zeros((n_queries, k), dtype=np.int32)
247264
print(f"Generating ANNS Ground Truth (Half Dataset = {len(half_base)} vectors, top-{k})...")
248265
start_t = time.perf_counter()
249-
266+
250267
for q in range(n_queries):
251268
dists = float_space.compute_distances(query_vecs[q], half_base)
252269
top_k = np.argpartition(dists, k)[:k]
253270
top_k = top_k[np.argsort(dists[top_k])]
254271
gt_matrix[q, :] = top_k
255272
if (q + 1) % 100 == 0 or (q + 1) == n_queries:
256273
elapsed_ms = int((time.perf_counter() - start_t) * 1000)
257-
print(f"\r Half GT progress: {q + 1}/{n_queries} ({100.0 * (q + 1) / n_queries:.1f}%) after {elapsed_ms}ms", end="", flush=True)
274+
print(
275+
f"\r Half GT progress: {q + 1}/{n_queries} ({100.0 * (q + 1) / n_queries:.1f}%) after {elapsed_ms}ms",
276+
end="",
277+
flush=True,
278+
)
258279
print()
259280
ivecs_write(out_path, gt_matrix)
260281
print(f"ANNS Half Ground Truth generated and saved to {out_path}")
261282

283+
262284
def load_dataset_for_dynamic(
263285
dataset_key: str, cache_dir: Path
264286
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, Dict[str, Any]]:

‎examples/dynamic_data/main.py‎

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class DataStreamType(Enum):
2525
- AddHalfRemoveAndAddOneAtATime: Interleaved insertion and deletion operations.
2626
- AddAllRemoveHalf: Insert all vectors, then remove the second half.
2727
"""
28+
2829
AddHalf = "AddHalf"
2930
AddHalfRemoveAndAddOneAtATime = "AddHalfRemoveAndAddOneAtATime"
3031
AddAllRemoveHalf = "AddAllRemoveHalf"
@@ -48,6 +49,7 @@ class DataStreamType(Enum):
4849
# Give up opening the plot window if no benchmark data arrived within this many seconds.
4950
_FIRST_DATA_TIMEOUT = 3600.0
5051

52+
5153
def _render_combined_anns_plot(
5254
queue: multiprocessing.Queue,
5355
dataset_name: str,
@@ -130,24 +132,27 @@ def draw_msg(stream_idx: int, msg):
130132
plt.close("all")
131133

132134

133-
134-
135135
# ---------------------------------------------------------------------------
136136
# Linear search baseline
137137
# ---------------------------------------------------------------------------
138138

139-
def compute_linear_search_baseline(base_vecs: np.ndarray, float_space: deglib.distances.FloatSpace, sample_size: int = 100) -> float:
139+
140+
def compute_linear_search_baseline(
141+
base_vecs: np.ndarray, float_space: deglib.distances.FloatSpace, sample_size: int = 100
142+
) -> float:
140143
n_base = len(base_vecs)
141144
query_count = min(sample_size, n_base)
142145
rng = np.random.default_rng(7)
143146
sample_indices = rng.choice(n_base, size=query_count, replace=False)
144147

145148
print(f"\n--- Computing Linear Search Baseline ---")
146-
print(f"Computing linear search baseline with {query_count} random queries on {n_base} base vectors "
147-
f"using C++ {float_space.metric().name} ({float_space.get_instruction().name})...")
149+
print(
150+
f"Computing linear search baseline with {query_count} random queries on {n_base} base vectors "
151+
f"using C++ {float_space.metric().name} ({float_space.get_instruction().name})..."
152+
)
148153

149154
start_time = time.perf_counter()
150-
min_dist = float('inf')
155+
min_dist = float("inf")
151156

152157
for idx in sample_indices:
153158
dists = float_space.compute_distances(base_vecs[idx], base_vecs)
@@ -158,8 +163,10 @@ def compute_linear_search_baseline(base_vecs: np.ndarray, float_space: deglib.di
158163
total_time_us = (time.perf_counter() - start_time) * 1e6
159164
time_per_query_us = total_time_us / max(query_count, 1)
160165
linear_baseline_us = time_per_query_us * 2
161-
print(f"Linear search baseline: {int(time_per_query_us)}us per query "
162-
f"(total: {int(total_time_us / 1000)}ms for {query_count} queries) with min distance {min_dist:.6f}")
166+
print(
167+
f"Linear search baseline: {int(time_per_query_us)}us per query "
168+
f"(total: {int(total_time_us / 1000)}ms for {query_count} queries) with min distance {min_dist:.6f}"
169+
)
163170
return linear_baseline_us
164171

165172

@@ -168,6 +175,7 @@ def compute_linear_search_baseline(base_vecs: np.ndarray, float_space: deglib.di
168175
# {dims}D_K{k}_{stream_type_str}.deg in <graph_dir>/
169176
# ---------------------------------------------------------------------------
170177

178+
171179
def build_dynamic_graph_filename(graph_dir: Path, dims: int, k: int, stream_type: DataStreamType) -> Path:
172180
"""
173181
Returns the graph file path matching C++ naming:
@@ -182,6 +190,7 @@ def build_dynamic_graph_filename(graph_dir: Path, dims: int, k: int, stream_type
182190
# Graph building – simulation of the three DataStreamTypes
183191
# ---------------------------------------------------------------------------
184192

193+
185194
def build_dynamic_graph(
186195
base_vecs: np.ndarray,
187196
stream_type: DataStreamType,
@@ -251,7 +260,9 @@ def build_dynamic_graph(
251260
# 2nd loop: add base_size_quarter pairs from [quarter, half) and remove same number
252261
for i in range(quarter):
253262
builder.add_entry(int(all_labels[quarter + i]), base_vecs[quarter + i : quarter + i + 1])
254-
builder.add_entry(int(all_labels[half + quarter + i]), base_vecs[half + quarter + i : half + quarter + i + 1])
263+
builder.add_entry(
264+
int(all_labels[half + quarter + i]), base_vecs[half + quarter + i : half + quarter + i + 1]
265+
)
255266
builder.remove_entry(int(all_labels[half + (i * 2) + 0]))
256267
builder.remove_entry(int(all_labels[half + (i * 2) + 1]))
257268

@@ -266,19 +277,20 @@ def build_dynamic_graph(
266277
build_time = time.perf_counter() - build_start
267278
print(f"Graph built in {build_time:.2f} seconds ({graph_mut.size()} vertices).")
268279

269-
if graph_path:
270-
graph_path.parent.mkdir(parents=True, exist_ok=True)
271-
graph_mut.save_graph(str(graph_path))
272-
print(f"Graph saved to: {graph_path}")
273-
return deglib.load_readonly_graph(str(graph_path))
274-
else:
275-
return graph_mut.to_readonly()
280+
if graph_path:
281+
graph_path.parent.mkdir(parents=True, exist_ok=True)
282+
graph_mut.save_graph(str(graph_path))
283+
print(f"Graph saved to: {graph_path}")
284+
return deglib.load_readonly_graph(str(graph_path))
285+
else:
286+
return graph_mut.to_readonly()
276287

277288

278289
# ---------------------------------------------------------------------------
279290
# Main benchmark function
280291
# ---------------------------------------------------------------------------
281292

293+
282294
def run_dynamic_benchmark(
283295
dataset_key: str,
284296
cache_dir: Path,
@@ -317,7 +329,9 @@ def run_dynamic_benchmark(
317329
print(f"Selected Instruction Set: {instruction_set}")
318330
print(f"Build Threads: {build_threads}")
319331
print(f"Base vectors: {base_vecs.shape[0]} | Dimension: {dims}")
320-
print(f"Query vectors: {query_vecs.shape[0]} | Full-GT shape: {gt_vecs_full.shape} | Half-GT shape: {gt_vecs_half.shape}")
332+
print(
333+
f"Query vectors: {query_vecs.shape[0]} | Full-GT shape: {gt_vecs_full.shape} | Half-GT shape: {gt_vecs_half.shape}"
334+
)
321335

322336
feature_space = deglib.distances.FloatSpace.create(dims, metric, instruction_enum)
323337
actual_inst = str(feature_space.get_instruction().name)
@@ -418,7 +432,9 @@ def run_dynamic_benchmark(
418432
anns_recalls.append(recall)
419433
anns_qps.append(qps)
420434

421-
print(f"eps {eps:6.3f} \trecall {recall:.5f} \ttime_us_per_query {time_us_per_query:6d}us \t{qps:9.1f} qps \tsearch time: {int(search_time_us / 1000):6d}ms")
435+
print(
436+
f"eps {eps:6.3f} \trecall {recall:.5f} \ttime_us_per_query {time_us_per_query:6d}us \t{qps:9.1f} qps \tsearch time: {int(search_time_us / 1000):6d}ms"
437+
)
422438

423439
if linear_baseline_us > 0 and time_us_per_query > linear_baseline_us:
424440
print(f"eps {eps:.3f} \t ABORTED ({time_us_per_query}us/query > {int(linear_baseline_us)}us baseline)")

‎examples/dynamic_data/presets.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,12 @@
5757
},
5858
}
5959

60+
6061
def get_preset(dataset_key: str) -> Dict[str, Any]:
6162
"""Returns preset graph build and benchmark parameters for dataset_key."""
6263
key = dataset_key.lower()
6364
if key not in DYNAMIC_DATASET_PRESETS:
64-
raise ValueError(f"No preset for dataset '{dataset_key}'. Standard presets: {list(DYNAMIC_DATASET_PRESETS.keys())}")
65+
raise ValueError(
66+
f"No preset for dataset '{dataset_key}'. Standard presets: {list(DYNAMIC_DATASET_PRESETS.keys())}"
67+
)
6568
return DYNAMIC_DATASET_PRESETS[key]

‎examples/knng/dataset_utils.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""
22
dataset_utils.py — Helper utilities for downloading and loading SISAP 2026 Task 1 datasets (HDF5 format).
33
"""
4+
45
from __future__ import annotations
56

67
import os
@@ -25,6 +26,7 @@ def ensure_small_dataset(cache_dir: Path | None = None) -> Path:
2526
# 1. Try HuggingFace Hub native download/cache
2627
try:
2728
from huggingface_hub import hf_hub_download
29+
2830
downloaded = hf_hub_download(
2931
repo_id=HF_REPO_ID,
3032
filename=HF_SMALL_FILE,
@@ -49,9 +51,12 @@ def ensure_small_dataset(cache_dir: Path | None = None) -> Path:
4951

5052
# Fallback to direct HTTP download with progress reporter
5153
print(f"Downloading small dataset from {HF_SMALL_URL} to {target_path}...")
54+
5255
def _progress(count, block_size, total_size):
5356
percent = int(count * block_size * 100 / max(total_size, 1))
54-
sys.stdout.write(f"\rDownloading dataset... {percent}% ({count * block_size / 1024 / 1024:.1f} MB / {total_size / 1024 / 1024:.1f} MB)")
57+
sys.stdout.write(
58+
f"\rDownloading dataset... {percent}% ({count * block_size / 1024 / 1024:.1f} MB / {total_size / 1024 / 1024:.1f} MB)"
59+
)
5560
sys.stdout.flush()
5661

5762
urllib.request.urlretrieve(HF_SMALL_URL, target_path, _progress)
@@ -71,7 +76,7 @@ def load_hdf5_dataset(file_path: str | Path, max_vecs: int | None = None) -> tup
7176
with h5py.File(path, "r") as f:
7277
if "train" not in f:
7378
raise KeyError(f"Expected dataset key 'train' not found in {path}. Keys found: {list(f.keys())}")
74-
79+
7580
dataset = f["train"]
7681
if max_vecs is not None and max_vecs < dataset.shape[0]:
7782
train_data = dataset[:max_vecs]
@@ -116,7 +121,7 @@ def generate_synthetic_dataset(num_vecs: int = 2000, dims: int = 1024, seed: int
116121

117122
# Compute exact inner product matrix for ground truth top 15
118123
sim_matrix = np.dot(data, data.T)
119-
np.fill_diagonal(sim_matrix, -np.inf) # Exclude self
124+
np.fill_diagonal(sim_matrix, -np.inf) # Exclude self
120125
gt_data = np.argsort(-sim_matrix, axis=1)[:, :15]
121126

122127
return data, gt_data

0 commit comments

Comments
 (0)