File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -19,6 +19,26 @@ typedef u32 VectorDims;
1919*/
2020#define MAX_VECTOR_SZ 65536
2121
22+ /*
23+ * on-disk binary format for vector of different types:
24+ * 1. float32
25+ * [data[0] as f32] [data[1] as f32] ... [data[dims - 1] as f32] [1 as u8]?
26+ * - last 'type'-byte is optional for float32 vectors
27+ *
28+ * 2. float64
29+ * [data[0] as f64] [data[1] as f64] ... [data[dims - 1] as f64] [2 as u8]
30+ * - last 'type'-byte is mandatory for float64 vectors
31+ *
32+ * 3. float1bit
33+ * [data[0] as u8] [data[1] as u8] ... [data[(dims + 7) / 8] as u8] [_ as u8; padding]? [leftover as u8] [3 as u8]
34+ * - every data byte (except for the last) represents exactly 8 components of the vector
35+ * - last data byte represents [1..8] components of the vector
36+ * - optional padding byte ensures that leftover byte will be written at the odd blob position (0-based)
37+ * - leftover byte specify amount of trailing *bits* in the blob without last 'type'-byte which must be omitted
38+ * (so, vector dimensions are equal to 8 * (blob_size - 1) - leftover)
39+ * - last 'type'-byte is mandatory for float1bit vectors
40+ */
41+
2242/*
2343 * Enumerate of supported vector types (0 omitted intentionally as we can use zero as "undefined" value)
2444*/
You can’t perform that action at this time.
0 commit comments