cleanup code a bit in order to simplify working with vector of different types

Author: sivukhin

libsql-sqlite3/src/vector.c

@@ -252,11 +252,29 @@ int vectorParseSqliteBlob(
   Vector *pVector,
   char **pzErrMsg
 ){
+  const unsigned char *pBlob;
+  size_t nBlobSize;
+
+  assert( sqlite3_value_type(arg) == SQLITE_BLOB );
+
+  pBlob = sqlite3_value_blob(arg);
+  nBlobSize = sqlite3_value_bytes(arg);
+  if( nBlobSize % 2 == 1 ){
+    nBlobSize--;
+  }
+
+  if( nBlobSize < vectorDataSize(pVector->type, pVector->dims) ){
+    *pzErrMsg = sqlite3_mprintf("invalid vector: not enough bytes: type=%d, dims=%d, size=%ull", pVector->type, pVector->dims, nBlobSize);
+    return SQLITE_ERROR;
+  }
+
   switch (pVector->type) {
     case VECTOR_TYPE_FLOAT32: 
-      return vectorF32ParseSqliteBlob(arg, pVector, pzErrMsg);
+      vectorF32DeserializeFromBlob(pVector, pBlob, nBlobSize);
+      return 0;
     case VECTOR_TYPE_FLOAT64: 
-      return vectorF64ParseSqliteBlob(arg, pVector, pzErrMsg);
+      vectorF64DeserializeFromBlob(pVector, pBlob, nBlobSize);
+      return 0;
     default: 
       assert(0);
   }
@@ -384,20 +402,47 @@ void vectorMarshalToText(
   }
 }
 
-void vectorSerialize(
+void vectorSerializeWithType(
   sqlite3_context *context,
   const Vector *pVector
 ){
+  unsigned char *pBlob;
+  size_t nBlobSize, nDataSize;
+
+  assert( pVector->dims <= MAX_VECTOR_SZ );
+
+  nDataSize = vectorDataSize(pVector->type, pVector->dims);
+  nBlobSize = nDataSize;
+  if( pVector->type != VECTOR_TYPE_FLOAT32 ){
+    nBlobSize += (nBlobSize % 2 == 0 ? 1 : 2);
+  }
+
+  if( nBlobSize == 0 ){
+    sqlite3_result_zeroblob(context, 0);
+    return;
+  }
+
+  pBlob = sqlite3_malloc64(nBlobSize);
+  if( pBlob == NULL ){
+    sqlite3_result_error_nomem(context);
+    return;
+  }
+
+  if( pVector->type != VECTOR_TYPE_FLOAT32 ){
+    pBlob[nBlobSize - 1] = pVector->type;
+  }
+
   switch (pVector->type) {
     case VECTOR_TYPE_FLOAT32:
-      vectorF32Serialize(context, pVector);
+      vectorF32SerializeToBlob(pVector, pBlob, nDataSize);
       break;
     case VECTOR_TYPE_FLOAT64:
-      vectorF64Serialize(context, pVector);
+      vectorF64SerializeToBlob(pVector, pBlob, nDataSize);
       break;
     default:
       assert(0);
   }
+  sqlite3_result_blob(context, (char*)pBlob, nBlobSize, sqlite3_free);
 }
 
 size_t vectorSerializeToBlob(const Vector *pVector, unsigned char *pBlob, size_t nBlobSize){
@@ -412,18 +457,6 @@ size_t vectorSerializeToBlob(const Vector *pVector, unsigned char *pBlob, size_t
   return 0;
 }
 
-size_t vectorDeserializeFromBlob(Vector *pVector, const unsigned char *pBlob, size_t nBlobSize){
-  switch (pVector->type) {
-    case VECTOR_TYPE_FLOAT32:
-      return vectorF32DeserializeFromBlob(pVector, pBlob, nBlobSize);
-    case VECTOR_TYPE_FLOAT64:
-      return vectorF64DeserializeFromBlob(pVector, pBlob, nBlobSize);
-    default:
-      assert(0);
-  }
-  return 0;
-}
-
 void vectorInitFromBlob(Vector *pVector, const unsigned char *pBlob, size_t nBlobSize){
   switch (pVector->type) {
     case VECTOR_TYPE_FLOAT32:
@@ -470,7 +503,7 @@ static void vectorFuncHintedType(
     sqlite3_free(pzErrMsg);
     goto out_free_vec;
   }
-  vectorSerialize(context, pVector);
+  vectorSerializeWithType(context, pVector);
 out_free_vec:
   vectorFree(pVector);
 }

libsql-sqlite3/src/vectorInt.h

@@ -65,13 +65,6 @@ size_t vectorSerializeToBlob   (const Vector *, unsigned char *, size_t);
 size_t vectorF32SerializeToBlob(const Vector *, unsigned char *, size_t);
 size_t vectorF64SerializeToBlob(const Vector *, unsigned char *, size_t);
 
-/*
- * Deserializes vector from the blob in little-endian format according to the IEEE-754 standard
-*/
-size_t vectorDeserializeFromBlob   (Vector *, const unsigned char *, size_t);
-size_t vectorF32DeserializeFromBlob(Vector *, const unsigned char *, size_t);
-size_t vectorF64DeserializeFromBlob(Vector *, const unsigned char *, size_t);
-
 /* 
  * Calculates cosine distance between two vectors (vector must have same type and same dimensions)
 */
@@ -91,16 +84,15 @@ double vectorF64DistanceL2(const Vector *, const Vector *);
  * LibSQL can append one trailing byte in the end of final blob. This byte will be later used to determine type of the blob
  * By default, blob with even length will be treated as a f32 blob
 */
-void vectorSerialize   (sqlite3_context *, const Vector *);
-void vectorF32Serialize(sqlite3_context *, const Vector *);
-void vectorF64Serialize(sqlite3_context *, const Vector *);
+void vectorSerializeWithType(sqlite3_context *, const Vector *);
 
 /*
  * Parses Vector content from the blob; vector type and dimensions must be filled already
 */
 int vectorParseSqliteBlob   (sqlite3_value *, Vector *, char **);
-int vectorF32ParseSqliteBlob(sqlite3_value *, Vector *, char **);
-int vectorF64ParseSqliteBlob(sqlite3_value *, Vector *, char **);
+
+void vectorF32DeserializeFromBlob(Vector *, const unsigned char *, size_t);
+void vectorF64DeserializeFromBlob(Vector *, const unsigned char *, size_t);
 
 void vectorInitStatic(Vector *, VectorType, const unsigned char *, size_t);
 void vectorInitFromBlob(Vector *, const unsigned char *, size_t);

libsql-sqlite3/src/vectordiskann.c

@@ -47,6 +47,7 @@
 **    diskAnnInsert()          Insert single new(!) vector in an opened index
 **    diskAnnDelete()          Delete row by key from an opened index
 */
+#include "vectorInt.h"
 #ifndef SQLITE_OMIT_VECTOR
 
 #include "math.h"
@@ -1490,6 +1491,7 @@ int diskAnnOpenIndex(
 ){
   DiskAnnIndex *pIndex;
   u64 nBlockSize;
+  int compressNeighbours;
   pIndex = sqlite3DbMallocRaw(db, sizeof(DiskAnnIndex));
   if( pIndex == NULL ){
     return SQLITE_NOMEM;
@@ -1536,9 +1538,17 @@ int diskAnnOpenIndex(
     pIndex->searchL = VECTOR_SEARCH_L_DEFAULT;
   }
   pIndex->nNodeVectorSize = vectorDataSize(pIndex->nNodeVectorType, pIndex->nVectorDims);
-  // will change in future when we will support compression of edges vectors
-  pIndex->nEdgeVectorType = pIndex->nNodeVectorType;
-  pIndex->nEdgeVectorSize = pIndex->nNodeVectorSize;
+
+  compressNeighbours = vectorIdxParamsGetU64(pParams, VECTOR_COMPRESS_NEIGHBORS_PARAM_ID);
+  if( compressNeighbours == 0 ){
+    pIndex->nEdgeVectorType = pIndex->nNodeVectorType;
+    pIndex->nEdgeVectorSize = pIndex->nNodeVectorSize;
+  }else if( compressNeighbours == VECTOR_TYPE_1BIT ){
+    pIndex->nEdgeVectorType = VECTOR_TYPE_1BIT;
+    pIndex->nEdgeVectorSize = vectorDataSize(VECTOR_TYPE_1BIT, pIndex->nVectorDims);
+  }else{
+    return SQLITE_ERROR;
+  }
 
   *ppIndex = pIndex;
   return SQLITE_OK;

libsql-sqlite3/src/vectorfloat32.c

@@ -94,26 +94,6 @@ size_t vectorF32SerializeToBlob(
   return sizeof(float) * pVector->dims;
 }
 
-size_t vectorF32DeserializeFromBlob(
-  Vector *pVector,
-  const unsigned char *pBlob,
-  size_t nBlobSize
-){
-  float *elems = pVector->data;
-  unsigned i;
-  pVector->type = VECTOR_TYPE_FLOAT32;
-  pVector->dims = nBlobSize / sizeof(float);
-
-  assert( pVector->dims <= MAX_VECTOR_SZ );
-  assert( nBlobSize % 2 == 0 || pBlob[nBlobSize - 1] == VECTOR_TYPE_FLOAT32 );
-
-  for(i = 0; i < pVector->dims; i++){
-    elems[i] = deserializeF32(pBlob);
-    pBlob += sizeof(float);
-  }
-  return vectorDataSize(pVector->type, pVector->dims);
-}
-
 void vectorF32Serialize(
   sqlite3_context *context,
   const Vector *pVector
@@ -220,32 +200,22 @@ void vectorF32InitFromBlob(Vector *pVector, const unsigned char *pBlob, size_t n
   pVector->data = (void*)pBlob;
 }
 
-int vectorF32ParseSqliteBlob(
-  sqlite3_value *arg,
+void vectorF32DeserializeFromBlob(
   Vector *pVector,
-  char **pzErr
+  const unsigned char *pBlob,
+  size_t nBlobSize
 ){
-  const unsigned char *pBlob;
   float *elems = pVector->data;
   unsigned i;
 
   assert( pVector->type == VECTOR_TYPE_FLOAT32 );
   assert( 0 <= pVector->dims && pVector->dims <= MAX_VECTOR_SZ );
-  assert( sqlite3_value_type(arg) == SQLITE_BLOB );
-
-  pBlob = sqlite3_value_blob(arg);
-  if( sqlite3_value_bytes(arg) < sizeof(float) * pVector->dims ){
-    *pzErr = sqlite3_mprintf("invalid f32 vector: not enough bytes for all dimensions");
-    goto error;
-  }
+  assert( nBlobSize >= pVector->dims * sizeof(float) );
 
   for(i = 0; i < pVector->dims; i++){
     elems[i] = deserializeF32(pBlob);
     pBlob += sizeof(float);
   }
-  return 0;
-error:
-  return -1;
 }
 
 #endif /* !defined(SQLITE_OMIT_VECTOR) */

libsql-sqlite3/src/vectorfloat64.c

@@ -98,57 +98,6 @@ size_t vectorF64SerializeToBlob(
   return sizeof(double) * pVector->dims;
 }
 
-size_t vectorF64DeserializeFromBlob(
-  Vector *pVector,
-  const unsigned char *pBlob,
-  size_t nBlobSize
-){
-  double *elems = pVector->data;
-  unsigned i;
-  pVector->type = VECTOR_TYPE_FLOAT64;
-  pVector->dims = nBlobSize / sizeof(double);
-
-  assert( pVector->dims <= MAX_VECTOR_SZ );
-  assert( nBlobSize % 2 == 1 && pBlob[nBlobSize - 1] == VECTOR_TYPE_FLOAT64 );
-
-  for(i = 0; i < pVector->dims; i++){
-    elems[i] = deserializeF64(pBlob);
-    pBlob += sizeof(double);
-  }
-  return vectorDataSize(pVector->type, pVector->dims);
-}
-
-void vectorF64Serialize(
-  sqlite3_context *context,
-  const Vector *pVector
-){
-  double *elems = pVector->data;
-  unsigned char *pBlob;
-  size_t nBlobSize;
-
-  assert( pVector->type == VECTOR_TYPE_FLOAT64 );
-  assert( pVector->dims <= MAX_VECTOR_SZ );
-
-  // allocate one extra trailing byte with vector blob type metadata
-  nBlobSize = vectorDataSize(pVector->type, pVector->dims) + 1;
-
-  if( nBlobSize == 0 ){
-    sqlite3_result_zeroblob(context, 0);
-    return;
-  }
-
-  pBlob = sqlite3_malloc64(nBlobSize);
-  if( pBlob == NULL ){
-    sqlite3_result_error_nomem(context);
-    return;
-  }
-
-  vectorF64SerializeToBlob(pVector, pBlob, nBlobSize - 1);
-  pBlob[nBlobSize - 1] = VECTOR_TYPE_FLOAT64;
-
-  sqlite3_result_blob(context, (char*)pBlob, nBlobSize, sqlite3_free);
-}
-
 #define SINGLE_DOUBLE_CHAR_LIMIT 32
 void vectorF64MarshalToText(
   sqlite3_context *context,
@@ -227,32 +176,22 @@ void vectorF64InitFromBlob(Vector *pVector, const unsigned char *pBlob, size_t n
   pVector->data = (void*)pBlob;
 }
 
-int vectorF64ParseSqliteBlob(
-  sqlite3_value *arg,
+void vectorF64DeserializeFromBlob(
   Vector *pVector,
-  char **pzErr
+  const unsigned char *pBlob,
+  size_t nBlobSize
 ){
-  const unsigned char *pBlob;
   double *elems = pVector->data;
   unsigned i;
 
   assert( pVector->type == VECTOR_TYPE_FLOAT64 );
   assert( 0 <= pVector->dims && pVector->dims <= MAX_VECTOR_SZ );
-  assert( sqlite3_value_type(arg) == SQLITE_BLOB );
-
-  pBlob = sqlite3_value_blob(arg);
-  if( sqlite3_value_bytes(arg) < sizeof(double) * pVector->dims ){
-    *pzErr = sqlite3_mprintf("invalid f64 vector: not enough bytes for all dimensions");
-    goto error;
-  }
+  assert( nBlobSize >= pVector->dims * sizeof(double) );
 
   for(i = 0; i < pVector->dims; i++){
     elems[i] = deserializeF64(pBlob);
     pBlob += sizeof(double);
   }
-  return 0;
-error:
-  return -1;
 }
 
 #endif /* !defined(SQLITE_OMIT_VECTOR) */