Vector index Interface (#6090)

* LZY: Implement the ddl for vector property

Signed-off-by: lzy <[email protected]>

* LZY: dml for vertex of vector type finished and add new mock test
LZY: dml fix bug and remove useless code

Signed-off-by: lzy <[email protected]>

* LZY: vector index interface and simple ann benchmark finished

Signed-off-by: lzy <[email protected]>

---------

Signed-off-by: lzy <[email protected]>

Author: tom-jerr

cmake/nebula/ThirdPartyConfig.cmake

@@ -119,6 +119,7 @@ find_package(FLEX REQUIRED)
 find_package(LibLZMA REQUIRED)
 find_package(Fizz REQUIRED)
 find_package(Sodium REQUIRED)
+link_directories("${NEBULA_THIRDPARTY_ROOT}/lib")
 if (ENABLE_BREAKPAD)
     find_package(Breakpad REQUIRED)
 endif()

src/codec/RowReaderV2.cpp

@@ -330,7 +330,6 @@ Value RowReaderV2::getValueByIndex(const int64_t index) const {
 
 Value RowReaderV2::getVectorValueByName(const std::string& prop) const {
   int64_t index = schema_->getVectorFieldIndex(prop);
-  LOG(ERROR) << "LZY getVectorValueByName() prop: " << prop << ", index: " << index;
   return getVectorValueByIndex(index);
 }
 
@@ -361,7 +360,7 @@ Value RowReaderV2::getVectorValueByIndex(const int64_t index) const {
     case PropertyType::UNKNOWN:
       break;
     default: {
-      LOG(ERROR) << "Unsupported vector property type: " << static_cast<int>(field->type());
+      LOG(FATAL) << "Unsupported vector property type: " << static_cast<int>(field->type());
     }
   }
   LOG(FATAL) << "Should not reach here, illegal property type: " << static_cast<int>(field->type());

src/codec/RowReaderWrapper.cpp

@@ -27,6 +27,26 @@ RowReaderWrapper RowReaderWrapper::getTagPropReader(meta::SchemaManager* schemaM
   }
 }
 
+RowReaderWrapper RowReaderWrapper::getTagPropReader(meta::SchemaManager* schemaMan,
+                                                    GraphSpaceID space,
+                                                    TagID tag,
+                                                    PropID vectorIndex,
+                                                    folly::StringPiece row) {
+  SchemaVer schemaVer;
+  int32_t readerVer;
+  RowReaderWrapper::getVersions(row, schemaVer, readerVer);
+  if (schemaVer >= 0) {
+    auto schema = schemaMan->getTagSchema(space, tag, schemaVer);
+    if (schema == nullptr) {
+      return RowReaderWrapper();
+    }
+    return RowReaderWrapper(schema.get(), row, readerVer, true, static_cast<int32_t>(vectorIndex));
+  } else {
+    LOG(WARNING) << "Invalid schema version in the row data!";
+    return RowReaderWrapper();
+  }
+}
+
 // static
 RowReaderWrapper RowReaderWrapper::getEdgePropReader(meta::SchemaManager* schemaMan,
                                                      GraphSpaceID space,

src/codec/RowReaderWrapper.h

@@ -64,6 +64,11 @@ class RowReaderWrapper {
                                            GraphSpaceID space,
                                            TagID tag,
                                            folly::StringPiece row);
+  static RowReaderWrapper getTagPropReader(meta::SchemaManager* schemaMan,
+                                           GraphSpaceID space,
+                                           TagID tag,
+                                           PropID vectorIndex,
+                                           folly::StringPiece row);
 
   /**
    * @brief Generate a row reader wrapper of edge data

src/codec/RowWriterV2.cpp

@@ -226,8 +226,6 @@ RowWriterV2::RowWriterV2(const meta::NebulaSchemaProvider* schema,
 
     // Null flags
     size_t numNullables = schema_->vectorField(vector_field_index_)->nullable();
-    LOG(ERROR) << "LZY Vector field index: " << vector_field_index_
-               << ", numNullables: " << numNullables;
     if (numNullables > 0) {
       numNullBytes_ = ((numNullables - 1) >> 3) + 1;
     }
@@ -382,8 +380,6 @@ void RowWriterV2::processV2EncodedStrVector() {
 
   // Null flags
   size_t numNullables = schema_->vectorField(vector_field_index_)->nullable();
-  LOG(ERROR) << "LZY Vector field index: " << vector_field_index_
-             << ", numNullables: " << numNullables;
   if (numNullables > 0) {
     numNullBytes_ = ((numNullables - 1) >> 3) + 1;
   } else {

src/common/CMakeLists.txt

@@ -27,3 +27,4 @@ nebula_add_subdirectory(geo)
 nebula_add_subdirectory(memory)
 nebula_add_subdirectory(id)
 nebula_add_subdirectory(log)
+nebula_add_subdirectory(vectorIndex)

src/common/meta/NebulaSchemaProvider.h

@@ -254,7 +254,6 @@ class NebulaSchemaProvider {
                       std::string defaultValue = "",
                       cpp2::GeoShape geoShape = cpp2::GeoShape::ANY);
 
-
   void setProp(cpp2::SchemaProp schemaProp);
 
   const cpp2::SchemaProp getProp() const;
@@ -267,7 +266,7 @@ class NebulaSchemaProvider {
 
   bool hasVectorCol() const {
     return !vector_fields_.empty();
-
+  }
 
  private:
   std::size_t fieldSize(nebula::cpp2::PropertyType type, std::size_t fixedStrLimit);

src/common/thrift/ThriftTypes.h

@@ -17,6 +17,7 @@ using PartitionID = int32_t;
 using TermID = int64_t;
 using LogID = int64_t;
 using Port = int32_t;
+using VectorID = int64_t;
 
 // Starting from 2.0, both string and int64 vertex ids will be supported.
 //

src/common/utils/NebulaKeyUtils.cpp

@@ -240,7 +240,8 @@ std::string NebulaKeyUtils::vectorTagPrefix(size_t vIdLen,
                                             PartitionID partId,
                                             const VertexID& vId) {
   CHECK_GE(vIdLen, vId.size());
-  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVector_);
+  PartitionID item = (partId << kPartitionOffset) | static_cast<uint32_t>(NebulaKeyType::kVector_) |
+                     static_cast<uint32_t>(NebulaKeyType::kTag_);
   std::string key;
   key.reserve(sizeof(PartitionID) + vIdLen);
   key.append(reinterpret_cast<const char*>(&item), sizeof(PartitionID))

src/common/utils/NebulaKeyUtils.h

@@ -6,6 +6,7 @@
 #ifndef COMMON_UTILS_NEBULAKEYUTILS_H_
 #define COMMON_UTILS_NEBULAKEYUTILS_H_
 
+#include "common/thrift/ThriftTypes.h"
 #include "common/utils/Types.h"
 
 namespace nebula {
@@ -210,6 +211,14 @@ class NebulaKeyUtils final {
     return readInt<TagID>(rawKey.data() + offset, sizeof(TagID));
   }
 
+  static PropID getTagPropId(size_t vIdLen, const folly::StringPiece& rawKey) {
+    if (rawKey.size() != kVectorTagLen + vIdLen) {
+      dumpBadKey(rawKey, kVectorTagLen + vIdLen, vIdLen);
+    }
+    auto offset = sizeof(PartitionID) + vIdLen + sizeof(TagID);
+    return readInt<PropID>(rawKey.data() + offset, sizeof(PropID));
+  }
+
   static bool isEdge(size_t vIdLen, const folly::StringPiece& rawKey, char suffix = kEdgeVersion) {
     if (rawKey.size() != kEdgeLen + (vIdLen << 1)) {
       return false;
@@ -229,9 +238,6 @@ class NebulaKeyUtils final {
   }
 
   static bool isVector(const folly::StringPiece& rawKey) {
-    // if (rawKey.size() != kVectorTagLen + vIdLen) {
-    //   return false;
-    // }
     constexpr int32_t len = static_cast<int32_t>(sizeof(NebulaKeyType));
     auto type = readInt<uint32_t>(rawKey.data(), len) & kTypeMaskWithVector;
     return static_cast<NebulaKeyType>(type) == NebulaKeyType::kVector_;
@@ -371,7 +377,7 @@ class NebulaKeyUtils final {
     auto offset = sizeof(PartitionID) + vIdLen + sizeof(EdgeType);
     return NebulaKeyUtils::decodeRank(rawKey.data() + offset);
   }
-  static PropID getPropID(size_t vIdLen, const folly::StringPiece& rawKey) {
+  static PropID getEdgePropID(size_t vIdLen, const folly::StringPiece& rawKey) {
     if (rawKey.size() < kVectorEdgeLen + (vIdLen << 1)) {
       dumpBadKey(rawKey, kVectorEdgeLen + (vIdLen << 1), vIdLen);
     }