pax_global_header00006660000000000000000000000064152057322170014516gustar00rootroot0000000000000052 comment=37494ead55986a790ec44686ed8965171bb1c45f libkysdk-vector-engine-client-1.2.0.0/000077500000000000000000000000001520573221700175075ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/.clang-format000066400000000000000000000002131520573221700220560ustar00rootroot00000000000000--- Language: Cpp BasedOnStyle: Google ColumnLimit: 120 IndentWidth: 4 AccessModifierOffset: -4 TabWidth: 4 #DerivePointerAlignment: false libkysdk-vector-engine-client-1.2.0.0/.gitignore000066400000000000000000000035171520573221700215050ustar00rootroot00000000000000.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ *.db compile_commands.json # clion .idea cmake-build-debug/ libkysdk-vector-engine-client-1.2.0.0/CMakeLists.txt000066400000000000000000000065211520573221700222530ustar00rootroot00000000000000# Licensed to the LF AI & Data foundation under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. cmake_minimum_required(VERSION 3.5) project(kysdk-vector-engine-client LANGUAGES CXX C) set(CMAKE_CXX_STANDARD 17) find_package(PkgConfig REQUIRED) find_package(nlohmann_json REQUIRED) # 向量数据库只支持 x86_64|aarch64，其他架构不支持，sw_64 if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|aarch64") message(STATUS "Building for ${CMAKE_SYSTEM_PROCESSOR} architecture") file(GLOB_RECURSE SOURCE_LIST ${PROJECT_SOURCE_DIR}/src/*.cpp) include_directories(${PROJECT_SOURCE_DIR}/include/kysdk-vector-engine-client ${PROJECT_SOURCE_DIR}/src/impl) pkg_check_modules(GRPC REQUIRED IMPORTED_TARGET grpc) find_package(Protobuf REQUIRED) find_package(absl REQUIRED) find_package(KylinAiProto REQUIRED) kylin_ai_generate_grpc_proto_source(RETURNED_GRPC_SOURCE_FILES vector-db schema common msg feder rg milvus) message("grpc source files: ${RETURNED_GRPC_SOURCE_FILES}") kylin_ai_generate_grpc_proto_service(RETURNED_GRPC_SERVICE_FILES vector-db milvus) message("grpc source files: ${RETURNED_GRPC_SERVICE_FILES}") add_library(kysdk-vector-engine-client SHARED ${SOURCE_LIST} ${RETURNED_GRPC_SOURCE_FILES} ${RETURNED_GRPC_SERVICE_FILES}) target_link_libraries(kysdk-vector-engine-client PUBLIC grpc++ PUBLIC protobuf PUBLIC absl::synchronization PUBLIC nlohmann_json::nlohmann_json) else() message(STATUS "Building for ${CMAKE_SYSTEM_PROCESSOR} architecture") add_definitions(-DENABLE_SW_ARCH) file(GLOB_RECURSE SOURCE_LIST ${PROJECT_SOURCE_DIR}/src/swImpl/*.cpp ${PROJECT_SOURCE_DIR}/src/types/*.cpp ${PROJECT_SOURCE_DIR}/src/Database.cpp) include_directories(${PROJECT_SOURCE_DIR}/include/kysdk-vector-engine-client ${PROJECT_SOURCE_DIR}/src/swImpl) add_library(kysdk-vector-engine-client SHARED ${SOURCE_LIST}) target_link_libraries(kysdk-vector-engine-client PUBLIC nlohmann_json::nlohmann_json) endif() # 设置动态库版本号 set_target_properties(kysdk-vector-engine-client PROPERTIES VERSION 0.0.1 SOVERSION 1) install(TARGETS kysdk-vector-engine-client DESTINATION ${CMAKE_INSTALL_LIBDIR}) install(DIRECTORY include/kysdk-vector-engine-client DESTINATION include) install(FILES usr/share/pkgconfig/kysdk-vector-engine-client.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) if (DEFINED ENABLE_DEMO) unset(ENABLE_DEMO CACHE) endif() option(ENABLE_DEMO "Build Demo" OFF) if (ENABLE_DEMO) add_subdirectory(demo) endif(ENABLE_DEMO) libkysdk-vector-engine-client-1.2.0.0/LICENSE000066400000000000000000000261351520573221700205230ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. libkysdk-vector-engine-client-1.2.0.0/README.md000066400000000000000000000002211520573221700207610ustar00rootroot00000000000000# libkysdk-vector-engine-client Cpp client of kylin-ai-vector-engine, modify from [milvus-sdk-cpp](https://github.com/milvus-io/milvus-sdk-cpp).libkysdk-vector-engine-client-1.2.0.0/demo/000077500000000000000000000000001520573221700204335ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/demo/CMakeLists.txt000066400000000000000000000001711520573221700231720ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.5) project(demo LANGUAGES CXX C) set(CMAKE_CXX_STANDARD 17) add_subdirectory(client) libkysdk-vector-engine-client-1.2.0.0/demo/client/000077500000000000000000000000001520573221700217115ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/demo/client/CMakeLists.txt000066400000000000000000000012401520573221700244460ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.5) project(demo LANGUAGES CXX C) set(CMAKE_CXX_STANDARD 17) find_package(PkgConfig REQUIRED) pkg_check_modules(UUID REQUIRED IMPORTED_TARGET uuid) add_executable(client-milvus milvus.cpp) target_link_libraries(client-milvus PUBLIC kysdk-vector-engine-client PUBLIC PkgConfig::UUID) add_executable(client-schema exampleSchema.cpp) target_link_libraries(client-schema PUBLIC kysdk-vector-engine-client) add_executable(client-example example.cpp example.h) target_link_libraries(client-example PUBLIC kysdk-vector-engine-client) add_executable(client-test test.cpp) target_link_libraries(client-test PUBLIC kysdk-vector-engine-client) libkysdk-vector-engine-client-1.2.0.0/demo/client/example.cpp000066400000000000000000000270231520573221700240540ustar00rootroot00000000000000// This file has been modified by KylinSoft Co., Ltd. // Modifications copyright (C) 2024-2026 KylinSoft Co., Ltd. // // Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "example.h" using namespace std; using namespace VectorDB; vector> insert_vectors; vector insert_jsons; void CheckStatus(string&& prefix, const Status& status) { if (!status.IsOk()) { cout << prefix << " " << static_cast(status.Code()) << " " << status.Message() << endl; exit(1); } } Example::Example() { client = Database::Create(); /** * connect 有两种调用方式： * 1、传入ConnectParam配置安全信息等，参考 exampleSchema.cpp * 2、简化方式：无需传参，未设置安全信息 */ auto status = client->Connect(); CheckStatus("Failed to connect milvus server:", status); cout << "Connect to milvus server success." << endl; } bool Example::HasCollection() { bool exist; auto status = client->HasCollection(collection_name, exist); // todo 临时处理，向量数据库要修改 HasCollection 的返回 // 如果返回报错也认为不存在，Failed to has TEST collection: 1002: collection not found if (status.Code() == StatusCode::SERVER_FAILED) { return false; } CheckStatus("Failed to has TEST collection:", status); cout << "Have TEST collection or not: " << exist << endl; return exist; } void Example::CreateCollection() { if (HasCollection()) { return; } /** * create collection 有两种调用方式： * 1、传入schema和index，参考 exampleSchema.cpp * 2、简化方式：只传name和dim，默认只创建 id 和 vector field，id默认开启自增 */ auto status = client->CreateCollection(collection_name, dimension); CheckStatus("Failed to create collection:", status); cout << "Successfully create collection." << endl; } void Example::DropCollection() { auto status = client->DropCollection(collection_name); CheckStatus("Failed to drop TEST collection:", status); cout << "Drop TEST collection success." << endl; } /** {"id": 1, "vector": rng.random((1, dim))[0], "a": 100, "g": 101}, {"id": 2, "vector": rng.random((1, dim))[0], "b": 200, "h": 201}, {"id": 3, "vector": rng.random((1, dim))[0], "c": 300, "i": 301}, {"id": 4, "vector": rng.random((1, dim))[0], "d": 400, "j": 401}, {"id": 5, "vector": rng.random((1, dim))[0], "e": 500, "k": 501}, {"id": 6, "vector": rng.random((1, dim))[0], "f": 600, "l": 601}, */ void GenerateData(int dimension) { uniform_int_distribution int_gen(1, 100); uniform_real_distribution float_gen(0.0, 1.0); default_random_engine ran(time(nullptr)); int row_count = 6; string seed = "abcdefghijklmn"; for (auto i = 0; i < row_count; ++i) { // insert_ids.push_back(i + 1); vector vector(dimension); for (auto j = 0; j < dimension; ++j) { vector[j] = float_gen(ran); } insert_vectors.emplace_back(vector); nlohmann::json value = nlohmann::json::object(); string key; key += seed[i]; value[key] = 100 * (i + 1); key = ""; key += seed[i + 6]; value[key] = 100 * (i + 1) + 1; insert_jsons.push_back(value); } } void Example::Insert() { GenerateData(dimension); // 设置动态扩展字段，名字一定要使用 DYNAMIC_FIELD_NAME 来获取，否则查询时找不到 // 快捷创建collection的方式，要用默认字段的名称 // 如果开启了id自增，无需传入 make_shared(DEFAULT_ID_FIELD_NAME, insert_ids) vector fields_data{make_shared(DEFAULT_VECTOR_FIELD_NAME, insert_vectors), make_shared(DYNAMIC_FIELD_NAME, insert_jsons)}; DmlResults dml_results; auto status = client->Insert(collection_name, fields_data, dml_results); CheckStatus("Failed to insert:", status); cout << "Successfully insert " << dml_results.IdArray().IntIDArray().size() << " rows." << endl; } void Example::Search() { // 设置返回条数 SearchArguments arguments(collection_name, 10); // 查询输出哪些列 arguments.AddOutputField(DEFAULT_ID_FIELD_NAME); arguments.AddOutputField(DYNAMIC_FIELD_NAME); arguments.AddOutputField(DEFAULT_VECTOR_FIELD_NAME); // 可加查询条件，写法参考：https://milvus.io/docs/zh/boolean.md // arguments.SetExpression(field_id_name + " in [0,1,2,3]"); // set to strong guarantee so that the search is executed after the inserted data is persisted arguments.SetGuaranteeTimestamp(GuaranteeStrongTs()); // 设置要搜索的向量，示例查询 id=2 的数据 cout << "Searching the Id.2 entity..." << endl; vector q_vector = insert_vectors[1]; arguments.AddTargetVector(DEFAULT_VECTOR_FIELD_NAME, move(q_vector)); SearchResults search_results{}; auto status = client->Search(arguments, search_results); CheckStatus("Failed to search:", status); cout << "Search success. result size: " << search_results.Results().size() << endl; for (auto& result : search_results.Results()) { auto& ids = result.Ids().IntIDArray(); auto& distances = result.Scores(); if (ids.size() != distances.size()) { cout << "Illegal result!" << endl; continue; } if (ids.empty()) { cout << "Empty result!" << endl; continue; } auto idField = result.OutputField(DEFAULT_ID_FIELD_NAME); shared_ptr idFieldPtr = static_pointer_cast(idField); auto& idData = idFieldPtr->Data(); cout << "Search id rows: " << idData.size() << endl; auto jsonField = result.OutputField(DYNAMIC_FIELD_NAME); shared_ptr jsonFieldPtr = static_pointer_cast(jsonField); auto& jsonData = jsonFieldPtr->Data(); cout << "Search json rows: " << jsonData.size() << endl; auto vectorField = result.OutputField(DEFAULT_VECTOR_FIELD_NAME); shared_ptr vectorFieldPtr = static_pointer_cast(vectorField); auto& vectorData = vectorFieldPtr->Data(); cout << "Search vector rows: " << vectorData.size() << endl; for (size_t i = 0; i < ids.size(); ++i) { cout << "ID: " << ids[i] << "\tDistance: " << distances[i] << "\t" << idFieldPtr->Name() << ": " << idData[i] << "\t" << jsonFieldPtr->Name() << ": " << jsonData[i] << "\tvector: ["; for (const auto& item : vectorData[i]) { cout << item << ", "; } cout << "]" << endl; // validate the age value // if (insert_ids[ids[i]] != idData[i]) { // cout << "ERROR! The returned value doesn't match the inserted value" << endl; // } } } } int64_t Example::Query() { QueryArguments arguments; arguments.SetCollectionName(collection_name); // 查询输出哪些列 arguments.AddOutputField(DEFAULT_ID_FIELD_NAME); arguments.AddOutputField(DYNAMIC_FIELD_NAME); arguments.AddOutputField(DEFAULT_VECTOR_FIELD_NAME); // 查询条件，写法参考：https://milvus.io/docs/zh/boolean.md arguments.SetExpression("b == 200"); QueryResults queryResults; auto status = client->Query(arguments, queryResults); CheckStatus("Failed to query:", status); cout << "Query success. output field num: " << queryResults.OutputFields().size() << endl; FieldDataPtr idField = queryResults.GetFieldByName(DEFAULT_ID_FIELD_NAME); shared_ptr idFieldPtr = static_pointer_cast(idField); auto& idData = idFieldPtr->Data(); cout << "Query id rows: " << idData.size() << endl; FieldDataPtr jsonField = queryResults.GetFieldByName(DYNAMIC_FIELD_NAME); shared_ptr jsonFieldPtr = static_pointer_cast(jsonField); auto& jsonData = jsonFieldPtr->Data(); cout << "Query json rows: " << jsonData.size() << endl; FieldDataPtr vectorField = queryResults.GetFieldByName(DEFAULT_VECTOR_FIELD_NAME); shared_ptr vectorFieldPtr = static_pointer_cast(vectorField); auto& vectorData = vectorFieldPtr->Data(); cout << "Query vector rows: " << vectorData.size() << endl; if (idData.empty()) { return 0; } for (int i = 0; i < idData.size(); ++i) { cout << idFieldPtr->Name() << ": " << idData[i] << "\t" << jsonFieldPtr->Name() << ": " << jsonData[i].dump() << "\t" << vectorFieldPtr->Name() << ": ["; for (const auto& item : vectorData[i]) { cout << item << ", "; } cout << "]" << endl; } return idData[0]; } // 更新数据是根据主键 id，示例是用Query() 返回的id值 void Example::Upsert(int64_t id) { nlohmann::json value = nlohmann::json::object(); value["b"] = 200; value["z"] = 205; vector fields_data{ make_shared(DEFAULT_ID_FIELD_NAME, vector{id}), make_shared(DEFAULT_VECTOR_FIELD_NAME, vector>{insert_vectors[0]}), make_shared(DYNAMIC_FIELD_NAME, vector{value})}; DmlResults results; auto status = client->Upsert(collection_name, fields_data, results); CheckStatus("Failed to upsert:", status); cout << "Successfully upsert " << results.IdArray().IntIDArray().size() << " rows." << endl; } void Example::Delete() { // string expression = "b == 200 || b == 300"; // string expression = "b not in [ 1,2 ]"; string expression = "b < 200"; DmlResults results; auto status = client->Delete(collection_name, expression, results); CheckStatus("Failed to delete:", status); cout << "Successfully Delete " << results.IdArray().IntIDArray().size() << " rows." << endl; } Example::~Example() { auto status = client->Disconnect(); CheckStatus("Failed to Disconnect milvus server:", status); cout << "Disconnect to milvus server success." << endl; } int main() { cout << "Example start..." << endl; shared_ptr example = make_shared(); // drop the collection if it exists example->DropCollection(); example->CreateCollection(); example->Insert(); cout << "=========== 查询数据 ===========" << endl; example->Search(); int64_t id = example->Query(); cout << "=========== 修改数据后查询数据 ===========" << endl; example->Upsert(id); example->Search(); example->Query(); cout << "=========== 删除数据后查询数据 ===========" << endl; example->Delete(); example->Search(); example->Query(); example->DropCollection(); return 0; } libkysdk-vector-engine-client-1.2.0.0/demo/client/example.h000066400000000000000000000027121520573221700235170ustar00rootroot00000000000000// This file has been modified by KylinSoft Co., Ltd. // Modifications copyright (C) 2024-2026 KylinSoft Co., Ltd. // // Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "Database.h" class Example { public: // 里面调用了 connect Example(); bool HasCollection(); void CreateCollection(); void DropCollection(); void Insert(); // 根据向量查询 void Search(); // 根据条件查询，比如主键 int64_t Query(); // 根据主键更新 void Upsert(int64_t id); void Delete(); ~Example(); private: std::shared_ptr client; std::string collection_name = "TEST"; int dimension = 4; }; libkysdk-vector-engine-client-1.2.0.0/demo/client/exampleSchema.cpp000066400000000000000000000132721520573221700251760ustar00rootroot00000000000000// This file has been modified by KylinSoft Co., Ltd. // Modifications copyright (C) 2024-2026 KylinSoft Co., Ltd. // // Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include "Database.h" using namespace std; using namespace VectorDB; shared_ptr client = Database::Create(); const string collection_name = "TEST"; const string field_id_name = "identity"; const string field_age_name = "age"; const string field_face_name = "face"; const uint32_t dimension = 4; const int64_t row_count = 10; vector insert_ids; vector insert_ages; vector> insert_vectors; default_random_engine ran(time(nullptr)); void CheckStatus(string&& prefix, const Status& status) { if (!status.IsOk()) { cout << prefix << " " << status.Message() << endl; exit(1); } } void Connect() { ConnectParam connect_param; auto status = client->Connect(connect_param); CheckStatus("Failed to connect milvus server:", status); cout << "Connect to milvus server success." << endl; } void DropCollection() { auto status = client->DropCollection(collection_name); CheckStatus("Failed to drop TEST collection:", status); cout << "Drop TEST collection success." << endl; } void CreateCollection() { // 通过 CollectionSchema 设置数据库列：identity（主键）、age、face（向量） CollectionSchema collection_schema(collection_name); collection_schema.AddField({field_id_name, DataType::INT64, "user id", true, false}); collection_schema.AddField({field_age_name, DataType::INT8, "user age"}); collection_schema.AddField( FieldSchema(field_face_name, DataType::FLOAT_VECTOR, "face signature").WithDimension(dimension)); // 给向量列建个索引 IndexDesc index_desc(field_face_name, "face_index", 0, IndexType::FLAT, MetricType::L2); auto status = client->CreateCollection(collection_schema, index_desc); CheckStatus("Failed to create collection:", status); cout << "Successfully create collection." << endl; } void Insert() { uniform_int_distribution int_gen(1, 100); uniform_real_distribution float_gen(0.0, 1.0); for (auto i = 0; i < row_count; ++i) { insert_ids.push_back(i); insert_ages.push_back(int_gen(ran)); vector vector(dimension); for (auto j = 0; j < dimension; ++j) { vector[j] = float_gen(ran); } insert_vectors.emplace_back(vector); } vector fields_data{make_shared(field_id_name, insert_ids), make_shared(field_age_name, insert_ages), make_shared(field_face_name, insert_vectors)}; DmlResults dml_results; auto status = client->Insert(collection_name, fields_data, dml_results); CheckStatus("Failed to insert:", status); cout << "Successfully insert " << dml_results.IdArray().IntIDArray().size() << " rows." << endl; } void Search() { SearchArguments arguments; arguments.SetCollectionName(collection_name); arguments.SetTopK(10); arguments.SetMetricType(MetricType::L2); // 要与索引一致 arguments.AddOutputField(field_age_name); arguments.SetExpression(field_age_name + " > 40"); // set to strong guarantee so that the search is executed after the inserted data is persisted arguments.SetGuaranteeTimestamp(GuaranteeStrongTs()); uniform_int_distribution int64_gen(0, row_count - 1); int64_t q_number = int64_gen(ran); vector q_vector = insert_vectors[q_number]; arguments.AddTargetVector(field_face_name, move(q_vector)); cout << "Searching the No." << q_number << " entity..." << endl; SearchResults search_results{}; auto status = client->Search(arguments, search_results, 100); CheckStatus("Failed to search:", status); cout << "Search success." << endl; for (auto& result : search_results.Results()) { auto& ids = result.Ids().IntIDArray(); auto& distances = result.Scores(); if (ids.size() != distances.size()) { cout << "Illegal result!" << endl; continue; } auto age_field = result.OutputField(field_age_name); Int8FieldDataPtr age_field_ptr = static_pointer_cast(age_field); auto& age_data = age_field_ptr->Data(); for (size_t i = 0; i < ids.size(); ++i) { cout << "ID: " << ids[i] << "\tDistance: " << distances[i] << "\tAge: " << static_cast(age_data[i]) << endl; // validate the age value if (insert_ages[ids[i]] != age_data[i]) { cout << "ERROR! The returned value doesn't match the inserted value" << endl; } } } } int main() { cout << "Example start..." << endl; Connect(); // drop the collection if it exists DropCollection(); CreateCollection(); Insert(); Search(); return 0; } libkysdk-vector-engine-client-1.2.0.0/demo/client/milvus.cpp000066400000000000000000000237201520573221700237400ustar00rootroot00000000000000// This file has been modified by KylinSoft Co., Ltd. // Modifications copyright (C) 2024-2026 KylinSoft Co., Ltd. // // Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include "MilvusClientImpl.h" using namespace std; using namespace VectorDB; void CheckStatus(string&& prefix, const Status& status) { if (!status.IsOk()) { cout << prefix << " " << status.Message() << endl; exit(1); } } std::shared_ptr client = std::make_shared(); const string collection_name = "TEST"; const string field_id_name = "identity"; const string field_age_name = "age"; const string field_face_name = "face"; const string field_metadata_name = "metadata"; const uint32_t dimension = 4; const int64_t row_count = 5; vector insert_ids; vector insert_ages; vector> insert_vectors; vector insert_jsons; default_random_engine ran(time(nullptr)); void Connect() { ConnectParam connect_param; auto status = client->Connect(connect_param); CheckStatus("Failed to connect milvus server:", status); cout << "Connect to milvus server success." << endl; } void DropCollection() { auto status = client->DropCollection(collection_name); CheckStatus("Failed to drop TEST collection:", status); cout << "Drop TEST collection success." << endl; } void HasCollection() { bool exist; auto status = client->HasCollection(collection_name, exist); CheckStatus("Failed to has TEST collection:", status); cout << "Have TEST collection or not: " << exist << endl; } void CreateCollection() { CollectionSchema collection_schema(collection_name, "++++----", true); collection_schema.AddField(FieldSchema(field_id_name, DataType::VARCHAR, "userid", true, false).WithMaxLength(50)); collection_schema.AddField(FieldSchema(field_age_name, DataType::VARCHAR, "userage").WithMaxLength(1024)); collection_schema.AddField( FieldSchema(field_face_name, DataType::FLOAT_VECTOR, "facesignature").WithDimension(dimension)); auto status = client->CreateCollection(collection_schema); CheckStatus("Failed to create TEST collection:", status); cout << "Create TEST collection success." << endl; } void ShowCollection() { CollectionDesc desc; auto status = client->DescribeCollection(collection_name, desc); CheckStatus("Failed to describe TEST collection:", status); cout << "--------------------" << endl; cout << "Have TEST collection desc: " << desc.Schema().ShardsNum() << endl << desc.Schema().Name() << endl << desc.Schema().Description() << endl; for (const auto& item : desc.Schema().AnnsFieldNames()) { cout << "AnnsFieldNames: " << item << endl; } cout << "--------------------" << endl; for (const auto& item : desc.Schema().Fields()) { cout << "Name: " << item.Name() << endl << "Description: " << item.Description() << endl << "FieldDataType: " << static_cast(item.FieldDataType()) << endl << "AutoID: " << item.AutoID() << endl << "Dimension: " << item.Dimension() << endl << "IsPrimaryKey: " << item.IsPrimaryKey() << endl; cout << "++++++++++++++++++++++++" << endl; for (const auto& tmp : item.TypeParams()) { cout << "key: " << tmp.first << ", val: " << tmp.second << endl; } } } void CreateIndex() { IndexDesc index_desc(field_face_name, "", 0); auto status = client->CreateIndex(collection_name, index_desc); CheckStatus("Failed to create index:", status); cout << "Create index success." << endl; } void LoadCollection() { auto status = client->LoadCollection(collection_name); CheckStatus("Failed to load collection:", status); cout << "Load collection success." << endl; } void ReleaseCollection() { auto status = client->ReleaseCollection(collection_name); CheckStatus("Failed to release collection:", status); cout << "Release collection success." << endl; } std::string generateUUID() { uuid_t uuid; char uuidStr[37]; uuid_generate_random(uuid); uuid_unparse(uuid, uuidStr); return uuidStr; } void Insert() { uniform_real_distribution float_gen(0.0, 1.0); string seed = "abcdefghijklmn"; for (auto i = 0; i < row_count; ++i) { insert_ids.push_back(generateUUID()); insert_ages.push_back(std::to_string(i)); vector vector(dimension); for (auto j = 0; j < dimension; ++j) { vector[j] = float_gen(ran); } insert_vectors.emplace_back(vector); nlohmann::json value = nlohmann::json::object(); string key = std::to_string(seed[i]); value[key] = 100 * (i + 1); insert_jsons.push_back(value); } std::vector fields_data{make_shared(field_id_name, insert_ids), make_shared(field_age_name, insert_ages), make_shared(field_face_name, insert_vectors), make_shared(field_metadata_name, insert_jsons)}; DmlResults dml_results; auto status = client->Insert(collection_name, fields_data, dml_results); CheckStatus("Failed to insert:", status); std::cout << "Successfully insert " << dml_results.IdArray().IntIDArray().size() << " rows." << std::endl; } void Search() { SearchArguments arguments(collection_name, 10, MetricType::COSINE); arguments.AddOutputField(field_age_name); arguments.AddOutputField(field_metadata_name); arguments.SetExpression(field_age_name + " > '1'"); // set to strong guarantee so that the search is executed after the inserted data is persisted arguments.SetGuaranteeTimestamp(GuaranteeStrongTs()); uniform_int_distribution int64_gen(0, row_count - 1); int64_t q_number = int64_gen(ran); vector q_vector = insert_vectors[q_number]; arguments.AddTargetVector(field_face_name, move(q_vector)); cout << "Searching the No." << q_number << " entity..." << endl; SearchResults search_results{}; auto status = client->Search(arguments, search_results, 100); CheckStatus("Failed to search:", status); cout << "Search success. find results: " << search_results.Results().size() << endl; for (auto& result : search_results.Results()) { auto& ids = result.Ids().StrIDArray(); auto& distances = result.Scores(); if (ids.size() != distances.size()) { cout << "Illegal result! " << ids.size() << " vs " << distances.size() << endl; continue; } auto age_field = result.OutputField(field_age_name); VarCharFieldDataPtr age_field_ptr = static_pointer_cast(age_field); auto& age_data = age_field_ptr->Data(); auto metadata_field = result.OutputField(field_metadata_name); JsonFieldDataPtr metadata_field_ptr = static_pointer_cast(metadata_field); auto& metadata = metadata_field_ptr->Data(); for (size_t i = 0; i < ids.size(); ++i) { cout << "ID: " << ids[i] << "\tDistance: " << distances[i] << "\tAge: " << age_data[i] << "\tMetadata: " << metadata[i] << endl; // validate the age value // if (insert_ages[ids[i]] != age_data[i]) { // cout << "ERROR! The returned value doesn't match the inserted value" << endl; // } } } } void Query() { QueryArguments queryArguments; queryArguments.SetCollectionName(collection_name); queryArguments.AddOutputField(field_age_name); queryArguments.SetExpression(field_age_name + " == '2'"); QueryResults queryResults; auto status = client->Query(queryArguments, queryResults, 100); CheckStatus("Failed to query:", status); cout << "Query success." << endl; FieldDataPtr idField = queryResults.GetFieldByName(field_id_name); shared_ptr idFieldPtr = static_pointer_cast(idField); for (const auto& item : idFieldPtr->Data()) { cout << idFieldPtr->Name() << ": " << item << endl; } FieldDataPtr ageField = queryResults.GetFieldByName(field_age_name); shared_ptr ageFieldPtr = static_pointer_cast(ageField); for (const auto& item : ageFieldPtr->Data()) { cout << ageFieldPtr->Name() << ": " << item << endl; } } int main() { cout << "Example start..." << endl; Connect(); // ShowCollection(); // drop the collection if it exists DropCollection(); // create a collection CreateCollection(); // CreateCollectionTmp(); ShowCollection(); // create index CreateIndex(); // insert some rows Insert(); // tell server prepare to load collection // LoadCollection(); // query by id Query(); // do search Search(); // SearchTmp(); // release collection after search ReleaseCollection(); // drop collection HasCollection(); DropCollection(); HasCollection(); // Failed to has TEST collection: : collection not found return 0; } libkysdk-vector-engine-client-1.2.0.0/demo/client/test.cpp000066400000000000000000000430001520573221700233710ustar00rootroot00000000000000// This file has been modified by KylinSoft Co., Ltd. // Modifications copyright (C) 2024-2026 KylinSoft Co., Ltd. // // Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "MilvusClientImpl.h" using namespace std; using namespace VectorDB; void CheckStatus(string&& prefix, const Status& status) { if (!status.IsOk()) { cout << prefix << " code: " << static_cast(status.Code()) << ", msg: " << status.Message() << endl; exit(1); } } std::shared_ptr client = std::make_shared(); const string collectionName = "TEST"; void Connect() { ConnectParam connect_param; auto status = client->Connect(connect_param); CheckStatus("Failed to connect milvus server:", status); cout << "Connect to milvus server success." << endl; } void Disconnect() { auto status = client->Disconnect(); CheckStatus("Failed to disconnect milvus server:", status); cout << "Disconnect to milvus server success." << endl; } void HasCollection() { bool exist; auto status = client->HasCollection(collectionName, exist); CheckStatus("Failed to has TEST collection:", status); cout << "Have TEST collection or not: " << exist << endl; } void Search() { SearchArguments arguments(collectionName, 10, MetricType::COSINE); arguments.SetGuaranteeTimestamp(GuaranteeStrongTs()); vector q_vector = { 0.000737071, -0.00703124, -0.0687869, 0.00444741, -0.0156804, -0.0729711, 0.0305887, 0.00973448, -0.0167483, 0.00164538, 0.0175224, 0.0134256, 0.00465957, -0.00993188, 0.0272341, -0.013162, 0.0416493, 0.0208695, -0.00945018, -0.0259077, 0.0296987, -0.0289422, 0.00764727, -0.0011585, -0.026936, -0.00790239, -0.0170204, -0.0194117, 0.0546304, 0.0265546, -0.0134801, -0.0576278, -0.00675568, -0.0337923, -0.0077309, -0.0452068, 0.0442868, -0.0130356, -0.0309886, 0.0119397, -0.00838874, -0.0010398, 0.0307289, -0.00911335, -0.0154665, -0.0417664, 0.0029093, -0.033919, -0.0190002, 0.0309206, -0.0139156, 0.00193856, 0.0280912, -0.0257606, 0.000950229, 0.024366, 0.00141795, 0.0327373, -0.0233746, 0.0804398, -0.0488382, 0.0228038, 0.00152665, -0.0077095, -0.012997, 0.0548115, 0.0130035, 0.0152437, -0.00705497, -0.0502254, -0.0269348, -0.0262926, 0.0651965, -0.0233256, -0.0263396, 0.045461, 0.0146741, -0.0159948, -0.0314924, -0.0122158, 0.0210211, 0.00894256, -0.00224452, 0.044283, 0.0242233, 0.0434728, -0.0175554, 0.0453315, 0.0276561, 0.0219293, -0.0233246, 0.0392724, -0.0179472, -0.0176247, -0.0716201, 0.00431582, -0.0108365, 0.00144094, 0.0260212, 0.0393035, -0.0400234, 0.011914, -0.00722578, -0.0291466, 0.0138115, 0.0104696, 0.0629848, 0.0266984, -0.0183656, 0.0187744, 0.0305592, 0.00959264, -0.0238058, -0.0186224, -0.0188598, -0.00724981, -0.0177582, -0.0157105, 0.0424064, -0.0117101, 0.00417663, -0.0233787, 0.0387323, -0.0257589, -0.0148557, -0.0520384, -0.036409, 0.0525422, 0.00130365, 0.0176998, -0.036935, 0.0400307, 0.0105048, 0.0164076, -0.0270931, -0.00485668, -0.020784, -0.00423116, -0.00334573, -0.0284704, -0.00413042, 0.0329782, 0.0264966, -0.0129031, 0.00617863, -0.0124285, -0.00446578, 0.0336394, -0.0187514, -0.0437405, -0.00592617, -0.0214081, 0.0333823, 0.0660683, -0.0194553, -0.0163364, -0.00684285, 0.0122657, -0.00808447, 0.0452127, 0.00755446, 0.0130502, 0.0101895, 0.0319258, 0.0160604, 0.0347243, 0.0295944, 0.0198427, -0.0400967, -0.0323674, -0.0707755, 0.0224757, -0.026102, -0.0100395, -0.0279084, 0.0303817, 0.0325454, 0.00864542, -0.00315931, -0.0195768, -0.0672579, -0.0356704, 0.00548743, 0.0149105, -0.0150758, -0.0119797, 0.0253497, -0.0183421, 0.0101822, 0.0445602, -0.00187065, -0.0105286, 0.0595269, 0.00753041, 0.0294662, 0.0154191, 0.00342504, -0.0258275, -0.0100786, -0.0203375, 0.00656276, 0.0351068, 0.0368628, 0.0386165, -0.0433855, 0.0150098, -0.0247799, -0.00469767, 0.0365177, -0.0523491, 0.00961586, -0.0183003, 0.00767648, -0.00488904, -0.0632531, -0.0250812, -0.00265041, 0.0028836, -0.0289958, -0.0362877, -0.0277824, 0.0107343, -0.026794, -0.0390103, 0.0348336, -0.0217657, 0.035916, 0.0259624, -0.0290615, 0.0167392, -0.0315001, 0.0210939, -0.0383325, -0.0217428, 0.0808947, -0.0243922, -0.00965134, -0.0169099, 0.012243, 0.0314801, 0.00312122, -0.00118377, -0.0136591, -0.0288059, -0.0556801, 0.0672965, 0.0175989, 0.0392005, -0.0407835, -0.0211708, 0.0369975, 0.00971775, -0.0666349, -0.0172246, 0.0241715, 0.0118146, 0.022647, 0.0227035, 0.0274502, -0.00382736, 0.0401722, 0.0337264, 0.00440881, 0.0397022, 0.0315891, -0.00413152, 0.0383169, -0.0120784, -0.0132535, 0.0445635, 0.0186953, -0.0266128, 0.0218828, -0.0114471, -0.0105944, 0.0111167, -0.025543, -0.0401533, 0.053706, 0.00103854, 0.0310566, 0.0646674, -0.037987, 0.0221328, -0.0301203, -0.0100773, -0.0258841, 0.00405111, 0.00588779, 0.00540682, -0.0328082, 0.00935923, 0.0692297, -0.00430604, -0.013022, 0.0237084, -0.0191372, -0.169614, 0.00946786, -0.0290456, 0.03006, -0.0231383, -0.0221911, 0.0418589, -0.00872659, -0.0637244, 0.0595296, 0.0113504, -0.0414809, -0.00596364, -0.0445393, -0.028522, -0.0218763, 0.00947455, -0.022034, 0.00552063, -0.0018455, -0.0515506, -0.0312021, 0.00729205, 0.0631468, 0.0165869, -0.0217401, 0.0100923, -0.0193803, 0.00151983, -0.033093, 0.0300238, -0.0124093, -0.0115479, 0.0523196, 0.0414753, -0.0162739, -0.0108975, 0.015194, 0.0322996, -0.0332378, 0.0247728, -0.0142335, -0.00411593, -0.00955629, 0.0283441, -0.00960746, 0.00214705, 0.000871523, 0.0242415, -0.0466347, 0.0180538, -0.0327906, 0.0126322, -0.0288312, -0.0336423, 0.0303741, -0.000767618, 0.05448, 0.0317741, -0.0171904, -0.0118548, -0.00583959, 0.0101494, 0.000872982, -0.0259114, 0.0482797, -0.0154265, -0.0125479, 0.0512704, 0.0161909, 0.0213436, 0.0408943, 0.0207816, -0.0193291, -0.0156448, 0.0327966, -0.0701249, -0.0268254, 0.00863666, -0.090234, 0.00184574, 0.0206352, 0.0115321, 0.0133756, -0.0183933, -0.0254897, -0.0216294, -0.00306281, 0.0315391, 0.213124, 0.0139518, -0.0107392, 0.0127432, 0.00894664, -0.0404861, 0.0211778, 0.0224338, -0.00210254, -0.00262023, 0.0113449, -0.0368389, 0.0192827, 0.00631438, 0.0111312, -0.0215562, -0.0175798, -0.0477379, 0.0497207, -0.000960736, 0.0150321, -0.00255412, 0.0284571, -0.00407944, -0.0419172, -0.0297105, -0.0292142, 0.0591464, -0.0376923, 0.0595587, -0.0642642, 0.0278166, -0.0130716, -0.0215922, -0.0446888, -0.00566484, 0.00668127, 0.00393049, 0.0206803, -0.0253389, 0.000941204, -0.0517655, 0.00558603, 0.00371393, -0.00449182, -0.0317975, 0.00674521, 0.0128277, -0.0117638, -0.0310506, -0.0195899, -0.02521, -0.0206919, -0.0399907, -0.0422651, -0.0142572, 0.00221016, -0.0150362, 0.0365398, 0.0163474, 0.00563635, -0.00264986, -0.0250133, 0.0350089, -0.0074462, -0.0311973, 0.0568557, -0.0245532, 0.00475806, -0.0372368, -0.0192534, -0.0315555, -0.00341438, -0.0171649, 0.0338445, 0.0405635, 0.0402743, 0.0637271, -0.0220447, 0.028543, -0.00587127, -0.014979, -0.0477299, 0.0211768, -0.000810466, 0.0101526, -0.0235368, 0.0481131, -0.0227215, -0.0260698, 0.00904307, 0.00515313, -0.0259969, 0.045991, 0.0469449, 0.00412111, -0.0518184, -0.0175164, -0.00975386, 0.0230083, -0.0502967, -0.0259783, -0.0238916, 0.00981072, -0.00781667, -0.0218546, -0.0158292, 0.0211703, 0.0421166, 0.0371443, -0.0329466, 0.0313589, -0.00767982, -0.0170293, 0.00803524, 0.0226892, -0.01608, 0.0468633, 0.0332584, 0.000285792, 0.0018456, -0.00879449, 0.00250982, 0.0202133, -0.0326322, 0.00425828, -0.014614, -0.00156962, -0.05753, 0.0362316, 0.0366633, -0.0102016, 0.0390997, -0.0210486, 0.00149776, -0.0320307, -0.011767, 0.0165315, -0.00726082, 0.0553159, 0.00921641, -0.0259929, -0.00864047, 0.00916218, 0.00567056, 0.0194474, 0.0405782, -0.0139647, -0.000789303, -0.00104748, -0.00912631, 0.0203381, 0.0403739, 0.00857938, -0.0160319, -0.0216435, -0.0105129, -0.0221919, 0.0236753, -0.0191803, -0.0220402, -0.0210657, 0.0228546, 0.0620411, 0.0561018, 0.0581801, -0.00734326, 0.0567014, 0.0229041, 0.0250433, 0.0447944, -0.0596579, -0.0349341, -0.0120473, 0.0109773, -0.0305183, -0.0765964, -0.0139982, 0.0149851, -0.0716118, 0.0527549, 0.025701, -0.0587188, -0.00250012, 0.0332447, 0.00404549, 0.0220679, -0.0153524, 0.00417326, 0.00126621, 0.0104854, 0.101697, -0.0164366, -0.0154994, 0.0204943, 0.00575161, -0.00315832, 0.0508082, 0.00973417, -0.00753689, 0.0622526, -0.0159157, 0.0135134, -0.0546397, 0.000268232, -0.00665681, -0.0111477, 0.0373854, -0.00323578, -0.0110765, 0.019704, 0.0220342, 0.000810616, -0.00537691, 0.0481257, 0.0241495, -0.00319456, 0.0442338, -0.0024984, -0.0391649, -0.0488256, 0.00533169, 0.0405003, -0.0755476, 0.00349945, -0.0126928, 0.0421029, -0.0228044, -0.00412698, -0.0036083, 0.0111979, 0.00649091, -0.0354002, -0.0705443, 0.0298126, -0.0232923, 0.0427496, 0.0457241, -0.00639633, -0.0247828, 0.0275638, 0.00942403, 0.0119432, -0.0244699, -0.0312989, 0.0597251, -0.0457604, 0.00492505, -0.0152289, 0.0500598, -0.0277095, 0.0143591, -0.0105227, -0.0325966, -0.0549545, -0.0164955, -0.0015838, 0.00348122, 0.0230482, -0.0346011, -0.0334628, 0.0118848, -0.0271867, 0.0542674, 0.0235505, 0.0288599, 0.0269265, 0.0167247, -0.041325, -0.0216503, 0.0122968, 0.0294474, -0.0501997, -0.0572272, 0.0562263, -0.0187261, -0.0474296, 0.014207, 0.00474904, 0.000277325, -0.0458693, -0.0120678, -0.00817774, 0.0160517, -0.0168719, 0.0271727, 0.0576711, -0.0327699, -0.0253424, -0.0734031, -0.0228364, 0.0233814, -0.0206857, 0.0481137, 0.00339575, 0.0182982, -0.000998005, -0.0120538, -0.0107723, 0.0475012, -0.00655715, 0.0318983, -0.0352251, -0.0487634, -0.03132, 0.0201727, -0.0441198, -0.0302566, -0.00251687, 0.0675132, -0.00660633, 0.0686423, -0.00816387, 0.0561215, 0.0195526, -0.00654678, 0.0283941, -0.0187459, -0.0122728, 0.0617469, 0.0258106, -0.0259754, 0.015741, -0.0124498, -0.0147934, -0.020426, -0.0171559, -0.00258501, -0.0263929, 0.0232164, -0.0207999, 0.00154625, 0.0124838, -0.060576, 0.0230516, -0.00154301, 0.0385999, -0.0474625, 0.028866, -0.0107686, 0.0227763, 0.00744687, -0.00643714, -0.0268916, 0.0226412, -0.026022, -0.0116714, -0.00798632, 0.00184316, 0.0185764, 0.0202585, 0.0190125, -0.0203902, -0.0404893, 0.0234556, 0.027506, -0.0673117, 0.0156817, 0.0122572, 0.0369492, -0.0436122, 0.0503773, 0.0156537, 0.0358643, 0.0141498, 0.0236252, 0.0143617, 0.023573, -0.00566898, -0.0324884, -0.027847, -0.0330234, -0.00147555, 0.013319, -0.0165894, 0.0718744, -0.00899064, -0.0157908, 0.0187293, 0.00650605, -0.0161394, -0.0661586, -0.075162, -0.0237897, 0.00996381, -0.00879105, 0.00329379, -0.00627782, 0.0725136, 0.0136857, -0.012595, -0.0301607, -0.0223246, 0.00272542, -0.129459, -0.00120674, -0.00788633, -0.00248823, -0.054703, -0.0134077, -0.00439809, 0.028805, 0.035529, 0.00275262, 0.000465464, 0.03285, 0.00384697, -0.00167302, 0.0462788, 0.0109006, 0.003756, -0.0214751, -0.00632735, -0.00290379, -0.0171169, -0.0496038, -0.0171499, 0.0566737, 0.000838922, -0.016549, -0.0214577, 0.0446921, -0.0424571, -0.0230891, 0.080515, 0.00269584, 0.0220298, 0.0473478, -0.0371835, -0.0523223, -0.0376095, -0.0192613, -0.0211296, -0.025246, -0.0225586, 0.032596, -0.0309126, -0.0160629, -0.0170765, 0.00040525, -0.0177945, -0.00329523, -0.0366711, 0.0475188, -0.0179997, -0.0244928, -0.00305196, -0.0200479, -0.00532758, -0.0496489, -0.0255739, 0.00630239, -0.0373233, 0.0172539, -0.0365232, 0.00815317, -0.0062507, -0.00145635, -0.0204522, 5.39723e-05, -0.0434492, -0.0356377, 0.000166819, 3.05265e-05, -0.000463334, -0.0291884, 0.0281475, -0.0311321, 0.00494268, 0.00847886, -0.0198267, 0.028767, 0.0255071, -0.0333282, 0.0263492, -0.0183827, -0.0428118, 0.0365444, 0.074468, -0.00118519, -0.00283448, -0.00892564, -0.0144867, -0.0241277, -0.0485611, -0.0290484, -0.0235856, -0.0290035, 0.00213124, -0.0194133, -0.0262752, -0.0234961, -0.0470328, -0.00620682, -0.012032, -0.00895563, -0.0229751, -0.054197, 0.0112159, -0.0316797, 0.055893, 0.0014967, 0.00872314, 0.0127881, -0.0560814, -0.00684751, -0.0018328, -0.00342872, -0.0347354, -0.034278, -0.00750538, -0.0259488, -0.0470177, -0.00504735, -0.0154864, 0.0355048, -0.0198981, -0.00793508, -0.0232894, 0.061664, 0.0120122, -0.0142472, 0.00109427, -0.0422949, 0.0558726, -0.00951176, -0.0535344, -0.0280421, 0.0332092, -0.0488579, 0.0328109, 0.0382458, 0.00525318, -0.0334026, 0.00547176, 0.0263166, -0.00325793, -0.01843, 0.032601, 0.00114303, -0.0106617, 0.0143329, -0.0471301, -0.0044237, 0.0391758, -0.0114654, -0.0169774, 0.0080129, 0.00719909, 0.0227233, 0.0141896, 0.0155002, -0.0215335, 0.0130194, 0.0443224, 0.0171386, -0.012077, -0.0289951, 0.0374457, -0.0104803, 0.002781, -0.0189992, 0.0183027, -0.00195638, 0.043169, 0.040325, -0.0319646, 0.0231146, 0.0477195, -0.0307641, 0.0416924, -0.0433486, 0.0644367, 0.0529532, -0.0025552, -0.00788088, 0.0198571, 0.0192064, 0.0117896, 0.0326591, 0.0302894, 0.0058136, 0.0216437, 0.010357, 0.0174423, 0.0362274, 0.0370588, -0.0399478, 0.0254278, 0.0328187, 0.0262189, 0.0391451, -0.00245065, 0.0392109, 0.00391924, -0.062361, -0.0596566, 0.0195372, -0.0285228, -0.0204674, -0.0386167, -0.0319991, -0.0456168, 0.0250034, 0.00794402, -0.00218377, 0.0054613, 0.0194416, -0.0274201, 0.0105512, -0.0144757, 0.0309936, 0.0454972, 0.0264897, -0.0545277, 0.0304817, -0.0165276, -0.0354638, -0.014114, -0.0136601, -0.0090112, 0.00531306, 0.0139964, -0.0622878, -0.0160822, -0.0433711, -0.0430655, 0.0502466, 0.0204747, -0.0684087, 0.0221671, 0.0357779, 0.00247462, -0.017308, 0.0206, 0.0225623, 0.0405955, 0.00208024}; arguments.AddTargetVector("vector", move(q_vector)); SearchResults search_results{}; auto status = client->Search(arguments, search_results, 0); CheckStatus("Failed to search:", status); cout << "Search success. find results: " << search_results.Results().size() << endl; for (auto& result : search_results.Results()) { auto& ids = result.Ids().StrIDArray(); auto& distances = result.Scores(); if (ids.size() != distances.size()) { cout << "Illegal result! " << ids.size() << " vs " << distances.size() << endl; continue; } // auto age_field = result.OutputField(field_age_name); // VarCharFieldDataPtr age_field_ptr = static_pointer_cast(age_field); // auto& age_data = age_field_ptr->Data(); for (size_t i = 0; i < ids.size(); ++i) { cout << "ID: " << ids[i] << "\tDistance: " << distances[i] << endl; } } } int main() { cout << "Example start..." << endl; Connect(); HasCollection(); // Search(); Disconnect(); return 0; } libkysdk-vector-engine-client-1.2.0.0/include/000077500000000000000000000000001520573221700211325ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/000077500000000000000000000000001520573221700262765ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/Database.h000066400000000000000000000145411520573221700301600ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "types/CollectionSchema.h" #include "types/ConnectParam.h" #include "types/Constants.h" #include "types/DmlResults.h" #include "types/FieldData.h" #include "types/IndexDesc.h" #include "types/QueryArguments.h" #include "types/QueryResults.h" #include "types/SearchArguments.h" #include "types/SearchResults.h" #include "types/Status.h" namespace VectorDB { class Database { public: /** * now only support create a MilvusClient instance. * * @return std::shared_ptr */ static std::shared_ptr Create(); /** * Connect to VectorDB server. * * @param [in] connect_param server address port and authorization * @return Status operation successfully or not */ virtual Status Connect(const ConnectParam& connect_param) = 0; /** * Connect to VectorDB server. Simple way * * @return Status operation successfully or not */ Status Connect() { return Connect(ConnectParam()); } /** * Break connections between client and server. * * @return Status operation successfully or not */ virtual Status Disconnect() = 0; /** * Create a collection with schema. And create an index on a field. * Not load collection data into CPU memory of query node. * * @param [in] schema schema of the collection * @param [in] index_desc the index descriptions and parameters * @return Status operation successfully or not */ virtual Status CreateCollection(const CollectionSchema& schema, const IndexDesc& index_desc) = 0; /** * Create a collection fast way. Create id and vector two fields. And create vector index. * Not load collection data into CPU memory of query node. * * @param [in] collection_name name of the collection * @param [in] dim vector dimension * @param [in] auto_id open auto_id or not, default is open. * @param [in] enable_dynamic_field 如果开启，插入数据时必须要有个字段的类型是动态字段 * @return Status operation successfully or not */ virtual Status CreateCollection(const std::string& collection_name, int dim, bool auto_id = true, bool enable_dynamic_field = true) = 0; /** * Check existence of a collection. * * @param [in] collection_name name of the collection * @param [out] has true: collection exists, false: collection doesn't exist * @return Status operation successfully or not */ virtual Status HasCollection(const std::string& collection_name, bool& has) = 0; /** * Drop a collection, with all its partitions, index and segments. * * @param [in] collection_name name of the collection * @return Status operation successfully or not */ virtual Status DropCollection(const std::string& collection_name) = 0; /** * Insert entities into a collection. * * @param [in] collection_name name of the collection * @param [in] fields insert data * @param [out] results insert results * @return Status operation successfully or not */ virtual Status Insert(const std::string& collection_name, const std::vector& fields, DmlResults& results) = 0; /** * Delete entities by filtering condition. * * @param [in] collection_name name of the collection * @param [in] expression the expression to filter out entities, currently only support primary key as filtering. * For example: "id in [1, 2, 3]" * @param [out] results insert results * @return Status operation successfully or not */ virtual Status Delete(const std::string& collection_name, const std::string& expression, DmlResults& results) = 0; /** * Upsert entities into a collection. * * @param [in] collection_name name of the collection * @param [in] fields upsert data * @param [out] results upsert results * @return Status operation successfully or not */ virtual Status Upsert(const std::string& collection_name, const std::vector& fields, DmlResults& results) = 0; /** * Search a collection based on the given parameters and return results. * * @param [in] arguments search arguments * @param [out] results search results * @param [in] timeout search timeout in milliseconds * @return Status operation successfully or not */ virtual Status Search(const SearchArguments& arguments, SearchResults& results, int timeout = 0) = 0; /** * Query with a set of criteria, and results in a list of records that match the query exactly. * * @param [in] arguments query arguments * @param [out] results query results * @param [in] timeout search timeout in milliseconds * @return Status operation successfully or not */ virtual Status Query(const QueryArguments& arguments, QueryResults& results, int timeout = 0) = 0; /** * Load collection data into CPU memory of query node. * now no need to call * * @param [in] collection_name name of the collection * @return Status operation successfully or not */ virtual Status LoadCollection(const std::string& collection_name) = 0; /** * Release collection data from query node. * now no need to call * * @param [in] collection_name name of the collection * @return Status operation successfully or not */ virtual Status ReleaseCollection(const std::string& collection_name) = 0; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/000077500000000000000000000000001520573221700274425ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/CollectionDesc.h000066400000000000000000000042361520573221700325120ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "CollectionSchema.h" namespace VectorDB { /** * @brief Collection schema and runtime information returned by MilvusClient::DescribeCollection(). */ class CollectionDesc { public: /** * @brief Collection schema. */ const CollectionSchema& Schema() const; /** * @brief Set collection schema. */ void SetSchema(const CollectionSchema& schema); /** * @brief Set collection schema. */ void SetSchema(CollectionSchema&& schema); /** * @brief Collection id. */ int64_t ID() const; /** * @brief Set collection id. */ void SetID(int64_t id); /** * @brief Collection alias. */ const std::vector& Alias() const; /** * @brief Set collection alias. */ void SetAlias(const std::vector& alias); /** * @brief Set collection alias. */ void SetAlias(std::vector&& alias); /** * @brief Timestamp when the collection created. */ uint64_t CreatedTime() const; /** * @brief Set timestamp when the collection created. */ void SetCreatedTime(uint64_t ts); private: CollectionSchema schema_; int64_t collection_id_; std::vector alias_; uint64_t created_utc_timestamp_ = 0; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/CollectionSchema.h000066400000000000000000000052731520573221700330360ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include #include "FieldSchema.h" namespace VectorDB { /** * @brief Collection schema for MilvusClient::CreateCollection(). */ class CollectionSchema { public: /** * @brief Constructor */ CollectionSchema(); /** * @brief Constructor */ explicit CollectionSchema(std::string name, std::string desc = "", bool enable_dynamic_field = false, int32_t shard_num = 0); /** * @brief Collection name, cannot be empty. */ const std::string& Name() const; /** * @brief Set collection name, cannot be empty. */ void SetName(std::string name); /** * @brief Collection description, can be empty. */ const std::string& Description() const; /** * @brief Set collection description, can be empty. */ void SetDescription(std::string description); bool EnableDynamicField() const; void SetEnableDynamicField(bool enable_dynamic_field); /** * @brief Collection shards number, the number must be larger than zero, default value is 2. */ int32_t ShardsNum() const; /** * @brief Set shards number, the number must be larger than zero, default value is 2. */ void SetShardsNum(int32_t num); /** * @brief Fields schema array. */ const std::vector& Fields() const; /** * @brief Add a field schema. */ bool AddField(const FieldSchema& field_schema); /** * @brief Add a field schema. */ bool AddField(FieldSchema&& field_schema); /** * @brief Return Anns field names. */ std::unordered_set AnnsFieldNames() const; private: std::string name_; std::string description_; int32_t shard_num_ = 0; bool enable_dynamic_field_; std::vector fields_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/ConnectParam.h000066400000000000000000000073571520573221700322010ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include namespace VectorDB { /** * @brief Connection parameters. Used by Database::Connect() */ class ConnectParam { public: /** * @brief Constructor */ ConnectParam(); /** * @brief Constructor */ ConnectParam(std::string host) : host_(host) {} /** * @brief Constructor */ ConnectParam(std::string host, uint16_t port); /** * @brief Constructor */ ConnectParam(std::string host, uint16_t port, std::string username, std::string password); /** * @brief IP of the milvus proxy. */ const std::string& Host() const; /** * @brief Port of the milvus proxy. */ uint16_t Port() const; /** * @brief Uri for connecting to the milvus. */ std::string Uri() const; /** * @brief Authorizations header value for connecting to the milvus. * Authorizations() = base64('username:password') */ const std::string& Authorizations() const; /** * @brief SetAuthorizations set username and password for connecting to the milvus. */ void SetAuthorizations(std::string username, std::string password); /** * @brief Connect timeout in milliseconds. * */ uint32_t ConnectTimeout() const; /** * @brief Set connect timeout in milliseconds. * */ void SetConnectTimeout(uint32_t timeout); /** * @brief With ssl */ ConnectParam& WithTls(); /** * @brief Enable ssl */ void EnableTls(); /** * @brief With ssl */ ConnectParam& WithTls(const std::string& server_name, const std::string& ca_cert); /** * @brief Enable ssl */ void EnableTls(const std::string& server_name, const std::string& ca_cert); /** * @brief With ssl and provides certificates */ ConnectParam& WithTls(const std::string& server_name, const std::string& cert, const std::string& key, const std::string& ca_cert); /** * @brief Enable ssl and provides certificates */ void EnableTls(const std::string& server_name, const std::string& cert, const std::string& key, const std::string& ca_cert); /** * @brief Disable ssl */ void DisableTls(); /** * @brief TlsEnabled */ bool TlsEnabled() const; /** * @brief ServerName tls hostname */ const std::string& ServerName() const; /** * @brief Cert tls cert file */ const std::string& Cert() const; /** * @brief Key tls key file */ const std::string& Key() const; /** * @brief CaCert tls ca cert file */ const std::string& CaCert() const; private: std::string host_; uint16_t port_ = 0; uint32_t connect_timeout_ = 5000; bool tls_{false}; std::string server_name_; std::string cert_; std::string key_; std::string ca_cert_; std::string authorizations_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/Constants.h000066400000000000000000000043031520573221700315670ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include namespace VectorDB { // 动态扩展字段名 #define DYNAMIC_FIELD_NAME "metadata" #define DEFAULT_ID_FIELD_NAME "id" #define DEFAULT_VECTOR_FIELD_NAME "vector" /** * @brief Global definition for row count label */ inline std::string KeyRowCount() { return "row_count"; } /** * @brief Global definition for index type label */ inline std::string KeyIndexType() { return "index_type"; } /** * @brief Global definition for metric type label */ inline std::string KeyMetricType() { return "metric_type"; } /** * @brief Global definition for metric type label */ inline std::string KeyParams() { return "params"; } /** * @brief Global definition for vector dimension label */ inline std::string FieldDim() { return "dim"; } /** * @brief Max length field name for varchar field */ inline std::string FieldMaxLength() { return "max_length"; } /** * @brief Global definition for strong guarantee timestamp */ inline uint64_t GuaranteeStrongTs() { return 0; } /** * @brief Global definition for eventually guarantee timestamp */ inline uint64_t GuaranteeEventuallyTs() { return 1; } /** * @brief The logical bits in hybrid timestamp */ constexpr size_t inline HybridTsLogicalBits() { return 18; } /** * @brief The logical bits mask for hybrid timestamp */ constexpr uint64_t inline HybridTsLogicalBitsMask() { return (1 << HybridTsLogicalBits()) - 1; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/DataType.h000066400000000000000000000023621520573221700313310ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once namespace VectorDB { /** * @brief Data type of field */ enum class DataType { UNKNOWN = 0, BOOL = 1, INT8 = 2, INT16 = 3, INT32 = 4, INT64 = 5, FLOAT = 10, DOUBLE = 11, // STRING not available STRING = 20, VARCHAR = 21, ARRAY = 22, JSON = 23, BINARY_VECTOR = 100, FLOAT_VECTOR = 101, FLOAT16_VECTOR = 102, BFLOAT16_VECTOR = 103, SPARSE_FLOAT_VECTOR = 104, }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/DmlResults.h000066400000000000000000000031351520573221700317130ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "IDArray.h" namespace VectorDB { /** * @brief Result returned by MilvusClient::Insert() and MilvusClient::Delete() */ class DmlResults { public: /** * @brief The id array for entities which are inserted or deleted. */ const IDArray& IdArray() const; /** * @brief Set the id array. */ void SetIdArray(const IDArray& id_array); /** * @brief Set the id array. */ void SetIdArray(IDArray&& id_array); /** * @brief The operation timestamp marked by server side. */ uint64_t Timestamp() const; /** * @brief Set operation timestamp. */ void SetTimestamp(uint64_t timestamp); private: IDArray id_array_{std::vector{}}; uint64_t timestamp_{0}; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/FieldData.h000066400000000000000000000160641520573221700314370ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "DataType.h" #include "Status.h" namespace VectorDB { class Field { public: /** * @brief Get field name */ const std::string& Name() const; /** * @brief Get field data type */ DataType Type() const; /** * @brief Total number of field elements */ virtual size_t Count() const = 0; protected: Field(std::string name, DataType data_type); private: std::string name_; DataType data_type_; }; using FieldDataPtr = std::shared_ptr; /** * @brief Template class represents column-based data of a field. Available inheritance classes: \n * BoolFieldData for boolean scalar field \n * Int8FieldData for 8-bits integer scalar field \n * Int16FieldData for 16-bits integer scalar field \n * Int32FieldData for 32-bits integer scalar field \n * Int64FieldData for 64-bits integer scalar field \n * FloatFieldData for float scalar field \n * DoubleFieldData for double scalar field \n * VarCharFieldData for string scalar field (not supported in 2.0, reserved) \n * BinaryVecFieldData for float vector scalar field \n * FloatVecFieldData for binary vector scalar field \n */ template class FieldData : public Field { public: /** * @brief Field element type */ using ElementT = T; /** * @brief Constructor */ FieldData(); /** * @brief Constructor */ explicit FieldData(std::string name); /** * @brief Constructor */ FieldData(std::string name, const std::vector& data); /** * @brief Constructor */ FieldData(std::string name, std::vector&& data); /** * @brief Add element to field data */ virtual StatusCode Add(const T& element); /** * @brief Add element to field data */ virtual StatusCode Add(T&& element); /** * @brief Total number of field elements */ size_t Count() const final; /** * @brief Field elements array */ virtual const std::vector& Data() const; /** * @brief Field elements array */ virtual std::vector& Data(); private: friend class BinaryVecFieldData; std::vector data_; }; class BinaryVecFieldData : public FieldData { public: /** * @brief Constructor */ BinaryVecFieldData(); /** * @brief Constructor */ explicit BinaryVecFieldData(std::string name); /** * @brief Constructor */ BinaryVecFieldData(std::string name, const std::vector& data); /** * @brief Constructor */ BinaryVecFieldData(std::string name, std::vector&& data); /** * @brief Constructor */ BinaryVecFieldData(std::string name, const std::vector>& data); /** * @brief Field elements array */ const std::vector& Data() const override; /** * @brief Field elements array */ std::vector& Data() override; /** * @brief Data export as uint8_t's vector */ std::vector> DataAsUnsignedChars() const; /** * @brief Add element to field data */ StatusCode Add(const std::string& element) override; /** * @brief Add element to field data */ StatusCode Add(std::string&& element) override; /** * @brief Add element to field data */ StatusCode Add(const std::vector& element); /** * @brief Create binary vector strings from uint8_t vectors */ static std::vector CreateBinaryStrings(const std::vector>& data); /** * @brief Create binary vector string from uint8_t vector */ static std::string CreateBinaryString(const std::vector& data); }; /** * @brief To test two FieldData are equal */ template bool operator==(const FieldData& lhs, const FieldData& rhs) { return lhs.Name() == rhs.Name() && lhs.Count() == rhs.Count() && lhs.Data() == rhs.Data(); } /** * @brief To test two FieldData are equal */ template bool operator==(const FieldData& lhs, const Field& rhs) { return lhs == dynamic_cast&>(rhs); } using BoolFieldData = FieldData; using Int8FieldData = FieldData; using Int16FieldData = FieldData; using Int32FieldData = FieldData; using Int64FieldData = FieldData; using FloatFieldData = FieldData; using DoubleFieldData = FieldData; using VarCharFieldData = FieldData; using JsonFieldData = FieldData; using FloatVecFieldData = FieldData, DataType::FLOAT_VECTOR>; using BoolFieldDataPtr = std::shared_ptr; using Int8FieldDataPtr = std::shared_ptr; using Int16FieldDataPtr = std::shared_ptr; using Int32FieldDataPtr = std::shared_ptr; using Int64FieldDataPtr = std::shared_ptr; using FloatFieldDataPtr = std::shared_ptr; using DoubleFieldDataPtr = std::shared_ptr; using VarCharFieldDataPtr = std::shared_ptr; using JsonFieldDataPtr = std::shared_ptr; using BinaryVecFieldDataPtr = std::shared_ptr; using FloatVecFieldDataPtr = std::shared_ptr; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData; extern template class FieldData, DataType::FLOAT_VECTOR>; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/FieldSchema.h000066400000000000000000000101141520573221700317540ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "DataType.h" namespace VectorDB { /** * @brief Field schema used by CollectionSchema */ class FieldSchema { public: FieldSchema(); /** * @brief Constructor */ FieldSchema(std::string name, DataType data_type, std::string description = "", bool is_primary_key = false, bool auto_id = false); /** * @brief Name of this field, cannot be empty. */ const std::string& Name() const; /** * @brief Set name of the field. */ void SetName(std::string name); /** * @brief Description of this field, can be empty. */ const std::string& Description() const; /** * @brief Set description of the field. */ void SetDescription(std::string description); /** * @brief Field data type. */ DataType FieldDataType() const; /** * @brief Set field data type. */ void SetDataType(DataType dt); /** * @brief The field is primary key or not. * * Each collection only has one primary key. * Currently only int64 type field can be primary key . */ bool IsPrimaryKey() const; /** * @brief Set field to be primary key. */ void SetPrimaryKey(bool is_primary_key); /** * @brief Field item's id is auto-generated or not. * * If ths flag is true, server will generate id when data is inserted. * Else the client must provide id for each entity when insert data. */ bool AutoID() const; /** * @brief Set field item's id to be auto-generated. */ void SetAutoID(bool auto_id); bool IsDynamic() const; void SetDynamic(bool is_dynamic); /** * @brief Extra key-value pair setting for this field * * Currently vector field need to input "dim":"x" to specify dimension. */ const std::map& TypeParams() const; /** * @brief Set extra key-value pair setting for this field * * Currently vector field need to input "dim":"x" to specify dimension. */ void SetTypeParams(const std::map& params); /** * @brief Set extra key-value pair setting for this field * * Currently vector field need to input "dim":"x" to specify dimension. */ void SetTypeParams(std::map&& params); /** * @brief Get dimension for a vector field */ uint32_t Dimension() const; /** * @brief Quickly set dimension for a vector field */ bool SetDimension(uint32_t dimension); /** * @brief Quickly set dimension for a vector field */ FieldSchema& WithDimension(uint32_t dimension); /** * @brief Get max length for a varchar field */ uint32_t MaxLength() const; /** * @brief Quickly set max length for a varchar field */ void SetMaxLength(uint32_t length); /** * @brief Quickly set max length for a varchar field */ FieldSchema& WithMaxLength(uint32_t length); private: std::string name_; std::string description_; DataType data_type_{DataType::FLOAT}; bool is_primary_key_ = false; bool auto_id_ = false; bool is_dynamic_ = false; std::map type_params_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/IDArray.h000066400000000000000000000035521520573221700311130ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include namespace VectorDB { /** * @brief ID array, each ID could be int64 type or string type. \n * Note: v2.0 only support int64 type id. */ class IDArray { public: /** * @brief Constructor */ explicit IDArray(const std::vector& id_array); /** * @brief Constructor */ explicit IDArray(std::vector&& id_array); /** * @brief Constructor */ explicit IDArray(const std::vector& id_array); /** * @brief Constructor */ explicit IDArray(std::vector&& id_array); /** * @brief Indicate this is an integer id array */ bool IsIntegerID() const; /** * @brief Return integer id array */ const std::vector& IntIDArray() const; /** * @brief Return string id array */ const std::vector& StrIDArray() const; private: bool is_int_array_{true}; std::vector int_id_array_; std::vector str_id_array_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/IndexDesc.h000066400000000000000000000066621520573221700314730ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include #include "IndexType.h" #include "MetricType.h" #include "Status.h" namespace VectorDB { /** * @brief Index description. Used by MilvusClient::CreateIndex() and MilvusClient::DescribeIndex(). */ class IndexDesc { public: /** * @brief Construct a new Index Desc object */ IndexDesc(); /** * @brief Construct a new Index Desc object * * @param field_name field name which the index belong to * @param index_name index name * @param index_type index type see IndexType * @param metric_type metric type see MetricType * @param index_id internal id of the index reserved for future feature */ IndexDesc(std::string field_name, std::string index_name, int64_t index_id = 0, IndexType index_type = IndexType::FLAT, MetricType metric_type = MetricType::COSINE); /** * @brief Filed name which the index belong to. */ const std::string& FieldName() const; /** * @brief Set field name which the index belong to. */ Status SetFieldName(std::string field_name); /** * @brief Index name. Index name cannot be empty. */ const std::string& IndexName() const; /** * @brief Set index name. */ Status SetIndexName(std::string index_name); /** * @brief Index ID. */ int64_t IndexId() const; /** * @brief Set index id. */ Status SetIndexId(int64_t index_id); /** * @brief Metric type. */ VectorDB::MetricType MetricType() const; /** * @brief Set metric type. */ Status SetMetricType(VectorDB::MetricType metric_type); /** * @brief Index type. */ VectorDB::IndexType IndexType() const; /** * @brief Set index type. */ Status SetIndexType(VectorDB::IndexType index_type); /** * @brief Parameters of the index. */ const std::string ExtraParams() const; /** * @brief Add param, current all param is a numberic value */ Status AddExtraParam(std::string key, int64_t value); /** * @brief Construct a new Index Desc:: From Json object * @param json Json string for parse */ Status ExtraParamsFromJson(std::string json); /** * @brief Validate for create index * */ Status Validate() const; private: std::string field_name_; std::string index_name_; int64_t index_id_{0}; VectorDB::MetricType metric_type_{MetricType::INVALID}; VectorDB::IndexType index_type_{IndexType::INVALID}; std::unordered_map extra_params_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/IndexType.h000066400000000000000000000021151520573221700315230ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once namespace VectorDB { /** * @brief Supported index types. */ enum class IndexType { INVALID = 0, FLAT, IVF_FLAT, IVF_SQ8, IVF_PQ, HNSW, IVF_HNSW, RHNSW_FLAT, RHNSW_SQ, RHNSW_PQ, ANNOY, BIN_FLAT = 1001, BIN_IVF_FLAT, }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/MetricType.h000066400000000000000000000020771520573221700317060ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once namespace VectorDB { /** * @brief Supported metric types. */ enum class MetricType { INVALID, L2, IP, COSINE, // The following values are for binary vectors HAMMING, JACCARD, TANIMOTO, SUBSTRUCTURE, SUPERSTRUCTURE }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/QueryArguments.h000066400000000000000000000067341520573221700326200ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "Status.h" namespace VectorDB { /** * @brief Arguments for MilvusClient::Query(). */ class QueryArguments { public: /** * @brief Get name of the target collection */ const std::string& CollectionName() const; /** * @brief Set name of this collection, cannot be empty */ Status SetCollectionName(std::string collection_name); /** * @brief Get partition names */ const std::set& PartitionNames() const; /** * @brief Specify partition name to control query scope, the name cannot be empty */ Status AddPartitionName(std::string partition_name); /** * @brief Get output field names */ const std::set& OutputFields() const; /** * @brief Specify output field names to return field data, the name cannot be empty */ Status AddOutputField(std::string field_name); /** * @brief Get filter expression */ const std::string& Expression() const; /** * @brief Set filter expression, the expression cannot be empty */ Status SetExpression(std::string expression); /** * @brief Get travel timestamp. */ uint64_t TravelTimestamp() const; /** * @brief @brief Specify an absolute timestamp in a query to get results based on a data view at a specified point * in time. \n Default value is 0, server executes query on a full data view. */ Status SetTravelTimestamp(uint64_t timestamp); /** * @brief Get guarantee timestamp. */ uint64_t GuaranteeTimestamp() const; /** * @brief Instructs server to see insert/delete operations performed before a provided timestamp. \n * If no such timestamp is specified, the server will wait for the latest operation to finish and query. \n * * Note: The timestamp is not an absolute timestamp, it is a hybrid value combined by UTC time and internal flags. * \n We call it TSO, for more information please refer to: \n * https://github.com/milvus-io/milvus/blob/master/docs/design_docs/milvus_hybrid_ts_en.md. * You can get a TSO from insert/delete results. Use an operation's TSO to set this parameter, \n the server will * execute query after this operation is finished. \n * * Default value is 1, server executes search immediately. */ Status SetGuaranteeTimestamp(uint64_t timestamp); private: std::string collection_name_; std::set partition_names_; std::set output_field_names_; std::string filter_expression_; uint64_t travel_timestamp_{0}; uint64_t guarantee_timestamp_{0}; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/QueryResults.h000066400000000000000000000030151520573221700323010ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "FieldData.h" namespace VectorDB { /** * @brief Results returned by MilvusClient::Query(). */ class QueryResults { public: QueryResults(); /** * @brief Constructor */ explicit QueryResults(const std::vector& output_fields); /** * @brief Constructor */ explicit QueryResults(std::vector&& output_fields); /** * @brief Get output field data by name. */ FieldDataPtr GetFieldByName(const std::string& name); /** * @brief Get all output fields data. */ const std::vector& OutputFields() const; private: std::vector output_fields_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/SearchArguments.h000066400000000000000000000146031520573221700327120ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include "Constants.h" #include "FieldData.h" #include "MetricType.h" #include "Status.h" namespace VectorDB { /** * @brief Arguments for MilvusClient::Search(). */ class SearchArguments { public: SearchArguments() = default; SearchArguments(std::string name, int64_t topK = 10, MetricType type = MetricType::COSINE) : collection_name_(std::move(name)), topk_(topK), metric_type_(type) {} /** * @brief Get name of the target collection */ const std::string& CollectionName() const; /** * @brief Set name of this collection, cannot be empty */ Status SetCollectionName(std::string collection_name); /** * @brief Get partition names */ const std::set& PartitionNames() const; /** * @brief Specify partition name to control search scope, the name cannot be empty */ Status AddPartitionName(std::string partition_name); /** * @brief Get output field names */ const std::set& OutputFields() const; /** * @brief Specify output field names to return field data, the name cannot be empty */ Status AddOutputField(std::string field_name); /** * @brief Get filter expression */ const std::string& Expression() const; /** * @brief Set filter expression */ Status SetExpression(std::string expression); /** * @brief Get target vectors */ FieldDataPtr TargetVectors() const; /** * @brief Add a binary vector to search */ Status AddTargetVector(std::string field_name, const std::string& vector); /** * @brief Add a binary vector to search with uint8_t vectors */ Status AddTargetVector(std::string field_name, const std::vector& vector); /** * @brief Add a binary vector to search */ Status AddTargetVector(std::string field_name, std::string&& vector); /** * @brief Add a float vector to search */ Status AddTargetVector(std::string field_name, const FloatVecFieldData::ElementT& vector); /** * @brief Add a float vector to search */ Status AddTargetVector(std::string field_name, FloatVecFieldData::ElementT&& vector); /** * @brief Get travel timestamp. */ uint64_t TravelTimestamp() const; /** * @brief Specify an absolute timestamp in a search to get results based on a data view at a specified point in * time. \n * * Default value is 0, server executes search on a full data view. */ Status SetTravelTimestamp(uint64_t timestamp); /** * @brief Get guarantee timestamp. */ uint64_t GuaranteeTimestamp() const; /** * @brief Instructs server to see insert/delete operations performed before a provided timestamp. \n * If no such timestamp is specified, the server will wait for the latest operation to finish and search. \n * * Note: The timestamp is not an absolute timestamp, it is a hybrid value combined by UTC time and internal flags. * \n We call it TSO, for more information please refer to: \n * https://github.com/milvus-io/milvus/blob/master/docs/design_docs/milvus_hybrid_ts_en.md. * You can get a TSO from insert/delete results. Use an operation's TSO to set this parameter, * the server will execute search after this operation is finished. \n * * Default value is 1, server executes search immediately. */ Status SetGuaranteeTimestamp(uint64_t timestamp); /** * @brief Specify search limit, AKA topk */ Status SetTopK(int64_t topk); /** * @brief Get Top K */ int64_t TopK() const; /** * @brief Get nprobe */ int64_t Nprobe() const; /** * @brief Set nprobe */ Status SetNprobe(int64_t nlist); /** * @brief Specifies the decimal place of the returned results. */ Status SetRoundDecimal(int round_decimal); /** * @brief Get the decimal place of the returned results */ int RoundDecimal() const; /** * @brief Specifies the metric type */ Status SetMetricType(MetricType metric_type); /** * @brief Get the metric type */ VectorDB::MetricType MetricType() const; /** * @brief Add extra param */ Status AddExtraParam(std::string key, int64_t value); /** * @brief Get extra param */ std::string ExtraParams() const; /** * @brief Validate for search arguments * */ Status Validate() const; /** * @brief Get range radius * @return */ float Radius() const; /** * @brief Get range filter * @return */ float RangeFilter() const; /** * @brief Set range radius * @param from range radius from * @param to range radius to */ Status SetRange(float from, float to); /** * @brief Get if do range search * @return */ bool RangeSearch() const; private: std::string collection_name_; std::set partition_names_; std::set output_field_names_; std::string filter_expression_; BinaryVecFieldDataPtr binary_vectors_; FloatVecFieldDataPtr float_vectors_; std::set output_fields_; std::unordered_map extra_params_; uint64_t travel_timestamp_{0}; uint64_t guarantee_timestamp_{GuaranteeEventuallyTs()}; int64_t topk_{1}; int round_decimal_{-1}; float radius_; float range_filter_; bool range_search_{false}; VectorDB::MetricType metric_type_{MetricType::COSINE}; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/SearchResults.h000066400000000000000000000040761520573221700324110ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "FieldData.h" #include "IDArray.h" namespace VectorDB { /** * @brief Topk results for one target vector of MilvusClient::Search() */ struct SingleResult { /** * @brief Constructor */ SingleResult(IDArray&& ids, std::vector&& scores, std::vector&& output_fields); /** * @brief Distances/scores array of one target vector */ const std::vector& Scores() const; /** * @brief Topk id array of one target vector */ const IDArray& Ids() const; /** * @brief Output fields data */ const std::vector& OutputFields() const; /** * @brief Get an output field by name */ FieldDataPtr OutputField(const std::string& name) const; private: IDArray ids_; std::vector scores_; std::vector output_fields_; }; /** * @brief Results returned by MilvusClient::Search(). */ class SearchResults { public: SearchResults(); /** * @brief Constructor */ explicit SearchResults(std::vector&& results); /** * @brief Get search results. */ std::vector& Results(); private: std::vector nq_results_{}; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/Status.h000066400000000000000000000037051520573221700311030ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include /** * @brief Milvus SDK namespace */ namespace VectorDB { /** * @brief Status code for SDK interface return */ enum class StatusCode { OK = 0, // system error section UNKNOWN_ERROR = 1, NOT_SUPPORTED, NOT_CONNECTED, // function error section INVALID_AGUMENT = 1000, RPC_FAILED, SERVER_FAILED, TIMEOUT, // database error section DATABASE_INIT_FAILED = 1100, // validation error DIMENSION_NOT_EQUAL = 2000, VECTOR_IS_EMPTY, JSON_PARSE_ERROR, }; /** * @brief Status code and message returned by SDK interface. */ class Status { public: /** * @brief Constructor of Status */ Status(StatusCode code, std::string msg); Status(); /** * @brief A success status */ static Status OK(); /** * @brief Indicate the status is ok */ bool IsOk() const; /** * @brief Return the status code */ StatusCode Code() const; /** * @brief Return the error message */ const std::string& Message() const; private: StatusCode code_{StatusCode::OK}; std::string msg_{"OK"}; }; // Status } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/include/kysdk-vector-engine-client/types/Utils.h000066400000000000000000000020541520573221700307140ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include "IndexType.h" #include "MetricType.h" namespace std { std::string to_string(VectorDB::IndexType); std::string to_string(VectorDB::MetricType); } // namespace std namespace VectorDB { std::string Base64Encode(const std::string& val); } libkysdk-vector-engine-client-1.2.0.0/src/000077500000000000000000000000001520573221700202765ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/src/Database.cpp000066400000000000000000000022141520573221700225050ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Database.h" #ifndef ENABLE_SW_ARCH #include "MilvusClientImpl.h" using namespace VectorDB; std::shared_ptr VectorDB::Database::Create() { return std::make_shared(); } #else #include "SwClientImpl.h" using namespace VectorDB; std::shared_ptr VectorDB::Database::Create() { return std::make_shared(); } #endif libkysdk-vector-engine-client-1.2.0.0/src/impl/000077500000000000000000000000001520573221700212375ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/src/impl/MilvusClientImpl.cpp000066400000000000000000000505121520573221700252060ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "MilvusClientImpl.h" #include #include "TypeUtils.h" #include "common.pb.h" #include "milvus.pb.h" #include "schema.pb.h" #include "types/Utils.h" namespace VectorDB { using namespace milvus; MilvusClientImpl::~MilvusClientImpl() { Disconnect(); } Status MilvusClientImpl::Connect(const ConnectParam& param) { if (connection_ != nullptr) { connection_->Disconnect(); } connection_ = std::make_shared(); int retryTime = 0; Status status = connection_->Connect(param); while (status.Code() == StatusCode::NOT_CONNECTED && retryTime <= RETRY_TIME) { if (retryTime == RETRY_TIME) { std::cerr << "ERROR connect milvus retry failed, code: " << static_cast(status.Code()) << ", msg: " << status.Message() << std::endl; return status; } std::cout << "WARN connect milvus failed, code: " << static_cast(status.Code()) << ", msg: " << status.Message() << ", retry: " << retryTime << std::endl; sleep(1); retryTime++; status = connection_->Connect(param); } return status; } Status MilvusClientImpl::Disconnect() { if (connection_ != nullptr) { return connection_->Disconnect(); } return Status::OK(); } Status MilvusClientImpl::CreateCollection(const CollectionSchema& schema, const IndexDesc& index_desc) { // create collection auto status = CreateCollection(schema); if (!status.IsOk()) { return status; } // create index status = CreateIndex(schema.Name(), index_desc); if (!status.IsOk()) { return status; } // load collection，目前先不加载，看查询性能，后续再修改 // LoadCollection(schema.Name()); return Status(); } Status MilvusClientImpl::CreateCollection(const std::string& collection_name, int dim, bool auto_id, bool enable_dynamic_field) { // create collection schema CollectionSchema schema(collection_name, "default schema", enable_dynamic_field); schema.AddField({DEFAULT_ID_FIELD_NAME, DataType::INT64, "user id", true, auto_id}); schema.AddField(FieldSchema(DEFAULT_VECTOR_FIELD_NAME, DataType::FLOAT_VECTOR, "vector").WithDimension(dim)); auto status = CreateCollection(schema); if (!status.IsOk()) { return status; } // create index IndexDesc desc(DEFAULT_VECTOR_FIELD_NAME, "vector_index", 0); status = CreateIndex(schema.Name(), desc); if (!status.IsOk()) { return status; } // load collection，目前先不加载，看查询性能，后续再修改 // LoadCollection(schema.Name()); return Status(); } Status MilvusClientImpl::CreateCollection(const CollectionSchema& schema) { auto pre = [&schema]() { proto::milvus::CreateCollectionRequest rpc_request; rpc_request.set_collection_name(schema.Name()); rpc_request.set_shards_num(schema.ShardsNum()); rpc_request.set_consistency_level(proto::common::ConsistencyLevel::Bounded); proto::schema::CollectionSchema rpc_collection; rpc_collection.set_name(schema.Name()); rpc_collection.set_description(schema.Description()); rpc_collection.set_enable_dynamic_field(schema.EnableDynamicField()); for (auto& field : schema.Fields()) { proto::schema::FieldSchema* rpc_field = rpc_collection.add_fields(); rpc_field->set_name(field.Name()); rpc_field->set_description(field.Description()); rpc_field->set_data_type(static_cast(field.FieldDataType())); rpc_field->set_is_primary_key(field.IsPrimaryKey()); rpc_field->set_autoid(field.AutoID()); rpc_field->set_is_dynamic(field.IsDynamic()); rpc_field->set_is_clustering_key(false); rpc_field->set_is_partition_key(false); if (!field.TypeParams().empty()) { proto::common::KeyValuePair* kv = rpc_field->add_type_params(); for (auto& pair : field.TypeParams()) { kv->set_key(pair.first); kv->set_value(pair.second); } } } // std::cout << "rpc collection: " << rpc_collection.DebugString() << std::endl; std::string binary; rpc_collection.SerializeToString(&binary); rpc_request.set_schema(binary); return rpc_request; }; return apiHandler( pre, &MilvusConnection::CreateCollection); } Status MilvusClientImpl::HasCollection(const std::string& collection_name, bool& has) { auto pre = [&collection_name]() { proto::milvus::HasCollectionRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_time_stamp(0); return rpc_request; }; auto post = [&has](const proto::milvus::BoolResponse& response) { has = response.value(); }; return apiHandler( pre, &MilvusConnection::HasCollection, post); } Status MilvusClientImpl::DropCollection(const std::string& collection_name) { auto pre = [&collection_name]() { proto::milvus::DropCollectionRequest rpc_request; rpc_request.set_collection_name(collection_name); return rpc_request; }; return apiHandler(pre, &MilvusConnection::DropCollection); } Status MilvusClientImpl::LoadCollection(const std::string& collection_name) { auto pre = [&collection_name]() { proto::milvus::LoadCollectionRequest rpc_request; rpc_request.set_collection_name(collection_name); // rpc_request.set_replica_number(replica_number); return rpc_request; }; // todo 要不要查询加载进度 return apiHandler(pre, &MilvusConnection::LoadCollection); } Status MilvusClientImpl::ReleaseCollection(const std::string& collection_name) { auto pre = [&collection_name]() { proto::milvus::ReleaseCollectionRequest rpc_request; rpc_request.set_collection_name(collection_name); return rpc_request; }; return apiHandler( pre, &MilvusConnection::ReleaseCollection); } Status MilvusClientImpl::DescribeCollection(const std::string& collection_name, CollectionDesc& collection_desc) { auto pre = [&collection_name]() { proto::milvus::DescribeCollectionRequest rpc_request; rpc_request.set_collection_name(collection_name); return rpc_request; }; auto post = [&collection_desc](const proto::milvus::DescribeCollectionResponse& response) { CollectionSchema schema; ConvertCollectionSchema(response.schema(), schema); schema.SetShardsNum(response.shards_num()); collection_desc.SetSchema(std::move(schema)); collection_desc.SetID(response.collectionid()); std::vector aliases; aliases.reserve(response.aliases_size()); aliases.insert(aliases.end(), response.aliases().begin(), response.aliases().end()); collection_desc.SetAlias(std::move(aliases)); collection_desc.SetCreatedTime(response.created_timestamp()); }; return apiHandler( pre, &MilvusConnection::DescribeCollection, post); } Status MilvusClientImpl::CreateIndex(const std::string& collection_name, const IndexDesc& index_desc) { auto validate = [&index_desc] { return index_desc.Validate(); }; auto pre = [&collection_name, &index_desc]() { proto::milvus::CreateIndexRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_field_name(index_desc.FieldName()); auto kv_pair = rpc_request.add_extra_params(); kv_pair->set_key(KeyIndexType()); kv_pair->set_value(std::to_string(index_desc.IndexType())); kv_pair = rpc_request.add_extra_params(); kv_pair->set_key(KeyMetricType()); kv_pair->set_value(std::to_string(index_desc.MetricType())); kv_pair = rpc_request.add_extra_params(); kv_pair->set_key(KeyParams()); kv_pair->set_value(index_desc.ExtraParams()); return rpc_request; }; // todo 要不要查询加载进度 return apiHandler(validate, pre, &MilvusConnection::CreateIndex); } Status MilvusClientImpl::DropIndex(const std::string& collection_name, const std::string& field_name) { auto pre = [&collection_name, &field_name]() { proto::milvus::DropIndexRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_field_name(field_name); return rpc_request; }; return apiHandler(pre, &MilvusConnection::DropIndex); } Status MilvusClientImpl::Insert(const std::string& collection_name, const std::vector& fields, DmlResults& results) { // TODO(matrixji): add common validations check for fields // TODO(matrixji): add scheme based validations check for fields // auto validate = nullptr; auto pre = [&collection_name, &fields] { proto::milvus::InsertRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_num_rows((*fields.front()).Count()); for (const auto& field : fields) { proto::schema::FieldData data = CreateProtoFieldData(*field); rpc_request.add_fields_data()->CopyFrom(data); } // std::cout << rpc_request.DebugString() << std::endl; return rpc_request; }; auto post = [&results](const proto::milvus::MutationResult& response) { auto id_array = CreateIDArray(response.ids()); results.SetIdArray(std::move(id_array)); results.SetTimestamp(response.timestamp()); }; return apiHandler(pre, &MilvusConnection::Insert, post); } Status MilvusClientImpl::Delete(const std::string& collection_name, const std::string& expression, DmlResults& results) { auto pre = [&collection_name, &expression]() { proto::milvus::DeleteRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_expr(expression); return rpc_request; }; auto post = [&results](const proto::milvus::MutationResult& response) { auto id_array = CreateIDArray(response.ids()); results.SetIdArray(std::move(id_array)); results.SetTimestamp(response.timestamp()); }; return apiHandler(pre, &MilvusConnection::Delete, post); } Status MilvusClientImpl::Upsert(const std::string& collection_name, const std::vector& fields, DmlResults& results) { auto pre = [&collection_name, &fields]() { proto::milvus::UpsertRequest rpc_request; rpc_request.set_collection_name(collection_name); rpc_request.set_num_rows((*fields.front()).Count()); for (const auto& field : fields) { proto::schema::FieldData data = CreateProtoFieldData(*field); rpc_request.add_fields_data()->CopyFrom(data); } return rpc_request; }; auto post = [&results](const proto::milvus::MutationResult& response) { auto id_array = CreateIDArray(response.ids()); results.SetIdArray(std::move(id_array)); results.SetTimestamp(response.timestamp()); }; return apiHandler(pre, &MilvusConnection::Upsert, post); } Status MilvusClientImpl::Search(const SearchArguments& arguments, SearchResults& results, int timeout) { std::string anns_field; auto validate = [this, &arguments, &anns_field]() { CollectionDesc collection_desc; auto status = DescribeCollection(arguments.CollectionName(), collection_desc); if (status.IsOk()) { // check anns fields auto& field_name = arguments.TargetVectors()->Name(); auto anns_fileds = collection_desc.Schema().AnnsFieldNames(); if (anns_fileds.find(field_name) != anns_fileds.end()) { anns_field = field_name; } else { return Status{StatusCode::INVALID_AGUMENT, std::string(field_name + " is not a valid anns field")}; } // basic check for extra params status = arguments.Validate(); } return status; }; auto pre = [&arguments, &anns_field]() { proto::milvus::SearchRequest rpc_request; rpc_request.set_collection_name(arguments.CollectionName()); rpc_request.set_dsl_type(proto::common::DslType::BoolExprV1); if (!arguments.Expression().empty()) { rpc_request.set_dsl(arguments.Expression()); } for (const auto& partition_name : arguments.PartitionNames()) { rpc_request.add_partition_names(partition_name); } for (const auto& output_field : arguments.OutputFields()) { rpc_request.add_output_fields(output_field); } // placeholders proto::common::PlaceholderGroup placeholder_group; auto& placeholder_value = *placeholder_group.add_placeholders(); placeholder_value.set_tag("$0"); auto target = arguments.TargetVectors(); if (target->Type() == DataType::BINARY_VECTOR) { // bins placeholder_value.set_type(proto::common::PlaceholderType::BinaryVector); auto& bins_vec = dynamic_cast(*target); for (const auto& bins : bins_vec.Data()) { std::string placeholder_data(reinterpret_cast(bins.data()), bins.size()); placeholder_value.add_values(std::move(placeholder_data)); } rpc_request.set_nq(bins_vec.Data().size()); } else { // floats placeholder_value.set_type(proto::common::PlaceholderType::FloatVector); auto& floats_vec = dynamic_cast(*target); for (const auto& floats : floats_vec.Data()) { std::string placeholder_data(reinterpret_cast(floats.data()), floats.size() * sizeof(float)); placeholder_value.add_values(std::move(placeholder_data)); } rpc_request.set_nq(floats_vec.Data().size()); } rpc_request.set_placeholder_group(std::move(placeholder_group.SerializeAsString())); auto kv_pair = rpc_request.add_search_params(); kv_pair->set_key("anns_field"); kv_pair->set_value(anns_field); kv_pair = rpc_request.add_search_params(); kv_pair->set_key("topk"); kv_pair->set_value(std::to_string(arguments.TopK())); kv_pair = rpc_request.add_search_params(); kv_pair->set_key(KeyMetricType()); kv_pair->set_value(std::to_string(arguments.MetricType())); kv_pair = rpc_request.add_search_params(); kv_pair->set_key("round_decimal"); kv_pair->set_value(std::to_string(arguments.RoundDecimal())); kv_pair = rpc_request.add_search_params(); kv_pair->set_key(KeyParams()); // merge extra params with range search auto json = nlohmann::json::parse(arguments.ExtraParams()); if (arguments.RangeSearch()) { json["range_filter"] = arguments.RangeFilter(); json["radius"] = arguments.Radius(); } kv_pair->set_value(json.dump()); rpc_request.set_travel_timestamp(arguments.TravelTimestamp()); rpc_request.set_guarantee_timestamp(arguments.GuaranteeTimestamp()); return rpc_request; }; auto post = [&results](const proto::milvus::SearchResults& response) { auto& result_data = response.results(); const auto& ids = result_data.ids(); const auto& scores = result_data.scores(); const auto& fields_data = result_data.fields_data(); auto num_of_queries = result_data.num_queries(); std::vector topks{}; topks.reserve(result_data.topks_size()); for (int i = 0; i < result_data.topks_size(); ++i) { topks.emplace_back(result_data.topks(i)); } std::vector single_results; single_results.reserve(num_of_queries); int offset{0}; for (int i = 0; i < num_of_queries; ++i) { std::vector item_scores; std::vector item_field_data; auto item_topk = topks[i]; item_scores.reserve(item_topk); for (int j = 0; j < item_topk; ++j) { item_scores.emplace_back(scores.Get(offset + j)); } item_field_data.reserve(fields_data.size()); for (const auto& field_data : fields_data) { item_field_data.emplace_back(std::move(CreateMilvusFieldData(field_data, offset, item_topk))); } single_results.emplace_back(std::move(CreateIDArray(ids, offset, item_topk)), std::move(item_scores), std::move(item_field_data)); offset += item_topk; } results = std::move(SearchResults(std::move(single_results))); }; return apiHandler( validate, pre, &MilvusConnection::Search, nullptr, post, GrpcOpts{timeout}); } Status MilvusClientImpl::Query(const QueryArguments& arguments, QueryResults& results, int timeout) { auto pre = [&arguments]() { proto::milvus::QueryRequest rpc_request; rpc_request.set_collection_name(arguments.CollectionName()); for (const auto& partition_name : arguments.PartitionNames()) { rpc_request.add_partition_names(partition_name); } rpc_request.set_expr(arguments.Expression()); for (const auto& field : arguments.OutputFields()) { rpc_request.add_output_fields(field); } rpc_request.set_travel_timestamp(arguments.TravelTimestamp()); rpc_request.set_guarantee_timestamp(arguments.GuaranteeTimestamp()); return rpc_request; }; auto post = [&results](const proto::milvus::QueryResults& response) { std::vector return_fields{}; return_fields.reserve(response.fields_data_size()); for (const auto& field_data : response.fields_data()) { return_fields.emplace_back(std::move(CreateMilvusFieldData(field_data))); } results = std::move(QueryResults(std::move(return_fields))); }; return apiHandler(pre, &MilvusConnection::Query, post, GrpcOpts{timeout}); } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/impl/MilvusClientImpl.h000066400000000000000000000151661520573221700246610ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include "Database.h" #include "MilvusConnection.h" #include "types/CollectionDesc.h" /** * @brief namespace milvus */ namespace VectorDB { class MilvusClientImpl : public Database { public: MilvusClientImpl() = default; virtual ~MilvusClientImpl(); Status Connect(const ConnectParam& connect_param) final; Status Disconnect() final; Status CreateCollection(const std::string& collection_name, int dim, bool auto_id, bool enable_dynamic_field) final; Status CreateCollection(const CollectionSchema& schema, const IndexDesc& index_desc) final; Status HasCollection(const std::string& collection_name, bool& has) final; Status DropCollection(const std::string& collection_name) final; Status LoadCollection(const std::string& collection_name) final; Status ReleaseCollection(const std::string& collection_name) final; Status Insert(const std::string& collection_name, const std::vector& fields, DmlResults& results) final; Status Delete(const std::string& collection_name, const std::string& expression, DmlResults& results) final; Status Upsert(const std::string& collection_name, const std::vector& fields, DmlResults& results) final; Status Search(const SearchArguments& arguments, SearchResults& results, int timeout) final; Status Query(const QueryArguments& arguments, QueryResults& results, int timeout) final; public: Status CreateCollection(const CollectionSchema& schema); Status DescribeCollection(const std::string& collection_name, CollectionDesc& collection_desc); Status CreateIndex(const std::string& collection_name, const IndexDesc& index_desc); Status DropIndex(const std::string& collection_name, const std::string& field_name); public: using GrpcOpts = MilvusConnection::GrpcContextOptions; /** * Internal wait for status query done. * * @param [in] query_function one time query for return Status, return TIMEOUT status if not done * @param [in] progress_monitor timeout setting for waiting progress * @return Status, the final status */ // static Status WaitForStatus(const std::function& query_function, // const ProgressMonitor& progress_monitor); /** * @brief template for public api call * validate -> pre -> rpc -> wait_for_status -> post */ template Status apiHandler(const std::function& validate, std::function pre, Status (MilvusConnection::*rpc)(const Request&, Response&, const GrpcOpts&), std::function wait_for_status, std::function post, const GrpcOpts& options = GrpcOpts{}) { if (connection_ == nullptr) { return {StatusCode::NOT_CONNECTED, "Connection is not ready!"}; } if (validate) { auto status = validate(); if (!status.IsOk()) { return status; } } Request rpc_request = pre(); Response rpc_response; auto status = std::bind(rpc, connection_.get(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)(rpc_request, rpc_response, options); if (!status.IsOk()) { // response's status already checked in connection class return status; } if (wait_for_status) { status = wait_for_status(rpc_response); } if (status.IsOk() && post) { post(rpc_response); } return status; } /** * @brief template for public api call */ template Status apiHandler(std::function validate, std::function pre, Status (MilvusConnection::*rpc)(const Request&, Response&, const GrpcOpts&), std::function post, const GrpcOpts& options = GrpcOpts{}) { return apiHandler(validate, pre, rpc, std::function{}, post, options); } /** * @brief template for public api call */ template Status apiHandler(std::function validate, std::function pre, Status (MilvusConnection::*rpc)(const Request&, Response&, const GrpcOpts&), const GrpcOpts& options = GrpcOpts{}) { return apiHandler(validate, pre, rpc, std::function{}, std::function{}, options); } /** * @brief template for public api call */ template Status apiHandler(std::function pre, Status (MilvusConnection::*rpc)(const Request&, Response&, const GrpcOpts&), std::function post, const GrpcOpts& options = GrpcOpts{}) { return apiHandler(std::function{}, pre, rpc, std::function{}, post, options); } /** * @brief template for public api call */ template Status apiHandler(std::function pre, Status (MilvusConnection::*rpc)(const Request&, Response&, const GrpcOpts&), const GrpcOpts& options = GrpcOpts{}) { return apiHandler(std::function{}, pre, rpc, std::function{}, std::function{}, options); } private: std::shared_ptr connection_; }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/impl/MilvusConnection.cpp000066400000000000000000000213131520573221700252420ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "MilvusConnection.h" #include #include using grpc::Channel; using grpc::ClientContext; using grpc::ClientReader; using grpc::ClientReaderWriter; using grpc::ClientWriter; using grpc::Status; using Stub = milvus::proto::milvus::MilvusService::Stub; namespace { std::shared_ptr createTlsCredentials(const std::string& cert, const std::string& key, const std::string& ca_cert) { auto read_contents = [](const std::string& filename) -> std::string { if (filename.empty()) { return ""; } std::ifstream fs; fs.open(filename); if (fs) { fs.seekg(0, std::ios::end); auto size = fs.tellg(); std::string buffer(size, '\0'); fs.seekg(0); fs.read(&buffer[0], size); return std::move(buffer); } return ""; }; grpc::SslCredentialsOptions opt{read_contents(ca_cert), read_contents(key), read_contents(cert)}; return ::grpc::SslCredentials(opt); } } // namespace namespace VectorDB { using namespace milvus; MilvusConnection::~MilvusConnection() { Disconnect(); } Status MilvusConnection::Connect(const ConnectParam& param) { authorization_value_ = param.Authorizations(); std::shared_ptr credentials{nullptr}; // auto uri = param.Uri(); const auto& uri = param.Host(); ::grpc::ChannelArguments args; args.SetMaxSendMessageSize(-1); // max send message size: 2GB args.SetMaxReceiveMessageSize(-1); // max receive message size: 2GB if (param.TlsEnabled()) { if (!param.ServerName().empty()) { args.SetSslTargetNameOverride(param.ServerName()); } credentials = createTlsCredentials(param.Cert(), param.Key(), param.CaCert()); } else { credentials = ::grpc::InsecureChannelCredentials(); } channel_ = ::grpc::CreateCustomChannel(uri, credentials, args); auto connected = channel_->WaitForConnected(std::chrono::system_clock::now() + std::chrono::milliseconds{param.ConnectTimeout()}); if (connected) { stub_ = proto::milvus::MilvusService::NewStub(channel_); connectParam_ = param; // 保存连接参数，用来重试连接 return Status::OK(); } std::string reason = "Failed to connect uri: " + uri; return {StatusCode::NOT_CONNECTED, reason}; } Status MilvusConnection::Disconnect() { stub_.reset(); channel_.reset(); return Status::OK(); } Status MilvusConnection::CreateCollection(const proto::milvus::CreateCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("CreateCollection", &Stub::CreateCollection, request, response, options); } Status MilvusConnection::DropCollection(const proto::milvus::DropCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("DropCollection", &Stub::DropCollection, request, response, options); } Status MilvusConnection::HasCollection(const proto::milvus::HasCollectionRequest& request, proto::milvus::BoolResponse& response, const GrpcContextOptions& options) { return grpcCall("HasCollection", &Stub::HasCollection, request, response, options); } Status MilvusConnection::LoadCollection(const proto::milvus::LoadCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("LoadCollection", &Stub::LoadCollection, request, response, options); } Status MilvusConnection::ReleaseCollection(const proto::milvus::ReleaseCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("ReleaseCollection", &Stub::ReleaseCollection, request, response, options); } Status MilvusConnection::DescribeCollection(const proto::milvus::DescribeCollectionRequest& request, proto::milvus::DescribeCollectionResponse& response, const GrpcContextOptions& options) { return grpcCall("DescribeCollection", &Stub::DescribeCollection, request, response, options); } Status MilvusConnection::ShowCollections(const proto::milvus::ShowCollectionsRequest& request, proto::milvus::ShowCollectionsResponse& response, const GrpcContextOptions& options) { return grpcCall("ShowCollections", &Stub::ShowCollections, request, response, options); } Status MilvusConnection::CreateIndex(const proto::milvus::CreateIndexRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("CreateIndex", &Stub::CreateIndex, request, response, options); } Status MilvusConnection::DescribeIndex(const proto::milvus::DescribeIndexRequest& request, proto::milvus::DescribeIndexResponse& response, const GrpcContextOptions& options) { return grpcCall("DescribeIndex", &Stub::DescribeIndex, request, response, options); } Status MilvusConnection::GetIndexState(const proto::milvus::GetIndexStateRequest& request, proto::milvus::GetIndexStateResponse& response, const GrpcContextOptions& options) { return grpcCall("GetIndexState", &Stub::GetIndexState, request, response, options); } Status MilvusConnection::GetIndexBuildProgress(const proto::milvus::GetIndexBuildProgressRequest& request, proto::milvus::GetIndexBuildProgressResponse& response, const GrpcContextOptions& options) { return grpcCall("GetIndexBuildProgress", &Stub::GetIndexBuildProgress, request, response, options); } Status MilvusConnection::DropIndex(const proto::milvus::DropIndexRequest& request, proto::common::Status& response, const GrpcContextOptions& options) { return grpcCall("DropIndex", &Stub::DropIndex, request, response, options); } Status MilvusConnection::Flush(const proto::milvus::FlushRequest& request, proto::milvus::FlushResponse& response, const GrpcContextOptions& options) { return grpcCall("Flush", &Stub::Flush, request, response, options); } Status MilvusConnection::Insert(const proto::milvus::InsertRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options) { return grpcCall("Insert", &Stub::Insert, request, response, options); } Status MilvusConnection::Delete(const proto::milvus::DeleteRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options) { return grpcCall("Delete", &Stub::Delete, request, response, options); } Status MilvusConnection::Upsert(const proto::milvus::UpsertRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options) { return grpcCall("Upsert", &Stub::Upsert, request, response, options); } Status MilvusConnection::Search(const proto::milvus::SearchRequest& request, proto::milvus::SearchResults& response, const GrpcContextOptions& options) { return grpcCall("Search", &Stub::Search, request, response, options); } Status MilvusConnection::Query(const proto::milvus::QueryRequest& request, proto::milvus::QueryResults& response, const GrpcContextOptions& options) { return grpcCall("Query", &Stub::Query, request, response, options); } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/impl/MilvusConnection.h000066400000000000000000000210701520573221700247070ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include #include #include #include #include #include #include "common.pb.h" #include "milvus.grpc.pb.h" #include "milvus.pb.h" #include "schema.pb.h" #include "types/ConnectParam.h" #include "types/Status.h" namespace VectorDB { using namespace milvus; const int RETRY_TIME = 5; class MilvusConnection { public: /** * options for grpc call */ struct GrpcContextOptions { /** timeout in milliseconds */ int timeout{0}; // constructors GrpcContextOptions() = default; explicit GrpcContextOptions(int timeout_) : timeout{timeout_} {} }; MilvusConnection() = default; virtual ~MilvusConnection(); Status Connect(const ConnectParam& param); Status Disconnect(); Status CreateCollection(const proto::milvus::CreateCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status DropCollection(const proto::milvus::DropCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status HasCollection(const proto::milvus::HasCollectionRequest& request, proto::milvus::BoolResponse& response, const GrpcContextOptions& options); Status LoadCollection(const proto::milvus::LoadCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status ReleaseCollection(const proto::milvus::ReleaseCollectionRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status DescribeCollection(const proto::milvus::DescribeCollectionRequest& request, proto::milvus::DescribeCollectionResponse& response, const GrpcContextOptions& options); Status ShowCollections(const proto::milvus::ShowCollectionsRequest& request, proto::milvus::ShowCollectionsResponse& response, const GrpcContextOptions& options); Status CreateIndex(const proto::milvus::CreateIndexRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status DescribeIndex(const proto::milvus::DescribeIndexRequest& request, proto::milvus::DescribeIndexResponse& response, const GrpcContextOptions& options); Status GetIndexState(const proto::milvus::GetIndexStateRequest& request, proto::milvus::GetIndexStateResponse& response, const GrpcContextOptions& options); Status GetIndexBuildProgress(const proto::milvus::GetIndexBuildProgressRequest& request, proto::milvus::GetIndexBuildProgressResponse& response, const GrpcContextOptions& options); Status DropIndex(const proto::milvus::DropIndexRequest& request, proto::common::Status& response, const GrpcContextOptions& options); Status Flush(const proto::milvus::FlushRequest& request, proto::milvus::FlushResponse& response, const GrpcContextOptions& options); Status Insert(const proto::milvus::InsertRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options); Status Delete(const proto::milvus::DeleteRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options); Status Upsert(const proto::milvus::UpsertRequest& request, proto::milvus::MutationResult& response, const GrpcContextOptions& options); Status Search(const proto::milvus::SearchRequest& request, proto::milvus::SearchResults& response, const GrpcContextOptions& options); Status Query(const proto::milvus::QueryRequest& request, proto::milvus::QueryResults& response, const GrpcContextOptions& options); private: std::unique_ptr stub_; std::shared_ptr channel_; std::string authorization_value_{}; ConnectParam connectParam_; static Status StatusByProtoResponse(const proto::common::Status& status) { // keys from kylin-ai-vector-engine/src/status.h static const std::map recover_code_to_status = { {0, StatusCode::OK}, {899, StatusCode::DATABASE_INIT_FAILED} }; if (recover_code_to_status.find(status.code()) != recover_code_to_status.end()) { return Status(recover_code_to_status.at(status.code()), status.reason()); } return Status(StatusCode::SERVER_FAILED, status.reason()); } template static Status StatusByProtoResponse(const Response& response) { const auto& status = response.status(); return StatusByProtoResponse(status); } static StatusCode StatusCodeFromGrpcStatus(const ::grpc::Status& grpc_status) { if (grpc_status.error_code() == ::grpc::StatusCode::DEADLINE_EXCEEDED) { return StatusCode::TIMEOUT; } return StatusCode::SERVER_FAILED; } std::shared_ptr<::grpc::ClientContext> generateGrpcClientContext(const GrpcContextOptions& options) { std::shared_ptr<::grpc::ClientContext> context = std::make_shared<::grpc::ClientContext>(); if (options.timeout > 0) { auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds{options.timeout}; context->set_deadline(deadline); } if (!authorization_value_.empty()) { context->AddMetadata("authorization", authorization_value_); context->set_authority(authorization_value_); } return context; } template Status grpcCall(const char* name, grpc::Status (proto::milvus::MilvusService::Stub::*func)(grpc::ClientContext*, const Request&, Response*), const Request& request, Response& response, const GrpcContextOptions& options) { if (stub_ == nullptr) { return {StatusCode::NOT_CONNECTED, "Connection is not ready!"}; } auto context = generateGrpcClientContext(options); int retryTime = 0; ::grpc::Status grpc_status = (stub_.get()->*func)(context.get(), request, &response); while (grpc_status.error_code() == ::grpc::StatusCode::UNAVAILABLE && retryTime <= RETRY_TIME) { if (retryTime == RETRY_TIME) { std::cerr << "ERROR grpc retry failed, code: " << grpc_status.error_code() << ", msg: " << grpc_status.error_message() << std::endl; return {StatusCodeFromGrpcStatus(grpc_status), grpc_status.error_message()}; } std::cout << "WARN grpc failed, code: " << grpc_status.error_code() << ", msg: " << grpc_status.error_message() << ", retry: " << retryTime << std::endl; // 断开当前连接 Disconnect(); sleep(1); retryTime++; // 重连 Status status = Connect(connectParam_); if (!status.IsOk()) { std::cout << "WARN reconnect milvus failed, code: " << static_cast(status.Code()) << ", msg: " << status.Message() << std::endl; continue; } auto retryContext = generateGrpcClientContext(options); grpc_status = (stub_.get()->*func)(retryContext.get(), request, &response); } return StatusByProtoResponse(response); } }; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/impl/TypeUtils.cpp000066400000000000000000000667511520573221700237240ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "TypeUtils.h" #include "types/Constants.h" namespace VectorDB { using namespace milvus; bool operator==(const proto::schema::FieldData& lhs, const BoolFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_bool_data()) { return false; } const auto& scalars_data = scalars.bool_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const Int8FieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_int_data()) { return false; } const auto& scalars_data = scalars.int_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const Int16FieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_int_data()) { return false; } const auto& scalars_data = scalars.int_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const Int32FieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_int_data()) { return false; } const auto& scalars_data = scalars.int_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const Int64FieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_long_data()) { return false; } const auto& scalars_data = scalars.long_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const FloatFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_float_data()) { return false; } const auto& scalars_data = scalars.float_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const DoubleFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_double_data()) { return false; } const auto& scalars_data = scalars.double_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const VarCharFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_string_data()) { return false; } const auto& scalars_data = scalars.string_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const JsonFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_scalars()) { return false; } const auto& scalars = lhs.scalars(); if (!scalars.has_json_data()) { return false; } const auto& scalars_data = scalars.json_data().data(); if (scalars_data.size() != rhs.Count()) { return false; } return std::equal(scalars_data.begin(), scalars_data.end(), rhs.Data().begin()); } bool operator==(const proto::schema::FieldData& lhs, const BinaryVecFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_vectors()) { return false; } const auto& vectors = lhs.vectors(); if (vectors.has_float_vector()) { return false; } const auto& vectors_data = vectors.binary_vector(); auto it = vectors_data.begin(); const auto& strings = rhs.Data(); for (const auto& s : strings) { for (const auto ch : s) { if (it == vectors_data.end() || *it != ch) { return false; } ++it; } } return it == vectors_data.end(); } bool operator==(const proto::schema::FieldData& lhs, const FloatVecFieldData& rhs) { if (lhs.field_name() != rhs.Name()) { return false; } if (!lhs.has_vectors()) { return false; } size_t dim = 0; if (rhs.Count() > 0) { dim = rhs.Data().front().size(); } const auto& vectors = lhs.vectors(); if (!vectors.has_float_vector()) { return false; } const auto& vectors_data = vectors.float_vector().data(); if (vectors_data.size() != (rhs.Count() * dim)) { return false; } auto it = vectors_data.begin(); for (const auto& item : rhs.Data()) { if (!std::equal(item.begin(), item.end(), it)) { return false; } std::advance(it, dim); } return true; } bool operator==(const proto::schema::FieldData& lhs, const Field& rhs) { auto data_type = rhs.Type(); switch (data_type) { case DataType::BOOL: return lhs == dynamic_cast(rhs); case DataType::INT8: return lhs == dynamic_cast(rhs); case DataType::INT16: return lhs == dynamic_cast(rhs); case DataType::INT32: return lhs == dynamic_cast(rhs); case DataType::INT64: return lhs == dynamic_cast(rhs); case DataType::FLOAT: return lhs == dynamic_cast(rhs); case DataType::DOUBLE: return lhs == dynamic_cast(rhs); case DataType::VARCHAR: return lhs == dynamic_cast(rhs); case DataType::JSON: return lhs == dynamic_cast(rhs); case DataType::BINARY_VECTOR: return lhs == dynamic_cast(rhs); case DataType::FLOAT_VECTOR: return lhs == dynamic_cast(rhs); default: return false; } } proto::schema::DataType DataTypeCast(DataType type) { switch (type) { case DataType::BOOL: return proto::schema::DataType::Bool; case DataType::INT8: return proto::schema::DataType::Int8; case DataType::INT16: return proto::schema::DataType::Int16; case DataType::INT32: return proto::schema::DataType::Int32; case DataType::INT64: return proto::schema::DataType::Int64; case DataType::FLOAT: return proto::schema::DataType::Float; case DataType::DOUBLE: return proto::schema::DataType::Double; case DataType::VARCHAR: return proto::schema::DataType::VarChar; case DataType::JSON: return proto::schema::DataType::JSON; case DataType::BINARY_VECTOR: return proto::schema::DataType::BinaryVector; case DataType::FLOAT_VECTOR: return proto::schema::DataType::FloatVector; default: return proto::schema::DataType::None; } } DataType DataTypeCast(proto::schema::DataType type) { switch (type) { case proto::schema::DataType::Bool: return DataType::BOOL; case proto::schema::DataType::Int8: return DataType::INT8; case proto::schema::DataType::Int16: return DataType::INT16; case proto::schema::DataType::Int32: return DataType::INT32; case proto::schema::DataType::Int64: return DataType::INT64; case proto::schema::DataType::Float: return DataType::FLOAT; case proto::schema::DataType::Double: return DataType::DOUBLE; case proto::schema::DataType::VarChar: return DataType::VARCHAR; case proto::schema::DataType::JSON: return DataType::JSON; case proto::schema::DataType::BinaryVector: return DataType::BINARY_VECTOR; case proto::schema::DataType::FloatVector: return DataType::FLOAT_VECTOR; default: return DataType::UNKNOWN; } } MetricType MetricTypeCast(const std::string& type) { if (type == "L2") { return MetricType::L2; } if (type == "IP") { return MetricType::IP; } if (type == "COSINE") { return MetricType::COSINE; } if (type == "HAMMING") { return MetricType::HAMMING; } if (type == "JACCARD") { return MetricType::JACCARD; } if (type == "TANIMOTO") { return MetricType::TANIMOTO; } if (type == "SUBSTRUCTURE") { return MetricType::SUBSTRUCTURE; } if (type == "SUPERSTRUCTURE") { return MetricType::SUPERSTRUCTURE; } return MetricType::INVALID; } IndexType IndexTypeCast(const std::string& type) { if (type == "FLAT") { return IndexType::FLAT; } if (type == "IVF_FLAT") { return IndexType::IVF_FLAT; } if (type == "IVF_SQ8") { return IndexType::IVF_SQ8; } if (type == "IVF_PQ") { return IndexType::IVF_PQ; } if (type == "HNSW") { return IndexType::HNSW; } if (type == "IVF_HNSW") { return IndexType::IVF_HNSW; } if (type == "RHNSW_FLAT") { return IndexType::RHNSW_FLAT; } if (type == "RHNSW_SQ") { return IndexType::RHNSW_SQ; } if (type == "RHNSW_PQ") { return IndexType::RHNSW_PQ; } if (type == "ANNOY") { return IndexType::ANNOY; } if (type == "BIN_FLAT") { return IndexType::BIN_FLAT; } if (type == "BIN_IVF_FLAT") { return IndexType::BIN_IVF_FLAT; } return IndexType::INVALID; } proto::schema::VectorField* CreateProtoFieldData(const BinaryVecFieldData& field) { auto ret = new proto::schema::VectorField{}; auto& data = field.Data(); auto dim = data.front().size() * 8; auto& vectors_data = *(ret->mutable_binary_vector()); vectors_data.reserve(data.size() * dim); for (const auto& item : data) { std::copy(item.begin(), item.end(), std::back_inserter(vectors_data)); } ret->set_dim(static_cast(dim)); return ret; } proto::schema::VectorField* CreateProtoFieldData(const FloatVecFieldData& field) { auto ret = new proto::schema::VectorField{}; auto& data = field.Data(); auto dim = data.front().size(); auto& vectors_data = *(ret->mutable_float_vector()->mutable_data()); vectors_data.Reserve(static_cast(data.size() * dim)); for (const auto& item : data) { for (const auto& tmp : item) { vectors_data.Add(tmp); } } ret->set_dim(static_cast(dim)); return ret; } proto::schema::ScalarField* CreateProtoFieldData(const BoolFieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_bool_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const Int8FieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_int_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const Int16FieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_int_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const Int32FieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_int_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const Int64FieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_long_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const FloatFieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_float_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const DoubleFieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_double_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const VarCharFieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_string_data()); for (const auto& item : data) { scalars_data.add_data(item); } return ret; } proto::schema::ScalarField* CreateProtoFieldData(const JsonFieldData& field) { auto ret = new proto::schema::ScalarField{}; auto& data = field.Data(); auto& scalars_data = *(ret->mutable_json_data()->mutable_data()); for (const auto& item : data) { scalars_data.Add(item.dump()); } return ret; } proto::schema::FieldData CreateProtoFieldData(const Field& field) { proto::schema::FieldData field_data; const auto field_type = field.Type(); field_data.set_field_name(field.Name()); field_data.set_type(DataTypeCast(field_type)); switch (field_type) { case DataType::BINARY_VECTOR: field_data.set_allocated_vectors(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::FLOAT_VECTOR: field_data.set_allocated_vectors(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::BOOL: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::INT8: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::INT16: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::INT32: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::INT64: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::FLOAT: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::DOUBLE: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::VARCHAR: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); break; case DataType::JSON: field_data.set_allocated_scalars(CreateProtoFieldData(dynamic_cast(field))); // todo 临时设置 field_data.set_is_dynamic(true); break; default: break; } return field_data; } FieldDataPtr CreateMilvusFieldData(const milvus::proto::schema::FieldData& field_data, size_t offset, size_t count) { auto field_type = field_data.type(); const auto& name = field_data.field_name(); switch (field_type) { case proto::schema::DataType::BinaryVector: return std::make_shared( name, BuildFieldDataVectors(field_data.vectors().dim() / 8, field_data.vectors().binary_vector(), offset, count)); case proto::schema::DataType::FloatVector: return std::make_shared( name, BuildFieldDataVectors>( field_data.vectors().dim(), field_data.vectors().float_vector().data(), offset, count)); case proto::schema::DataType::Bool: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().bool_data().data(), offset, count)); case proto::schema::DataType::Int8: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data(), offset, count)); case proto::schema::DataType::Int16: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data(), offset, count)); case proto::schema::DataType::Int32: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data(), offset, count)); case proto::schema::DataType::Int64: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().long_data().data(), offset, count)); case proto::schema::DataType::Float: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().float_data().data(), offset, count)); case proto::schema::DataType::Double: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().double_data().data(), offset, count)); case proto::schema::DataType::VarChar: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().string_data().data(), offset, count)); case proto::schema::DataType::JSON: if (field_data.is_dynamic()) { // todo 临时实现，动态列的名字是 $meta，不是insert时设置的field name // std::vector tmp = ; return std::make_shared( DYNAMIC_FIELD_NAME, BuildJsonFieldDataScalars(field_data.scalars().json_data().data(), offset, count)); } else { return std::make_shared( name, BuildJsonFieldDataScalars(field_data.scalars().json_data().data(), offset, count)); } default: return nullptr; } } FieldDataPtr CreateMilvusFieldData(const milvus::proto::schema::FieldData& field_data) { auto field_type = field_data.type(); const auto& name = field_data.field_name(); switch (field_type) { case proto::schema::DataType::BinaryVector: return std::make_shared( name, BuildFieldDataVectors(field_data.vectors().dim() / 8, field_data.vectors().binary_vector())); case proto::schema::DataType::FloatVector: return std::make_shared( name, BuildFieldDataVectors>(field_data.vectors().dim(), field_data.vectors().float_vector().data())); case proto::schema::DataType::Bool: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().bool_data().data())); case proto::schema::DataType::Int8: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data())); case proto::schema::DataType::Int16: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data())); case proto::schema::DataType::Int32: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().int_data().data())); case proto::schema::DataType::Int64: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().long_data().data())); case proto::schema::DataType::Float: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().float_data().data())); case proto::schema::DataType::Double: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().double_data().data())); case proto::schema::DataType::VarChar: return std::make_shared( name, BuildFieldDataScalars(field_data.scalars().string_data().data())); case proto::schema::DataType::JSON: if (field_data.is_dynamic()) { // todo 临时实现，动态列的名字是 $meta，不是insert时设置的field name return std::make_shared( DYNAMIC_FIELD_NAME, BuildJsonFieldDataScalars(field_data.scalars().json_data().data())); } else { return std::make_shared( name, BuildJsonFieldDataScalars(field_data.scalars().json_data().data())); } default: return nullptr; } } IDArray CreateIDArray(const proto::schema::IDs& ids) { if (ids.has_int_id()) { std::vector int_array; auto& int_ids = ids.int_id(); int_array.reserve(int_ids.data_size()); std::copy(int_ids.data().begin(), int_ids.data().end(), std::back_inserter(int_array)); return IDArray(int_array); } else { std::vector str_array; auto& str_ids = ids.str_id(); str_array.reserve(str_ids.data_size()); std::copy(str_ids.data().begin(), str_ids.data().end(), std::back_inserter(str_array)); return IDArray(str_array); } } IDArray CreateIDArray(const proto::schema::IDs& ids, size_t offset, size_t size) { if (ids.has_int_id()) { std::vector int_array; auto& int_ids = ids.int_id(); int_array.reserve(size); auto it = int_ids.data().begin(); std::advance(it, offset); auto it_end = it; std::advance(it_end, size); std::copy(it, it_end, std::back_inserter(int_array)); return IDArray(int_array); } else { std::vector str_array; auto& str_ids = ids.str_id(); str_array.reserve(size); auto it = str_ids.data().begin(); std::advance(it, offset); auto it_end = it; std::advance(it_end, size); std::copy(it, it_end, std::back_inserter(str_array)); return IDArray(str_array); } } void ConvertFieldSchema(const proto::schema::FieldSchema& proto_schema, FieldSchema& field_schema) { field_schema.SetName(proto_schema.name()); field_schema.SetDescription(proto_schema.description()); field_schema.SetPrimaryKey(proto_schema.is_primary_key()); field_schema.SetAutoID(proto_schema.autoid()); field_schema.SetDataType(DataTypeCast(proto_schema.data_type())); std::map params; for (int k = 0; k < proto_schema.type_params_size(); ++k) { auto& kv = proto_schema.type_params(k); params.emplace(kv.key(), kv.value()); } field_schema.SetTypeParams(std::move(params)); } void ConvertCollectionSchema(const proto::schema::CollectionSchema& proto_schema, CollectionSchema& schema) { schema.SetName(proto_schema.name()); schema.SetDescription(proto_schema.description()); for (int i = 0; i < proto_schema.fields_size(); ++i) { auto& proto_field = proto_schema.fields(i); FieldSchema field_schema; ConvertFieldSchema(proto_field, field_schema); schema.AddField(std::move(field_schema)); } } void ConvertFieldSchema(const FieldSchema& schema, proto::schema::FieldSchema& proto_schema) { proto_schema.set_name(schema.Name()); proto_schema.set_description(schema.Description()); proto_schema.set_is_primary_key(schema.IsPrimaryKey()); proto_schema.set_autoid(schema.AutoID()); proto_schema.set_data_type(DataTypeCast(schema.FieldDataType())); for (auto& kv : schema.TypeParams()) { auto pair = proto_schema.add_type_params(); pair->set_key(kv.first); pair->set_value(kv.second); } } void ConvertCollectionSchema(const CollectionSchema& schema, proto::schema::CollectionSchema& proto_schema) { proto_schema.set_name(schema.Name()); proto_schema.set_description(schema.Description()); for (auto& field : schema.Fields()) { auto proto_field = proto_schema.add_fields(); ConvertFieldSchema(field, *proto_field); } } bool IsVectorType(DataType type) { return (DataType::BINARY_VECTOR == type || DataType::FLOAT_VECTOR == type); } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/impl/TypeUtils.h000066400000000000000000000136751520573221700233660ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include "milvus.pb.h" #include "types/CollectionSchema.h" #include "types/FieldData.h" #include "types/IDArray.h" #include "types/IndexType.h" #include "types/MetricType.h" namespace VectorDB { using namespace milvus; bool operator==(const proto::schema::FieldData& lhs, const BoolFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const Int8FieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const Int16FieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const Int32FieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const Int64FieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const FloatFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const DoubleFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const VarCharFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const JsonFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const BinaryVecFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const FloatVecFieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const proto::schema::FieldData& rhs); bool operator==(const proto::schema::FieldData& lhs, const Field& rhs); proto::schema::DataType DataTypeCast(DataType type); DataType DataTypeCast(proto::schema::DataType type); MetricType MetricTypeCast(const std::string& type); IndexType IndexTypeCast(const std::string& type); proto::schema::VectorField* CreateProtoFieldData(const BinaryVecFieldData& field); proto::schema::VectorField* CreateProtoFieldData(const FloatVecFieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const BoolFieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const Int8FieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const Int16FieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const Int32FieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const Int64FieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const FloatFieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const DoubleFieldData& field); proto::schema::ScalarField* CreateProtoFieldData(const VarCharFieldData& field); proto::schema::FieldData CreateProtoFieldData(const Field& field); template std::vector BuildFieldDataVectors(int64_t dim_bytes, const VectorData& vector_data, size_t offset, size_t count) { std::vector data{}; data.reserve(count * dim_bytes); auto cursor = vector_data.begin(); std::advance(cursor, offset * dim_bytes); auto end = cursor; std::advance(end, count * dim_bytes); while (cursor != end) { T item{}; item.reserve(dim_bytes); std::copy_n(cursor, dim_bytes, std::back_inserter(item)); data.emplace_back(std::move(item)); std::advance(cursor, dim_bytes); } return data; } template std::vector BuildFieldDataVectors(int64_t dim_bytes, const VectorData& vector_data) { return BuildFieldDataVectors(dim_bytes, vector_data, 0, vector_data.size() / dim_bytes); } template std::vector BuildFieldDataScalars(const ScalarData& scalar_data, size_t offset, size_t count) { std::vector data{}; data.reserve(count); auto begin = scalar_data.begin(); std::advance(begin, offset); auto end = begin; std::advance(end, count); std::copy(begin, end, std::back_inserter(data)); return data; } template std::vector BuildJsonFieldDataScalars(const ScalarData& scalar_data, size_t offset, size_t count) { std::vector data{}; data.reserve(count); auto begin = scalar_data.begin(); std::advance(begin, offset); auto end = begin; std::advance(end, count); for (auto it = begin; it != end; it++) { data.push_back(nlohmann::json::parse(*it)); } return data; } template std::vector BuildJsonFieldDataScalars(const ScalarData& scalar_data) { return BuildJsonFieldDataScalars(scalar_data, 0, scalar_data.size()); } template std::vector BuildFieldDataScalars(const ScalarData& scalar_data) { return BuildFieldDataScalars(scalar_data, 0, scalar_data.size()); } FieldDataPtr CreateMilvusFieldData(const proto::schema::FieldData& field_data, size_t offset, size_t count); FieldDataPtr CreateMilvusFieldData(const proto::schema::FieldData& field_data); IDArray CreateIDArray(const proto::schema::IDs& ids); IDArray CreateIDArray(const proto::schema::IDs& ids, size_t offset, size_t size); void ConvertFieldSchema(const proto::schema::FieldSchema& proto_schema, FieldSchema& schema); void ConvertCollectionSchema(const proto::schema::CollectionSchema& proto_schema, CollectionSchema& schema); void ConvertFieldSchema(const FieldSchema& schema, proto::schema::FieldSchema& proto_schema); void ConvertCollectionSchema(const CollectionSchema& schema, proto::schema::CollectionSchema& proto_schema); bool IsVectorType(DataType type); } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/swImpl/000077500000000000000000000000001520573221700215515ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/src/swImpl/SwClientImpl.cpp000066400000000000000000000057251520573221700246400ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifdef ENABLE_SW_ARCH #include "SwClientImpl.h" using namespace VectorDB; Status SwClientImpl::Connect(const ConnectParam& connect_param) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Disconnect() { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::CreateCollection(const CollectionSchema& schema, const IndexDesc& index_desc) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::CreateCollection(const std::string& collection_name, int dim, bool auto_id, bool enable_dynamic_field) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::HasCollection(const std::string& collection_name, bool& has) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::DropCollection(const std::string& collection_name) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Insert(const std::string& collection_name, const std::vector& fields, DmlResults& results) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Delete(const std::string& collection_name, const std::string& expression, DmlResults& results) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Upsert(const std::string& collection_name, const std::vector& fields, DmlResults& results) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Search(const SearchArguments& arguments, SearchResults& results, int timeout) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::Query(const QueryArguments& arguments, QueryResults& results, int timeout) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::LoadCollection(const std::string& collection_name) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } Status SwClientImpl::ReleaseCollection(const std::string& collection_name) { return Status(StatusCode::NOT_SUPPORTED, "NOT_SUPPORTED"); } #endif libkysdk-vector-engine-client-1.2.0.0/src/swImpl/SwClientImpl.h000066400000000000000000000042301520573221700242730ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #ifdef ENABLE_SW_ARCH #include "Database.h" namespace VectorDB { class SwClientImpl : public Database { public: Status Connect(const ConnectParam& connect_param) override; Status Disconnect() override; Status CreateCollection(const CollectionSchema& schema, const IndexDesc& index_desc) override; Status CreateCollection(const std::string& collection_name, int dim, bool auto_id, bool enable_dynamic_field) override; Status HasCollection(const std::string& collection_name, bool& has) override; Status DropCollection(const std::string& collection_name) override; Status Insert(const std::string& collection_name, const std::vector& fields, DmlResults& results) override; Status Delete(const std::string& collection_name, const std::string& expression, DmlResults& results) override; Status Upsert(const std::string& collection_name, const std::vector& fields, DmlResults& results) override; Status Search(const SearchArguments& arguments, SearchResults& results, int timeout) override; Status Query(const QueryArguments& arguments, QueryResults& results, int timeout) override; Status LoadCollection(const std::string& collection_name) override; Status ReleaseCollection(const std::string& collection_name) override; }; } // namespace VectorDB #endif libkysdk-vector-engine-client-1.2.0.0/src/types/000077500000000000000000000000001520573221700214425ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/src/types/CollectionDesc.cpp000066400000000000000000000032241520573221700250410ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/CollectionDesc.h" namespace VectorDB { const CollectionSchema& CollectionDesc::Schema() const { return schema_; } void CollectionDesc::SetSchema(const CollectionSchema& schema) { schema_ = schema; } void CollectionDesc::SetSchema(CollectionSchema&& schema) { schema_ = std::move(schema); } int64_t CollectionDesc::ID() const { return collection_id_; } void CollectionDesc::SetID(int64_t id) { collection_id_ = id; } const std::vector& CollectionDesc::Alias() const { return alias_; } void CollectionDesc::SetAlias(const std::vector& alias) { alias_ = alias; } void CollectionDesc::SetAlias(std::vector&& alias) { alias_ = std::move(alias); } uint64_t CollectionDesc::CreatedTime() const { return created_utc_timestamp_; } void CollectionDesc::SetCreatedTime(uint64_t ts) { created_utc_timestamp_ = ts; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/CollectionSchema.cpp000066400000000000000000000060051520573221700253630ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/CollectionSchema.h" #include namespace VectorDB { CollectionSchema::CollectionSchema() = default; CollectionSchema::CollectionSchema(std::string name, std::string desc, bool enable_dynamic_field, int32_t shard_num) : name_(std::move(name)), description_(std::move(desc)), enable_dynamic_field_(enable_dynamic_field), shard_num_(shard_num) {} const std::string& CollectionSchema::Name() const { return name_; } void CollectionSchema::SetName(std::string name) { name_ = std::move(name); } const std::string& CollectionSchema::Description() const { return description_; } void CollectionSchema::SetDescription(std::string description) { description_ = std::move(description); } bool CollectionSchema::EnableDynamicField() const { return enable_dynamic_field_; } void CollectionSchema::SetEnableDynamicField(bool enable_dynamic_field) { enable_dynamic_field_ = enable_dynamic_field; } int32_t CollectionSchema::ShardsNum() const { return shard_num_; } void CollectionSchema::SetShardsNum(int32_t num) { shard_num_ = num; } const std::vector& CollectionSchema::Fields() const { return fields_; } bool CollectionSchema::AddField(const FieldSchema& field_schema) { const auto it = std::find_if(fields_.cbegin(), fields_.cend(), [&field_schema](const FieldSchema& field) { return field.Name() == field_schema.Name(); }); if (it != fields_.cend()) { return false; } fields_.emplace_back(field_schema); return true; } bool CollectionSchema::AddField(FieldSchema&& field_schema) { const auto it = std::find_if(fields_.cbegin(), fields_.cend(), [&field_schema](const FieldSchema& field) { return field.Name() == field_schema.Name(); }); if (it != fields_.cend()) { return false; } fields_.emplace_back(std::move(field_schema)); return true; } std::unordered_set CollectionSchema::AnnsFieldNames() const { std::unordered_set ret; for (const auto& field : fields_) { auto data_type = field.FieldDataType(); if (data_type == DataType::BINARY_VECTOR || data_type == DataType::FLOAT_VECTOR) { ret.emplace(field.Name()); } } return ret; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/ConnectParam.cpp000066400000000000000000000065651520573221700245340ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/ConnectParam.h" #include #include #include "types/Utils.h" namespace VectorDB { ConnectParam::ConnectParam() { host_ = "unix:/tmp/kylin-ai-vector-engine-" + std::to_string(geteuid()) + ".sock"; } ConnectParam::ConnectParam(std::string host, uint16_t port) : host_(std::move(host)), port_(port) {} ConnectParam::ConnectParam(std::string host, uint16_t port, std::string username, std::string password) : host_(std::move(host)), port_(port) { SetAuthorizations(std::move(username), std::move(password)); } const std::string& ConnectParam::Host() const { return host_; } uint16_t ConnectParam::Port() const { return port_; } std::string ConnectParam::Uri() const { return host_ + ":" + std::to_string(port_); } const std::string& ConnectParam::Authorizations() const { return authorizations_; } void ConnectParam::SetAuthorizations(std::string username, std::string password) { authorizations_ = VectorDB::Base64Encode(std::move(username) + ':' + std::move(password)); } uint32_t ConnectParam::ConnectTimeout() const { return connect_timeout_; } void ConnectParam::SetConnectTimeout(uint32_t timeout) { connect_timeout_ = timeout; } ConnectParam& ConnectParam::WithTls() { EnableTls(); return *this; } void ConnectParam::EnableTls() { EnableTls("", "", "", ""); } ConnectParam& ConnectParam::WithTls(const std::string& server_name, const std::string& ca_cert) { EnableTls(server_name, ca_cert); return *this; } void ConnectParam::EnableTls(const std::string& server_name, const std::string& ca_cert) { EnableTls(server_name, "", "", ca_cert); } ConnectParam& ConnectParam::WithTls(const std::string& server_name, const std::string& cert, const std::string& key, const std::string& ca_cert) { EnableTls(server_name, cert, key, ca_cert); return *this; } void ConnectParam::EnableTls(const std::string& server_name, const std::string& cert, const std::string& key, const std::string& ca_cert) { tls_ = true; server_name_ = server_name; cert_ = cert; key_ = key; ca_cert_ = ca_cert; } void ConnectParam::DisableTls() { tls_ = false; server_name_.clear(); cert_.clear(); key_.clear(); ca_cert_.clear(); } bool ConnectParam::TlsEnabled() const { return tls_; } const std::string& ConnectParam::ServerName() const { return server_name_; } const std::string& ConnectParam::Cert() const { return cert_; } const std::string& ConnectParam::Key() const { return key_; } const std::string& ConnectParam::CaCert() const { return ca_cert_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/DmlResults.cpp000066400000000000000000000023301520573221700242420ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/DmlResults.h" namespace VectorDB { const IDArray& DmlResults::IdArray() const { return id_array_; } void DmlResults::SetIdArray(const IDArray& id_array) { id_array_ = id_array; } void DmlResults::SetIdArray(IDArray&& id_array) { id_array_ = std::move(id_array); } uint64_t DmlResults::Timestamp() const { return timestamp_; } void DmlResults::SetTimestamp(uint64_t timestamp) { timestamp_ = timestamp; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/FieldData.cpp000066400000000000000000000133721520573221700237710ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/FieldData.h" namespace VectorDB { namespace { template struct DataTypeTraits { static const bool is_vector = false; }; template <> struct DataTypeTraits { static const bool is_vector = true; }; template <> struct DataTypeTraits { static const bool is_vector = true; }; template ::is_vector, bool> = true> StatusCode AddElement(const T& element, std::vector& array) { array.push_back(element); return StatusCode::OK; } template ::is_vector, bool> = true> StatusCode AddElement(const T& element, std::vector& array) { if (element.empty()) { return StatusCode::VECTOR_IS_EMPTY; } if (!array.empty() && element.size() != array.at(0).size()) { return StatusCode::DIMENSION_NOT_EQUAL; } array.emplace_back(element); return StatusCode::OK; } } // namespace const std::string& Field::Name() const { return name_; } DataType Field::Type() const { return data_type_; } Field::Field(std::string name, DataType data_type) : name_(std::move(name)), data_type_(data_type) {} template FieldData::FieldData() : Field("", Dt) {} template FieldData::FieldData(std::string name) : Field(std::move(name), Dt) {} template FieldData::FieldData(std::string name, const std::vector& data) : Field(std::move(name), Dt), data_(data) {} template FieldData::FieldData(std::string name, std::vector&& data) : Field(std::move(name), Dt), data_(std::move(data)) {} template StatusCode FieldData::Add(const T& element) { return AddElement(element, data_); } template StatusCode FieldData::Add(T&& element) { return AddElement(std::move(element), data_); } template size_t FieldData::Count() const { return data_.size(); } template const std::vector& FieldData::Data() const { return data_; } template std::vector& FieldData::Data() { return data_; } BinaryVecFieldData::BinaryVecFieldData() : FieldData() {} BinaryVecFieldData::BinaryVecFieldData(std::string name) : FieldData(std::move(name)) {} BinaryVecFieldData::BinaryVecFieldData(std::string name, const std::vector& data) : FieldData(std::move(name), data) {} BinaryVecFieldData::BinaryVecFieldData(std::string name, std::vector&& data) : FieldData(std::move(name), std::move(data)) {} BinaryVecFieldData::BinaryVecFieldData(std::string name, const std::vector>& data) : FieldData(std::move(name), CreateBinaryStrings(data)) {} const std::vector& BinaryVecFieldData::Data() const { return data_; } std::vector& BinaryVecFieldData::Data() { return data_; } std::vector> BinaryVecFieldData::DataAsUnsignedChars() const { std::vector> ret; ret.reserve(data_.size()); for (const auto& item : data_) { ret.emplace_back(item.begin(), item.end()); } return ret; } StatusCode BinaryVecFieldData::Add(const std::string& element) { return AddElement(element, data_); } StatusCode BinaryVecFieldData::Add(std::string&& element) { return AddElement(element, data_); } StatusCode BinaryVecFieldData::Add(const std::vector& element) { return Add(std::string{element.begin(), element.end()}); } std::vector BinaryVecFieldData::CreateBinaryStrings(const std::vector>& data) { std::vector ret; ret.reserve(data.size()); for (const auto& item : data) { ret.emplace_back(item.begin(), item.end()); } return ret; } std::string BinaryVecFieldData::CreateBinaryString(const std::vector& data) { return std::string{data.begin(), data.end()}; } // explicit declare FieldData template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData; template class FieldData, DataType::FLOAT_VECTOR>; } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/FieldSchema.cpp000066400000000000000000000065441520573221700243230ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/FieldSchema.h" #include "types/Constants.h" namespace VectorDB { FieldSchema::FieldSchema() = default; FieldSchema::FieldSchema(std::string name, DataType data_type, std::string description, bool is_primary_key, bool auto_id) : name_(std::move(name)), description_(std::move(description)), data_type_(data_type), is_primary_key_(is_primary_key), auto_id_(auto_id), is_dynamic_(false) {} const std::string& FieldSchema::Name() const { return name_; } void FieldSchema::SetName(std::string name) { name_ = std::move(name); } const std::string& FieldSchema::Description() const { return description_; } void FieldSchema::SetDescription(std::string description) { description_ = std::move(description); } DataType FieldSchema::FieldDataType() const { return data_type_; } void FieldSchema::SetDataType(DataType dt) { data_type_ = dt; } bool FieldSchema::IsPrimaryKey() const { return is_primary_key_; } void FieldSchema::SetPrimaryKey(bool is_primary_key) { is_primary_key_ = is_primary_key; } bool FieldSchema::AutoID() const { return auto_id_; } void FieldSchema::SetAutoID(bool auto_id) { auto_id_ = auto_id; } bool FieldSchema::IsDynamic() const { return is_dynamic_; } void FieldSchema::SetDynamic(bool is_dynamic) { is_dynamic_ = is_dynamic; } const std::map& FieldSchema::TypeParams() const { return type_params_; } void FieldSchema::SetTypeParams(const std::map& params) { type_params_ = params; } void FieldSchema::SetTypeParams(std::map&& params) { type_params_ = std::move(params); } uint32_t FieldSchema::Dimension() const { auto iter = type_params_.find(FieldDim()); if (iter != type_params_.end()) { return std::strtol(iter->second.c_str(), nullptr, 10); } return 0; } bool FieldSchema::SetDimension(uint32_t dimension) { if (dimension == 0) { return false; } type_params_[FieldDim()] = std::to_string(dimension); return true; } FieldSchema& FieldSchema::WithDimension(uint32_t dimension) { (void)SetDimension(dimension); return *this; } uint32_t FieldSchema::MaxLength() const { auto iter = type_params_.find(FieldMaxLength()); if (iter != type_params_.end()) { return std::strtol(iter->second.c_str(), nullptr, 10); } return 0; } void FieldSchema::SetMaxLength(uint32_t length) { type_params_[FieldMaxLength()] = std::to_string(length); } FieldSchema& FieldSchema::WithMaxLength(uint32_t length) { SetMaxLength(length); return *this; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/IDArray.cpp000066400000000000000000000027321520573221700234450ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/IDArray.h" namespace VectorDB { IDArray::IDArray(const std::vector& id_array) : int_id_array_(id_array) {} IDArray::IDArray(std::vector&& id_array) : int_id_array_(std::move(id_array)) {} IDArray::IDArray(const std::vector& id_array) : str_id_array_(id_array), is_int_array_{false} {} IDArray::IDArray(std::vector&& id_array) : str_id_array_(std::move(id_array)), is_int_array_{false} {} bool IDArray::IsIntegerID() const { return is_int_array_; } const std::vector& IDArray::IntIDArray() const { return int_id_array_; } const std::vector& IDArray::StrIDArray() const { return str_id_array_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/IndexDesc.cpp000066400000000000000000000155271520573221700240260ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/IndexDesc.h" #include #include "types/Utils.h" namespace VectorDB { namespace { struct Validation { IndexType index_type; std::string param; int64_t min; int64_t max; bool required; Status Validate(const IndexDesc& data, std::unordered_map params) const { if (data.IndexType() == index_type) { auto it = params.find(param); // required and not found if (it == params.end() && required) { return {StatusCode::INVALID_AGUMENT, "missing required parameter: " + param}; } // found, check value if (it != params.end()) { auto value = it->second; if (value < min || value > max) { return {StatusCode::INVALID_AGUMENT, "invalid value: " + param + "=" + std::to_string(value) + ", requires [" + std::to_string(min) + ", " + std::to_string(max) + "]"}; } } } return Status::OK(); } }; Status validate_index_and_metric(const MetricType metric_type, const IndexType index_type) { if ((metric_type == MetricType::IP || metric_type == MetricType::L2 || metric_type == MetricType::COSINE) && (index_type == IndexType::FLAT || index_type == IndexType::IVF_FLAT || index_type == IndexType::IVF_SQ8 || index_type == IndexType::IVF_PQ || index_type == IndexType::HNSW || index_type == IndexType::IVF_HNSW || index_type == IndexType::RHNSW_FLAT || index_type == IndexType::RHNSW_SQ || index_type == IndexType::RHNSW_PQ || index_type == IndexType::ANNOY)) { return Status::OK(); } if ((metric_type == MetricType::JACCARD || metric_type == MetricType::TANIMOTO || metric_type == MetricType::HAMMING) && (index_type == IndexType::BIN_FLAT || index_type == IndexType::BIN_IVF_FLAT)) { return Status::OK(); } if ((metric_type == MetricType::SUBSTRUCTURE || metric_type == MetricType::SUPERSTRUCTURE) && (index_type == IndexType::BIN_FLAT)) { return Status::OK(); } return {StatusCode::INVALID_AGUMENT, "Index type and metric type not match, index " + std::to_string(index_type) + " with metric " + std::to_string(metric_type)}; } Status validate_params(const IndexDesc& data, const std::unordered_map& params) { auto status = Status::OK(); auto validations = { Validation{IndexType::IVF_FLAT, "nlist", 1, 65536, true}, Validation{IndexType::IVF_SQ8, "nlist", 1, 65536, true}, Validation{IndexType::IVF_PQ, "nlist", 1, 65536, true}, Validation{IndexType::IVF_PQ, "m", 1, 65536, true}, // TODO: m requires mod(dim) == 0 Validation{IndexType::IVF_PQ, "nbits", 1, 16, false}, Validation{IndexType::HNSW, "M", 4, 64, true}, Validation{IndexType::HNSW, "efConstruction", 8, 512, true}, Validation{IndexType::IVF_HNSW, "nlist", 1, 65536, true}, Validation{IndexType::IVF_HNSW, "M", 4, 64, true}, Validation{IndexType::IVF_HNSW, "efConstruction", 8, 512, true}, Validation{IndexType::RHNSW_FLAT, "M", 4, 64, true}, Validation{IndexType::RHNSW_FLAT, "efConstruction", 8, 512, true}, Validation{IndexType::RHNSW_SQ, "M", 4, 64, true}, Validation{IndexType::RHNSW_SQ, "efConstruction", 8, 512, true}, Validation{IndexType::RHNSW_PQ, "M", 4, 64, true}, Validation{IndexType::RHNSW_PQ, "efConstruction", 8, 512, true}, Validation{IndexType::RHNSW_PQ, "PQM", 1, 65536, true}, // TODO: PQM requires mod(dim) == 0 Validation{IndexType::ANNOY, "n_trees", 1, 1024, true}, }; for (const auto& validation : validations) { status = validation.Validate(data, params); if (!status.IsOk()) { return status; } } return status; } } // namespace IndexDesc::IndexDesc() = default; IndexDesc::IndexDesc(std::string field_name, std::string index_name, int64_t index_id, VectorDB::IndexType index_type, VectorDB::MetricType metric_type) : field_name_(std::move(field_name)), index_name_(std::move(index_name)), index_type_(index_type), metric_type_(metric_type), index_id_(index_id) {} const std::string& IndexDesc::FieldName() const { return field_name_; } Status IndexDesc::SetFieldName(std::string field_name) { field_name_ = std::move(field_name); return Status::OK(); } const std::string& IndexDesc::IndexName() const { return index_name_; } Status IndexDesc::SetIndexName(std::string index_name) { index_name_ = std::move(index_name); return Status::OK(); } int64_t IndexDesc::IndexId() const { return index_id_; } Status IndexDesc::SetIndexId(int64_t index_id) { index_id_ = index_id; return Status::OK(); } VectorDB::MetricType IndexDesc::MetricType() const { return metric_type_; } Status IndexDesc::SetMetricType(VectorDB::MetricType metric_type) { metric_type_ = metric_type; return Status::OK(); } VectorDB::IndexType IndexDesc::IndexType() const { return index_type_; } Status IndexDesc::SetIndexType(VectorDB::IndexType index_type) { index_type_ = index_type; return Status::OK(); } const std::string IndexDesc::ExtraParams() const { return ::nlohmann::json(extra_params_).dump(); } Status IndexDesc::AddExtraParam(std::string key, int64_t value) { extra_params_[std::move(key)] = value; return Status::OK(); } Status IndexDesc::ExtraParamsFromJson(std::string json) { try { // extra_params_ = ::nlohmann::json::parse(std::move(json)); } catch (const ::nlohmann::json::exception& e) { return {StatusCode::JSON_PARSE_ERROR, e.what()}; } return Status::OK(); } Status IndexDesc::Validate() const { auto status = validate_index_and_metric(metric_type_, index_type_); if (status.IsOk()) { status = validate_params(*this, extra_params_); } return status; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/QueryArguments.cpp000066400000000000000000000052641520573221700251500ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/QueryArguments.h" namespace VectorDB { const std::string& QueryArguments::CollectionName() const { return collection_name_; } Status QueryArguments::SetCollectionName(std::string collection_name) { if (collection_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Collection name cannot be empty!"}; } collection_name_ = std::move(collection_name); return Status::OK(); } const std::set& QueryArguments::PartitionNames() const { return partition_names_; } Status QueryArguments::AddPartitionName(std::string partition_name) { if (partition_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Partition name cannot be empty!"}; } partition_names_.emplace(std::move(partition_name)); return Status::OK(); } const std::set& QueryArguments::OutputFields() const { return output_field_names_; } Status QueryArguments::AddOutputField(std::string field_name) { if (field_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Field name cannot be empty!"}; } output_field_names_.emplace(std::move(field_name)); return Status::OK(); } const std::string& QueryArguments::Expression() const { return filter_expression_; } Status QueryArguments::SetExpression(std::string expression) { if (expression.empty()) { return {StatusCode::INVALID_AGUMENT, "Filter expression cannot be empty!"}; } filter_expression_ = std::move(expression); return Status::OK(); } uint64_t QueryArguments::TravelTimestamp() const { return travel_timestamp_; } Status QueryArguments::SetTravelTimestamp(uint64_t timestamp) { travel_timestamp_ = timestamp; return Status::OK(); } uint64_t QueryArguments::GuaranteeTimestamp() const { return guarantee_timestamp_; } Status QueryArguments::SetGuaranteeTimestamp(uint64_t timestamp) { guarantee_timestamp_ = timestamp; return Status::OK(); } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/QueryResults.cpp000066400000000000000000000027611520573221700246430ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/QueryResults.h" namespace VectorDB { QueryResults::QueryResults() = default; QueryResults::QueryResults(const std::vector& output_fields) { output_fields_ = output_fields; } QueryResults::QueryResults(std::vector&& output_fields) { output_fields_ = std::move(output_fields); } FieldDataPtr QueryResults::GetFieldByName(const std::string& name) { for (FieldDataPtr& field : output_fields_) { if (nullptr == field) { continue; } if (field->Name() == name) { return field; } } return nullptr; } const std::vector& QueryResults::OutputFields() const { return output_fields_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/SearchArguments.cpp000066400000000000000000000201201520573221700252340ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/SearchArguments.h" #include #include namespace VectorDB { namespace { struct Validation { std::string param; int64_t min; int64_t max; bool required; Status Validate(const SearchArguments&, std::unordered_map params) const { auto it = params.find(param); if (it != params.end()) { auto value = it->second; if (value < min || value > max) { return {StatusCode::INVALID_AGUMENT, "invalid value: " + param + "=" + std::to_string(value) + ", requires [" + std::to_string(min) + ", " + std::to_string(max) + "]"}; } } return Status::OK(); } }; Status validate(const SearchArguments& data, const std::unordered_map& params) { auto status = Status::OK(); auto validations = { Validation{"nprobe", 1, 65536, false}, Validation{"ef", 1, 32768, false}, Validation{"search_k", -1, 65536, false}, }; for (const auto& validation : validations) { status = validation.Validate(data, params); if (!status.IsOk()) { return status; } } return status; } } // namespace const std::string& SearchArguments::CollectionName() const { return collection_name_; } Status SearchArguments::SetCollectionName(std::string collection_name) { if (collection_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Collection name cannot be empty!"}; } collection_name_ = std::move(collection_name); return Status::OK(); } const std::set& SearchArguments::PartitionNames() const { return partition_names_; } Status SearchArguments::AddPartitionName(std::string partition_name) { if (partition_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Partition name cannot be empty!"}; } partition_names_.emplace(std::move(partition_name)); return Status::OK(); } const std::set& SearchArguments::OutputFields() const { return output_field_names_; } Status SearchArguments::AddOutputField(std::string field_name) { if (field_name.empty()) { return {StatusCode::INVALID_AGUMENT, "Field name cannot be empty!"}; } output_field_names_.emplace(std::move(field_name)); return Status::OK(); } const std::string& SearchArguments::Expression() const { return filter_expression_; } Status SearchArguments::SetExpression(std::string expression) { filter_expression_ = std::move(expression); return Status::OK(); } FieldDataPtr SearchArguments::TargetVectors() const { if (binary_vectors_ != nullptr) { return binary_vectors_; } else if (float_vectors_ != nullptr) { return float_vectors_; } return nullptr; } Status SearchArguments::AddTargetVector(std::string field_name, const std::string& vector) { return AddTargetVector(std::move(field_name), std::string{vector}); } Status SearchArguments::AddTargetVector(std::string field_name, const std::vector& vector) { return AddTargetVector(std::move(field_name), BinaryVecFieldData::CreateBinaryString(vector)); } Status SearchArguments::AddTargetVector(std::string field_name, std::string&& vector) { if (float_vectors_ != nullptr) { return {StatusCode::INVALID_AGUMENT, "Target vector must be float type!"}; } if (nullptr == binary_vectors_) { binary_vectors_ = std::make_shared(std::move(field_name)); } auto code = binary_vectors_->Add(std::move(vector)); if (code != StatusCode::OK) { return {code, "Failed to add vector"}; } return Status::OK(); } Status SearchArguments::AddTargetVector(std::string field_name, const FloatVecFieldData::ElementT& vector) { if (binary_vectors_ != nullptr) { return {StatusCode::INVALID_AGUMENT, "Target vector must be binary type!"}; } if (nullptr == float_vectors_) { float_vectors_ = std::make_shared(std::move(field_name)); } auto code = float_vectors_->Add(vector); if (code != StatusCode::OK) { return {code, "Failed to add vector"}; } return Status::OK(); } Status SearchArguments::AddTargetVector(std::string field_name, FloatVecFieldData::ElementT&& vector) { if (binary_vectors_ != nullptr) { return {StatusCode::INVALID_AGUMENT, "Target vector must be binary type!"}; } if (nullptr == float_vectors_) { float_vectors_ = std::make_shared(std::move(field_name)); } auto code = float_vectors_->Add(std::move(vector)); if (code != StatusCode::OK) { return {code, "Failed to add vector"}; } return Status::OK(); } uint64_t SearchArguments::TravelTimestamp() const { return travel_timestamp_; } Status SearchArguments::SetTravelTimestamp(uint64_t timestamp) { travel_timestamp_ = timestamp; return Status::OK(); } uint64_t SearchArguments::GuaranteeTimestamp() const { return guarantee_timestamp_; } Status SearchArguments::SetGuaranteeTimestamp(uint64_t timestamp) { guarantee_timestamp_ = timestamp; return Status::OK(); } Status SearchArguments::SetTopK(int64_t topk) { topk_ = topk; return Status::OK(); } int64_t SearchArguments::TopK() const { return topk_; } int64_t SearchArguments::Nprobe() const { if (extra_params_.find("nprobe") != extra_params_.end()) { return extra_params_.at("nprobe"); } return 1; } Status SearchArguments::SetNprobe(int64_t nprobe) { extra_params_["nprobe"] = nprobe; return Status::OK(); } Status SearchArguments::SetRoundDecimal(int round_decimal) { round_decimal_ = round_decimal; return Status::OK(); } int SearchArguments::RoundDecimal() const { return round_decimal_; } Status SearchArguments::SetMetricType(VectorDB::MetricType metric_type) { if (((metric_type == MetricType::IP && metric_type_ == MetricType::L2) || (metric_type == MetricType::L2 && metric_type_ == MetricType::IP)) && range_search_) { // switch radius and range_filter std::swap(radius_, range_filter_); } metric_type_ = metric_type; return Status::OK(); } VectorDB::MetricType SearchArguments::MetricType() const { return metric_type_; } Status SearchArguments::AddExtraParam(std::string key, int64_t value) { extra_params_[std::move(key)] = value; return Status::OK(); } std::string SearchArguments::ExtraParams() const { return ::nlohmann::json(extra_params_).dump(); } Status SearchArguments::Validate() const { return validate(*this, extra_params_); } float SearchArguments::Radius() const { return radius_; } float SearchArguments::RangeFilter() const { return range_filter_; } Status SearchArguments::SetRange(float from, float to) { auto low = std::min(from, to); auto high = std::max(from, to); if (metric_type_ == MetricType::IP) { radius_ = low; range_filter_ = high; range_search_ = true; } else if (metric_type_ == MetricType::L2) { radius_ = high; range_filter_ = low; range_search_ = true; } else { return {StatusCode::INVALID_AGUMENT, "Metric type is not supported"}; } return Status::OK(); } bool SearchArguments::RangeSearch() const { return range_search_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/SearchResults.cpp000066400000000000000000000033571520573221700247450ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/SearchResults.h" namespace VectorDB { SingleResult::SingleResult(IDArray&& ids, std::vector&& scores, std::vector&& output_fields) : ids_{std::move(ids)}, scores_{std::move(scores)}, output_fields_{std::move(output_fields)} {} const std::vector& SingleResult::Scores() const { return scores_; } const IDArray& SingleResult::Ids() const { return ids_; } const std::vector& SingleResult::OutputFields() const { return output_fields_; } FieldDataPtr SingleResult::OutputField(const std::string& name) const { for (const auto& output_field : output_fields_) { if (output_field->Name() == name) { return output_field; } } return nullptr; } SearchResults::SearchResults() = default; SearchResults::SearchResults(std::vector&& results) { nq_results_.swap(results); } std::vector& SearchResults::Results() { return nq_results_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/Status.cpp000066400000000000000000000022471520573221700234360ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/Status.h" namespace VectorDB { Status::Status(StatusCode code, std::string msg) : code_(code), msg_(std::move(msg)) {} Status::Status() = default; Status Status::OK() { return {}; } bool Status::IsOk() const { return code_ == StatusCode::OK; } StatusCode Status::Code() const { return code_; } const std::string& Status::Message() const { return msg_; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/src/types/Utils.cpp000066400000000000000000000073431520573221700232550ustar00rootroot00000000000000// Licensed to the LF AI & Data foundation under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "types/Utils.h" namespace std { std::string to_string(VectorDB::MetricType metric_type) { switch (metric_type) { case VectorDB::MetricType::L2: return "L2"; case VectorDB::MetricType::IP: return "IP"; case VectorDB::MetricType::COSINE: return "COSINE"; case VectorDB::MetricType::HAMMING: return "HAMMING"; case VectorDB::MetricType::JACCARD: return "JACCARD"; case VectorDB::MetricType::TANIMOTO: return "TANIMOTO"; case VectorDB::MetricType::SUBSTRUCTURE: return "SUBSTRUCTURE"; case VectorDB::MetricType::SUPERSTRUCTURE: return "SUPERSTRUCTURE"; case VectorDB::MetricType::INVALID: default: return "INVALID"; } } std::string to_string(VectorDB::IndexType index_type) { switch (index_type) { case VectorDB::IndexType::FLAT: return "FLAT"; case VectorDB::IndexType::IVF_FLAT: return "IVF_FLAT"; case VectorDB::IndexType::IVF_PQ: return "IVF_PQ"; case VectorDB::IndexType::IVF_SQ8: return "IVF_SQ8"; case VectorDB::IndexType::HNSW: return "HNSW"; case VectorDB::IndexType::IVF_HNSW: return "IVF_HNSW"; case VectorDB::IndexType::RHNSW_FLAT: return "RHNSW_FLAT"; case VectorDB::IndexType::RHNSW_SQ: return "RHNSW_SQ"; case VectorDB::IndexType::RHNSW_PQ: return "RHNSW_PQ"; case VectorDB::IndexType::ANNOY: return "ANNOY"; case VectorDB::IndexType::BIN_FLAT: return "BIN_FLAT"; case VectorDB::IndexType::BIN_IVF_FLAT: return "BIN_IVF_FLAT"; default: return "INVALID"; } } } // namespace std namespace VectorDB { std::string Base64Encode(const std::string& val) { const char* base64_chars = { "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789" "+/"}; auto len = val.size(); auto len_encoded = (len + 2) / 3 * 4; std::string ret; ret.reserve(len_encoded); size_t pos = 0; while (pos < len) { ret.push_back(base64_chars[(val[pos + 0] & 0xfc) >> 2]); if (pos + 1 < len) { ret.push_back(base64_chars[((val[pos + 0] & 0x03) << 4) + ((val[pos + 1] & 0xf0) >> 4)]); if (pos + 2 < len) { ret.push_back(base64_chars[((val[pos + 1] & 0x0f) << 2) + ((val[pos + 2] & 0xc0) >> 6)]); ret.push_back(base64_chars[val[pos + 2] & 0x3f]); } else { ret.push_back(base64_chars[(val[pos + 1] & 0x0f) << 2]); ret.push_back('='); } } else { ret.push_back(base64_chars[(val[pos + 0] & 0x03) << 4]); ret.push_back('='); ret.push_back('='); } pos += 3; } return ret; } } // namespace VectorDB libkysdk-vector-engine-client-1.2.0.0/usr/000077500000000000000000000000001520573221700203205ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/usr/share/000077500000000000000000000000001520573221700214225ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/usr/share/pkgconfig/000077500000000000000000000000001520573221700233715ustar00rootroot00000000000000libkysdk-vector-engine-client-1.2.0.0/usr/share/pkgconfig/kysdk-vector-engine-client.pc000066400000000000000000000002611520573221700310600ustar00rootroot00000000000000Name: kysdk-vector-engine-client Description: Kylin AI Vector Database SDK Version: 1.2.0.0 Libs: -lkysdk-vector-engine-client Cflags: -I/usr/include/kysdk-vector-engine-client