VECOTR DATABASE Archives

[PYTHON/PYMILVUS] MilvusClient 클래스 : delete 메소드에서 collection_name/filter 인자를 사용해 엔터티 삭제하기

■ MilvusClient 클래스의 delete 메소드에서 collection_name/filter 인자를 사용해 엔터티를 삭제하는 방법을 보여준다. ▶ main.py


import numpy as np

from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):
    milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(
    collection_name = "temp",
    dimension       = 384
)

stringList = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList = [[ np.random.uniform(-1, 1) for _ in range(384) ] for _ in range(len(stringList)) ] # NDArray list

itemList = [{"id" : i, "vector" : stringVectorList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(stringVectorList))]

milvusClient.insert(
    collection_name = "temp",
    data            = itemList
)

extraList1 = milvusClient.search(
    collection_name = "temp",
    data            = [stringVectorList[0]],
    filter          = "subject == 'history'",
    limit           = 2,
    output_fields   = ["text", "subject"]
)

print(extraList1[0][0])
print(extraList1[0][1])
print("-" * 100)

extraList2 = milvusClient.query(
    collection_name = "temp",
    filter          = "subject == 'history'",
    output_fields   = ["text", "subject"]
)

print(extraList2[0])
print(extraList2[1])
print(extraList2[2])
print("-" * 100)

deletedIDList = milvusClient.delete(
    collection_name = "temp",
    filter          = "subject == 'history'"
) # int list

print(deletedIDList)
print("-" * 100)

"""
{'id': 0, 'distance': 1.0000003576278687, 'entity': {'text': 'Artificial intelligence was founded as an academic discipline in 1956.', 'subject': 'history'}}
{'id': 2, 'distance': 0.09245504438877106, 'entity': {'text': 'Born in Maida Vale, London, Turing was raised in southern England.', 'subject': 'history'}}

import numpy as np

from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):

milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(

collection_name = "temp",

dimension = 384

)

stringList = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList = [[ np.random.uniform(-1, 1) for _ in range(384) ] for _ in range(len(stringList)) ] # NDArray list

itemList = [{"id" : i, "vector" : stringVectorList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(stringVectorList))]

milvusClient.insert(

collection_name = "temp",

data = itemList

)

extraList1 = milvusClient.search(

collection_name = "temp",

data = [stringVectorList[0]],

filter = "subject == 'history'",

limit = 2,

output_fields = ["text", "subject"]

)

print(extraList1[0][0])

print(extraList1[0][1])

print("-" * 100)

extraList2 = milvusClient.query(

collection_name = "temp",

filter = "subject == 'history'",

output_fields = ["text", "subject"]

)

print(extraList2[0])

print(extraList2[1])

print(extraList2[2])

print("-" * 100)

deletedIDList = milvusClient.delete(

collection_name = "temp",

filter = "subject == 'history'"

) # int list

print(deletedIDList)

print("-" * 100)

"""

{'id': 0, 'distance': 1.0000003576278687, 'entity': {'text': 'Artificial intelligence was founded as an academic discipline in 1956.', 'subject': 'history'}}

{'id': 2, 'distance': 0.09245504438877106, 'entity': {'text': 'Born in Maida Vale, London, Turing was raised in southern England.', 'subject': 'history'}}

{'id': 0, 'text': 'Artificial intelligence was founded

[PYTHON/PYMILVUS] MilvusClient 클래스 : 생성자에서 uri/token 인자를 사용해 MilvusClient 객체 만들기

■ MilvusClient 클래스의 생성자에서 uri/token 인자를 사용해 MilvusClient 객체를 만드는 방법을 보여준다. ※ Milvus Lite는 로컬 파이썬 프로그램을 시작하기에 좋다. ※ 대규모

[PYTHON/PYMILVUS] MilvusClient 클래스 : delete 메소드에서 collection_name/ids 인자를 사용해 엔터티 삭제하기

■ MilvusClient 클래스의 delete 메소드에서 collection_name/ids 인자를 사용해 엔터티를 삭제하는 방법을 보여준다. ▶ main.py


from pymilvus import MilvusClient
from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):
    milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(
    collection_name = "temp",
    dimension       = 768
)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList1 = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList1 = onnxEmbeddingFunction.encode_documents(stringList1) # NDArray list

itemList = []

itemList.extend(
    [
        {"id" : i, "vector" : stringVectorList1[i], "text" : stringList1[i], "subject" : "history"} for i in range(len(stringVectorList1))
    ]
)

stringList2 = [
    "Machine learning has been used for drug design.",
    "Computational synthesis with AI algorithms predicts molecular properties.",
    "DDR1 is involved in cancers and fibrosis.",
]

stringVectorList2 = onnxEmbeddingFunction.encode_documents(stringList2) # NDArray list

itemList.extend(
    [
        {"id" : 3 + i, "vector" : stringVectorList2[i], "text" : stringList2[i], "subject" : "biology"} for i in range(len(stringVectorList2))
    ]
)

milvusClient.insert(collection_name = "temp", data = itemList)

deletedIDList = milvusClient.delete(collection_name = "temp", ids = [0, 2]) # int list

print(deletedIDList)

"""
[0, 2]
"""

from pymilvus import MilvusClient

from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):

milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(

collection_name = "temp",

dimension = 768

)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList1 = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList1 = onnxEmbeddingFunction.encode_documents(stringList1) # NDArray list

itemList = []

itemList.extend(

[

{"id" : i, "vector" : stringVectorList1[i], "text" : stringList1[i], "subject" : "history"} for i in range(len(stringVectorList1))

]

)

stringList2 = [

"Machine learning has been used for drug design.",

"Computational synthesis with AI algorithms predicts molecular properties.",

"DDR1 is involved in cancers and fibrosis.",

]

stringVectorList2 = onnxEmbeddingFunction.encode_documents(stringList2) # NDArray list

itemList.extend(

[

{"id" : 3 + i, "vector" : stringVectorList2[i], "text" : stringList2[i], "subject" : "biology"} for i in range(len(stringVectorList2))

]

)

milvusClient.insert(collection_name = "temp", data = itemList)

deletedIDList = milvusClient.delete(collection_name = "temp", ids = [0, 2]) # int list

print(deletedIDList)

"""

[0, 2]

"""

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model]

[PYTHON/PYMILVUS] MilvusClient 클래스 : query 메소드에서 ids 인자를 사용해 쿼리하기

■ MilvusClient 클래스의 query 메소드에서 ids 인자를 사용해 쿼리하는 방법을 보여준다. ※ 기본 키로 엔터티를 직접 검색한다. ▶ main.py


from pymilvus import MilvusClient
from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):
    milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(
    collection_name = "temp",
    dimension       = 768
)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList1 = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList1 = onnxEmbeddingFunction.encode_documents(stringList1) # NDArray List

itemList = []

itemList.extend(
    [
        {"id" : i, "vector" : stringVectorList1[i], "text" : stringList1[i], "subject" : "history"} for i in range(len(stringVectorList1))
    ]
)

stringList2 = [
    "Machine learning has been used for drug design.",
    "Computational synthesis with AI algorithms predicts molecular properties.",
    "DDR1 is involved in cancers and fibrosis.",
]

stringVectorList2 = onnxEmbeddingFunction.encode_documents(stringList2) # NDArray list

itemList.extend(
    [
        {"id" : 3 + i, "vector" : stringVectorList2[i], "text" : stringList2[i], "subject" : "biology"} for i in range(len(stringVectorList2))
    ]
)

milvusClient.insert(collection_name = "temp", data = itemList)

extraList = milvusClient.query(
    collection_name = "temp",
    ids             = [0, 2],
    output_fields   = ["text", "subject"]
)

print(extraList[0])
print(extraList[1])

"""
{'id' : 0, 'text' : 'Artificial intelligence was founded as an academic discipline in 1956.', 'subject' : 'history'}
{'id' : 2, 'text' : 'Born in Maida Vale, London, Turing was raised in southern England.'    , 'subject' : 'history'}
"""

from pymilvus import MilvusClient

from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

if milvusClient.has_collection(collection_name= "temp"):

milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(

collection_name = "temp",

dimension = 768

)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList1 = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList1 = onnxEmbeddingFunction.encode_documents(stringList1) # NDArray List

itemList = []

itemList.extend(

[

{"id" : i, "vector" : stringVectorList1[i], "text" : stringList1[i], "subject" : "history"} for i in range(len(stringVectorList1))

]

)

stringList2 = [

"Machine learning has been used for drug design.",

"Computational synthesis with AI algorithms predicts molecular properties.",

"DDR1 is involved in cancers and fibrosis.",

]

stringVectorList2 = onnxEmbeddingFunction.encode_documents(stringList2) # NDArray list

itemList.extend(

[

{"id" : 3 + i, "vector" : stringVectorList2[i], "text" : stringList2[i], "subject" : "biology"} for i in range(len(stringVectorList2))

]

)

milvusClient.insert(collection_name = "temp", data = itemList)

extraList = milvusClient.query(

collection_name = "temp",

ids = [0, 2],

output_fields = ["text", "subject"]

)

print(extraList[0])

print(extraList[1])

"""

{'id' : 0, 'text' : 'Artificial intelligence was founded as an academic discipline in 1956.', 'subject' : 'history'}

{'id' : 2, 'text' : 'Born in Maida Vale, London, Turing was raised in southern England.' , 'subject' : 'history'}

"""

▶ requirements.txt

[PYTHON/PYMILVUS] MilvusClient 클래스 : query 메소드에서 collection_name/filter/output_fields 인자를 사용해 쿼리하기

■ MilvusClient 클래스의 query 메소드에서 collection_name/filter/output_fields 인자를 사용해 쿼리하는 방법을 보여준다. ※ 필터 표현식이나 일부 ID와 일치하는 크레트리아와 일치하는 모든 엔터티를 검색하는

[PYTHON/PYMILVUS] MilvusClient 클래스 : search 메소드에서 collection_name/data/limit/output_fields 인자를 사용해 벡터 검색하기

■ MilvusClient 클래스의 search 메소드에서 collection_name/data/limit/output_fields 인자를 사용해 벡터를 검색하는 방법을 보여준다. ※ Milvus는 동시에 하나 또는 여러 개의 벡터 검색 요청을

[PYTHON/PYMILVUS] MilvusClient 클래스 : insert 메소드에서 collection_name/data 인자를 사용해 데이터 삽입하기

■ MilvusClient 클래스의 insert 메소드에서 collection_name/data 인자를 사용해 데이터를 삽입하는 방법을 보여준다. ▶ main.py


from pymilvus import MilvusClient
from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = " temp")

if milvusClient.has_collection(collection_name= "temp"):
    milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(
    collection_name = "temp",
    dimension       = 768
)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

ndArrayList = onnxEmbeddingFunction.encode_documents(stringList)

itemList = [
    {"id" : i, "vector" : ndArrayList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(ndArrayList))
]

omitZeroDict = milvusClient.insert(collection_name = "temp", data = itemList)

print(omitZeroDict)

"""
{'insert_count': 3, 'ids': [0, 1, 2]}
"""

from pymilvus import MilvusClient

from pymilvus import model

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = " temp")

if milvusClient.has_collection(collection_name= "temp"):

milvusClient.drop_collection(collection_name = "temp")

milvusClient.create_collection(

collection_name = "temp",

dimension = 768

)

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

ndArrayList = onnxEmbeddingFunction.encode_documents(stringList)

itemList = [

{"id" : i, "vector" : ndArrayList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(ndArrayList))

]

omitZeroDict = milvusClient.insert(collection_name = "temp", data = itemList)

print(omitZeroDict)

"""

{'insert_count': 3, 'ids': [0, 1, 2]}

"""

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model]

[PYTHON/PYMILVUS] OnnxEmbeddingFunction 클래스 : encode_queries 메소드를 사용해 쿼리 벡터 리스트 만들기

■ OnnxEmbeddingFunction 클래스의 encode_queries 메소드를 사용해 쿼리 벡터 리스트를 만드는 방법을 보여준다. ▶ main.py


from pymilvus import model

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

queryVectorList = onnxEmbeddingFunction.encode_queries(["Who is Alan Turing?"]) # NDArray list

from pymilvus import model

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

queryVectorList = onnxEmbeddingFunction.encode_queries(["Who is Alan Turing?"]) # NDArray list

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model]

[PYTHON/PYMILVUS] 임의 벡터로 텍스트 표현하기

■ 임의 벡터로 텍스트를 표현하는 방법을 보여준다. ※ 네트워크 문제로 인해 모델을 다운로드할 수 없는 경우, 임시 방편으로 임의의 벡터를 사용하여 텍스트를

[PYTHON/PYMILVUS] 벡터로 텍스트 표현하기

■ 벡터로 텍스트를 표현하는 방법을 보여준다. ▶ main.py


from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.
# import os
# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.
onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

itemList = [
    {"id" : i, "vector" : stringVectorList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(stringVectorList))
]

print("Data has", len(itemList), "entities, each with fields : ", itemList[0].keys())
print("Vector dim :", len(itemList[0]["vector"]))

"""
Data has 3 entities, each with fields :  dict_keys(['id', 'vector', 'text', 'subject'])
Vector dim : 768
"""

from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.

# import os

# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

itemList = [

{"id" : i, "vector" : stringVectorList[i], "text" : stringList[i], "subject" : "history"} for i in range(len(stringVectorList))

]

print("Data has", len(itemList), "entities, each with fields : ", itemList[0].keys())

print("Vector dim :", len(itemList[0]["vector"]))

"""

Data has 3 entities, each with fields : dict_keys(['id', 'vector', 'text', 'subject'])

Vector dim : 768

"""

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model] 명령을 실행했다.

[PYTHON/PYMILVUS] OnnxEmbeddingFunction 클래스 : dim 속성을 사용해 차원 구하기

■ OnnxEmbeddingFunction 클래스의 dim 속성을 사용해 차원을 구하는 방법을 보여준다. ▶ main.py


from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.
# import os
# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.
onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

print("차원 :", onnxEmbeddingFunction.dim, stringVectorList[0].shape)  # Dim: 768 (768,)

"""
차원 : 768 (768,)
"""

from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.

# import os

# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

print("차원 :", onnxEmbeddingFunction.dim, stringVectorList[0].shape) # Dim: 768 (768,)

"""

차원 : 768 (768,)

"""

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model] 명령을 실행했다.

[PYTHON/PYMILVUS] OnnxEmbeddingFunction 클래스 : encode_documents 메소드를 사용해 벡터 리스트 만들기

■ OnnxEmbeddingFunction 클래스의 encode_documents 메소드를 사용해 벡터 리스트를 만드는 방법을 보여준다. ▶ main.py


from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.
# import os
# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.
onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [
    "Artificial intelligence was founded as an academic discipline in 1956.",
    "Alan Turing was the first person to conduct substantial research in AI.",
    "Born in Maida Vale, London, Turing was raised in southern England."
]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

print(len(stringVectorList))

"""
3
"""

from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.

# import os

# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

stringList = [

"Artificial intelligence was founded as an academic discipline in 1956.",

"Alan Turing was the first person to conduct substantial research in AI.",

"Born in Maida Vale, London, Turing was raised in southern England."

]

stringVectorList = onnxEmbeddingFunction.encode_documents(stringList) # NDArray list

print(len(stringVectorList))

"""

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model] 명령을

[PYTHON/PYMILVUS] DefaultEmbeddingFunction 함수 : OnnxEmbeddingFunction 객체 만들기

■ DefaultEmbeddingFunction 함수를 사용해 OnnxEmbeddingFunction 객체를 만드는 방법을 보여준다. ▶ main.py


from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.
# import os
# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.
onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

from pymilvus import model

# https://huggingface.co/에 연결이 실패한 경우 다음 경로의 주석 처리를 제거한다.

# import os

# os.environ["HF_ENDPOINT":"] = "https://hf-mirror.com"

# 작은 임베딩 모델 "paraphrase-albert-small-v2"(~50MB)이 다운로드된다.

onnxEmbeddingFunction = model.DefaultEmbeddingFunction()

▶ requirements.txt


certifi==2024.8.30
charset-normalizer==3.3.2
coloredlogs==15.0.1
environs==9.5.0
filelock==3.16.1
flatbuffers==24.3.25
fsspec==2024.9.0
grpcio==1.66.2
huggingface-hub==0.25.1
humanfriendly==10.0
idna==3.10
marshmallow==3.22.0
milvus-lite==2.4.10
milvus-model==0.2.7
mpmath==1.3.0
numpy==2.1.2
onnxruntime==1.19.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
safetensors==0.4.5
scipy==1.14.1
six==1.16.0
sympy==1.13.3
tokenizers==0.20.0
tqdm==4.66.5
transformers==4.45.1
typing_extensions==4.12.2
tzdata==2024.2
ujson==5.10.0
urllib3==2.2.3

certifi==2024.8.30

charset-normalizer==3.3.2

coloredlogs==15.0.1

environs==9.5.0

filelock==3.16.1

flatbuffers==24.3.25

fsspec==2024.9.0

grpcio==1.66.2

huggingface-hub==0.25.1

humanfriendly==10.0

idna==3.10

marshmallow==3.22.0

milvus-lite==2.4.10

milvus-model==0.2.7

mpmath==1.3.0

numpy==2.1.2

onnxruntime==1.19.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

PyYAML==6.0.2

regex==2024.9.11

requests==2.32.3

safetensors==0.4.5

scipy==1.14.1

six==1.16.0

sympy==1.13.3

tokenizers==0.20.0

tqdm==4.66.5

transformers==4.45.1

typing_extensions==4.12.2

tzdata==2024.2

ujson==5.10.0

urllib3==2.2.3

※ pip install pymilvus[model] 명령을 실행했다.

[PYTHON/PYMILVUS] MilvusClient 클래스 : create_collection 메소드에서 collection_name/dimension 인자를 사용해 컬렉션 만들기

■ MilvusClient 클래스의 create_collection 메소드에서 collection_name/dimension 인자를 사용해 컬렉션을 만드는 방법을 보여준다. ※ 기본 키와 벡터 필드는 기본 이름("id" 및 "vector")을 사용한다.

[PYTHON/PYMILVUS] MilvusClient 클래스 : drop_collection 메소드에서 collection_name 인자를 사용해 컬렉션 삭제하기

■ MilvusClient 클래스의 drop_collection 메소드에서 collection_name 인자를 사용해 컬렉션을 삭제하는 방법을 보여준다. ▶ main.py


from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = " temp")

if milvusClient.has_collection(collection_name= "temp"):
    milvusClient.drop_collection(collection_name = "temp")

from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = " temp")

if milvusClient.has_collection(collection_name= "temp"):

milvusClient.drop_collection(collection_name = "temp")

▶ requirements.txt


environs==9.5.0
grpcio==1.66.2
marshmallow==3.22.0
milvus-lite==2.4.10
numpy==2.1.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
six==1.16.0
tqdm==4.66.5
tzdata==2024.2
ujson==5.10.0

environs==9.5.0

grpcio==1.66.2

marshmallow==3.22.0

milvus-lite==2.4.10

numpy==2.1.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

six==1.16.0

tqdm==4.66.5

tzdata==2024.2

ujson==5.10.0

※ pip install pymilvus

[PYTHON/PYMILVUS] MilvusClient 클래스 : has_collection 메소드에서 collection_name 인자를 사용해 컬렉션 존재 여부 구하기

■ MilvusClient 클래스의 has_collection 메소드에서 collection_name 인자를 사용해 컬렉션 존재 여부를 구하는 방법을 보여준다. ▶ main.py


from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

print(hasCollection)

from pymilvus import MilvusClient

milvusClient = MilvusClient("test.db")

hasCollection = milvusClient.has_collection(collection_name = "temp")

print(hasCollection)

▶ requirements.txt


environs==9.5.0
grpcio==1.66.2
marshmallow==3.22.0
milvus-lite==2.4.10
numpy==2.1.2
packaging==24.1
pandas==2.2.3
protobuf==5.28.2
pymilvus==2.4.7
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
six==1.16.0
tqdm==4.66.5
tzdata==2024.2
ujson==5.10.0

environs==9.5.0

grpcio==1.66.2

marshmallow==3.22.0

milvus-lite==2.4.10

numpy==2.1.2

packaging==24.1

pandas==2.2.3

protobuf==5.28.2

pymilvus==2.4.7

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

pytz==2024.2

six==1.16.0

tqdm==4.66.5

tzdata==2024.2

ujson==5.10.0

※ pip

[PYTHON/PYMILVUS] MilvusClient 클래스 : 생성자를 사용해 MilvusClient 객체 만들기

■ MilvusClient 클래스의 생성자를 사용해 MilvusClient 객체를 만드는 방법을 보여준다. ※ 로컬 Milvus 벡터 데이터베이스를 만들려면 "test.db"와 같이 모든 데이터를 저장할 파일명을