LLAMA-INDEX Archives - icodebroker

[PYTHON/LLAMA-INDEX] FAISS 벡터 데이터베이스를 사용해 질의 응답하기

■ FAISS 벡터 데이터베이스를 사용해 질의 응답하는 방법을 보여준다. ▶ main.py


import os

import faiss

from llama_index.core                import SimpleDirectoryReader, Settings, GPTVectorStoreIndex
from llama_index.vector_stores.faiss import FaissVectorStore
from llama_index.llms.openai         import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm          = OpenAI(model = "gpt-3.5-turbo", temperature = 0.1)
Settings.vector_store = FaissVectorStore(faiss_index = faiss.IndexFlatL2(1536))

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

responsea = retrieverQueryEngine.query("미코의 열정은? 한국어로")

print(responsea)

"""
미코의 열정은 네오 도쿄를 더 나은 도시로 바꾸어 나가는 것에 대한 다짐으로 나타납니다.
"""

import os

import faiss

from llama_index.core import SimpleDirectoryReader, Settings, GPTVectorStoreIndex

from llama_index.vector_stores.faiss import FaissVectorStore

from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(model = "gpt-3.5-turbo", temperature = 0.1)

Settings.vector_store = FaissVectorStore(faiss_index = faiss.IndexFlatL2(1536))

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

responsea = retrieverQueryEngine.query("미코의 열정은? 한국어로")

print(responsea)

"""

미코의 열정은 네오 도쿄를 더 나은 도시로 바꾸어 나가는 것에 대한 다짐으로 나타납니다.

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
faiss-gpu==1.7.2
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpointer==2.4
langchain==0.2.3
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.75
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-index-vector-stores-faiss==0.1.2
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.33.0
orjson==3.10.3
packaging==23.2
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.2
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

faiss-gpu==1.7.2

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpointer==2.4

langchain==0.2.3

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.75

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-index-vector-stores-faiss==0.1.2

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.33.0

orjson==3.10.3

packaging==23.2

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.2

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

※ pip install openai langchain llama-index faiss-gpu

[PYTHON/LLAMA-INDEX] YoutubeTranscriptReader 클래스 : load_data 메소드를 사용해 유튜브 동영상에서 자막을 가져오는 데이터 커넥터 설정하기

■ YoutubeTranscriptReader 클래스의 load_data 메소드를 사용해 유튜브 동영상에서 자막을 가져오는 데이터 커넥터를 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

YoutubeTranscriptReader = download_loader("YoutubeTranscriptReader")

youtubeTranscriptReader = YoutubeTranscriptReader()

documentList = youtubeTranscriptReader.load_data(ytlinks = ["https://www.youtube.com/watch?v=oc6RV5c1yd0"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

responsea = retrieverQueryEngine.query("이 동영상에서 전하고 싶은 말은 무엇인가요? 한국어로 대답해 주세요.")

print(responsea)

import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

YoutubeTranscriptReader = download_loader("YoutubeTranscriptReader")

youtubeTranscriptReader = YoutubeTranscriptReader()

documentList = youtubeTranscriptReader.load_data(ytlinks = ["https://www.youtube.com/watch?v=oc6RV5c1yd0"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

responsea = retrieverQueryEngine.query("이 동영상에서 전하고 싶은 말은 무엇인가요? 한국어로 대답해 주세요.")

print(responsea)

▶ 실행 결과


Requirement already satisfied: llama-index-readers-youtube-transcript in ./env/lib/python3.10/site-packages (0.1.4)
Requirement already satisfied: youtube-transcript-api>=0.5.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-youtube-transcript) (0.6.2)
Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-youtube-transcript) (0.10.43)
Requirement already satisfied: numpy in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.26.4)
Requirement already satisfied: nltk<4.0.0,>=3.8.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.8.1)
Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (8.3.0)
Requirement already satisfied: typing-inspect>=0.8.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.9.0)
Requirement already satisfied: requests>=2.31.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.32.3)
Requirement already satisfied: pillow>=9.0.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (10.3.0)
Requirement already satisfied: pandas in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.2.2)
Requirement already satisfied: fsspec>=2023.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.6.0)
Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.66.4)
Requirement already satisfied: PyYAML>=6.0.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (6.0.1)
Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.9.5)
Requirement already satisfied: wrapt in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.16.0)
Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.18 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.1.19)
Requirement already satisfied: typing-extensions>=4.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.12.2)
Requirement already satisfied: deprecated>=1.2.9.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.2.14)
Requirement already satisfied: httpx in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.27.0)
Requirement already satisfied: networkx>=3.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.3)
Requirement already satisfied: tiktoken>=0.3.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.7.0)
Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.0.30)
Requirement already satisfied: dataclasses-json in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.6.6)
Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.6.0)
Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.8)
Requirement already satisfied: openai>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.33.0)
Requirement already satisfied: multidict<7.0,>=4.5 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (6.0.5)
Requirement already satisfied: aiosignal>=1.1.2 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.3.1)
Requirement already satisfied: frozenlist>=1.1.1 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.4.1)
Requirement already satisfied: yarl<2.0,>=1.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.9.4)
Requirement already satisfied: async-timeout<5.0,>=4.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.0.3)
Requirement already satisfied: attrs>=17.3.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (23.2.0)
Requirement already satisfied: pydantic>=1.10 in ./env/lib/python3.10/site-packages (from llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.7.3)
Requirement already satisfied: sniffio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.3.1)
Requirement already satisfied: idna in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.7)
Requirement already satisfied: certifi in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.6.2)
Requirement already satisfied: anyio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.4.0)
Requirement already satisfied: httpcore==1.* in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.5)
Requirement already satisfied: h11<0.15,>=0.13 in ./env/lib/python3.10/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.14.0)
Requirement already satisfied: regex>=2021.8.3 in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.5.15)
Requirement already satisfied: click in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (8.1.7)
Requirement already satisfied: joblib in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.4.2)
Requirement already satisfied: distro<2,>=1.7.0 in ./env/lib/python3.10/site-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.9.0)
Requirement already satisfied: urllib3<3,>=1.21.1 in ./env/lib/python3.10/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.2.1)
Requirement already satisfied: charset-normalizer<4,>=2 in ./env/lib/python3.10/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.3.2)
Requirement already satisfied: greenlet!=0.4.17 in ./env/lib/python3.10/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.0.3)
Requirement already satisfied: mypy-extensions>=0.3.0 in ./env/lib/python3.10/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.0)
Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in ./env/lib/python3.10/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.21.3)
Requirement already satisfied: python-dateutil>=2.8.2 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.9.0.post0)
Requirement already satisfied: pytz>=2020.1 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.1)
Requirement already satisfied: tzdata>=2022.7 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.1)
Requirement already satisfied: exceptiongroup>=1.0.2 in ./env/lib/python3.10/site-packages (from anyio->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.2.1)
Requirement already satisfied: packaging>=17.0 in ./env/lib/python3.10/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (23.2)
Requirement already satisfied: annotated-types>=0.4.0 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.7.0)
Requirement already satisfied: pydantic-core==2.18.4 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.18.4)
Requirement already satisfied: six>=1.5 in ./env/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.16.0)
이 동영상에서 전하고 싶은 메시지는 GPT-4가 OpenAI에서 개발된 최신 AI 시스템이며, 문제 해결 능력에서의 엄청난 발전을 이루었다는 것입니다. 또한 GPT-4는 다양한 작업을 수행할 수 있으며, 사용자의 요구에 더 안전하고 맞춰진 방식으로 개발되었습니다.

Requirement already satisfied: llama-index-readers-youtube-transcript in ./env/lib/python3.10/site-packages (0.1.4)

Requirement already satisfied: youtube-transcript-api>=0.5.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-youtube-transcript) (0.6.2)

Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-youtube-transcript) (0.10.43)

Requirement already satisfied: numpy in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.26.4)

Requirement already satisfied: nltk<4.0.0,>=3.8.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.8.1)

Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (8.3.0)

Requirement already satisfied: typing-inspect>=0.8.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.9.0)

Requirement already satisfied: requests>=2.31.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.32.3)

Requirement already satisfied: pillow>=9.0.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (10.3.0)

Requirement already satisfied: pandas in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.2.2)

Requirement already satisfied: fsspec>=2023.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.6.0)

Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.66.4)

Requirement already satisfied: PyYAML>=6.0.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (6.0.1)

Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.9.5)

Requirement already satisfied: wrapt in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.16.0)

Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.18 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.1.19)

Requirement already satisfied: typing-extensions>=4.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.12.2)

Requirement already satisfied: deprecated>=1.2.9.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.2.14)

Requirement already satisfied: httpx in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.27.0)

Requirement already satisfied: networkx>=3.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.3)

Requirement already satisfied: tiktoken>=0.3.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.7.0)

Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.0.30)

Requirement already satisfied: dataclasses-json in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.6.6)

Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.6.0)

Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.8)

Requirement already satisfied: openai>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.33.0)

Requirement already satisfied: multidict<7.0,>=4.5 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (6.0.5)

Requirement already satisfied: aiosignal>=1.1.2 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.3.1)

Requirement already satisfied: frozenlist>=1.1.1 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.4.1)

Requirement already satisfied: yarl<2.0,>=1.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.9.4)

Requirement already satisfied: async-timeout<5.0,>=4.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.0.3)

Requirement already satisfied: attrs>=17.3.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (23.2.0)

Requirement already satisfied: pydantic>=1.10 in ./env/lib/python3.10/site-packages (from llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.7.3)

Requirement already satisfied: sniffio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.3.1)

Requirement already satisfied: idna in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.7)

Requirement already satisfied: certifi in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.6.2)

Requirement already satisfied: anyio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (4.4.0)

Requirement already satisfied: httpcore==1.* in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.5)

Requirement already satisfied: h11<0.15,>=0.13 in ./env/lib/python3.10/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.14.0)

Requirement already satisfied: regex>=2021.8.3 in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.5.15)

Requirement already satisfied: click in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (8.1.7)

Requirement already satisfied: joblib in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.4.2)

Requirement already satisfied: distro<2,>=1.7.0 in ./env/lib/python3.10/site-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.9.0)

Requirement already satisfied: urllib3<3,>=1.21.1 in ./env/lib/python3.10/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.2.1)

Requirement already satisfied: charset-normalizer<4,>=2 in ./env/lib/python3.10/site-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.3.2)

Requirement already satisfied: greenlet!=0.4.17 in ./env/lib/python3.10/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.0.3)

Requirement already satisfied: mypy-extensions>=0.3.0 in ./env/lib/python3.10/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.0.0)

Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in ./env/lib/python3.10/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (3.21.3)

Requirement already satisfied: python-dateutil>=2.8.2 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.9.0.post0)

Requirement already satisfied: pytz>=2020.1 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.1)

Requirement already satisfied: tzdata>=2022.7 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2024.1)

Requirement already satisfied: exceptiongroup>=1.0.2 in ./env/lib/python3.10/site-packages (from anyio->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.2.1)

Requirement already satisfied: packaging>=17.0 in ./env/lib/python3.10/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (23.2)

Requirement already satisfied: annotated-types>=0.4.0 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (0.7.0)

Requirement already satisfied: pydantic-core==2.18.4 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (2.18.4)

Requirement already satisfied: six>=1.5 in ./env/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-youtube-transcript) (1.16.0)

이 동영상에서 전하고 싶은 메시지는 GPT-4가 OpenAI에서 개발된 최신 AI 시스템이며, 문제 해결 능력에서의 엄청난 발전을 이루었다는 것입니다. 또한 GPT-4는 다양한 작업을 수행할 수 있으며, 사용자의 요구에 더 안전하고 맞춰진 방식으로 개발되었습니다.

[PYTHON/LLAMA-INDEX] VectorStoreIndex 클래스 : as_query_engine 메소드를 사용해 RetrieverQueryEngine 객체 만들기

■ VectorStoreIndex 클래스의 as_query_engine 메소드를 사용해 RetrieverQueryEngine 객체를 만드는 방법을 보여준다. ▶ main.py


import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")

beautifulSoupWebReader = BeautifulSoupWebReader()

documentList = beautifulSoupWebReader.load_data(urls = ["https://openai.com/blog/planning-for-agi-and-beyond"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")

beautifulSoupWebReader = BeautifulSoupWebReader()

documentList = beautifulSoupWebReader.load_data(urls = ["https://openai.com/blog/planning-for-agi-and-beyond"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
chromedriver-autoinstaller==0.6.4
click==8.1.7
cssselect==1.2.0
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
feedfinder2==0.0.4
feedparser==6.0.11
filelock==3.14.0
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
html2text==2020.1.16
httpcore==1.0.5
httpx==0.27.0
idna==3.7
jieba3k==0.35.1
joblib==1.4.2
jsonpatch==1.33
jsonpointer==2.4
langchain==0.2.3
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.75
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-index-readers-web==0.1.18
llama-parse==0.4.4
llamaindex-py-client==0.1.19
lxml==5.2.2
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
newspaper3k==0.2.8
nltk==3.8.1
numpy==1.26.4
openai==1.33.0
orjson==3.10.3
outcome==1.3.0.post0
packaging==23.2
pandas==2.2.2
pillow==10.3.0
playwright==1.44.0
pydantic==2.7.3
pydantic_core==2.18.4
pyee==11.1.0
pypdf==4.2.0
PySocks==1.7.1
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
requests-file==2.1.0
selenium==4.21.0
sgmllib3k==1.0.0
six==1.16.0
sniffio==1.3.1
sortedcontainers==2.4.0
soupsieve==2.5
spider-client==0.0.11
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tinysegmenter==0.3
tldextract==5.1.2
tqdm==4.66.4
trio==0.25.1
trio-websocket==0.11.1
typing-inspect==0.9.0
typing_extensions==4.12.2
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
wsproto==1.2.0
yarl==1.9.4

100

101

102

103

104

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

chromedriver-autoinstaller==0.6.4

click==8.1.7

cssselect==1.2.0

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

feedfinder2==0.0.4

feedparser==6.0.11

filelock==3.14.0

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

html2text==2020.1.16

httpcore==1.0.5

httpx==0.27.0

idna==3.7

jieba3k==0.35.1

joblib==1.4.2

jsonpatch==1.33

jsonpointer==2.4

langchain==0.2.3

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.75

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-index-readers-web==0.1.18

llama-parse==0.4.4

llamaindex-py-client==0.1.19

lxml==5.2.2

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

newspaper3k==0.2.8

nltk==3.8.1

numpy==1.26.4

openai==1.33.0

orjson==3.10.3

outcome==1.3.0.post0

packaging==23.2

pandas==2.2.2

pillow==10.3.0

playwright==1.44.0

pydantic==2.7.3

pydantic_core==2.18.4

pyee==11.1.0

pypdf==4.2.0

PySocks==1.7.1

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

requests-file==2.1.0

selenium==4.21.0

sgmllib3k==1.0.0

six==1.16.0

sniffio==1.3.1

sortedcontainers==2.4.0

soupsieve==2.5

spider-client==0.0.11

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tinysegmenter==0.3

tldextract==5.1.2

tqdm==4.66.4

trio==0.25.1

trio-websocket==0.11.1

typing-inspect==0.9.0

typing_extensions==4.12.2

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

wsproto==1.2.0

yarl==1.9.4

※ pip install openai langchain

[PYTHON/LLAMA-INDEX] BeautifulSoupWebReader 클래스 : load_data 메소드를 사용해 웹 사이트에서 텍스트를 가져오는 데이터 커넥터 설정하기

■ BeautifulSoupWebReader 클래스의 load_data 메소드를 사용해 웹 사이트에서 텍스트를 가져오는 데이터 커넥터를 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")

beautifulSoupWebReader = BeautifulSoupWebReader()

documentList = beautifulSoupWebReader.load_data(urls = ["https://openai.com/blog/planning-for-agi-and-beyond"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

response = retrieverQueryEngine.query("이 웹페이지에서 전하고 싶은 말은 무엇인가요? 한국어로 대답해 주세요.")

print(response)

import os

from llama_index.core import download_loader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")

beautifulSoupWebReader = BeautifulSoupWebReader()

documentList = beautifulSoupWebReader.load_data(urls = ["https://openai.com/blog/planning-for-agi-and-beyond"])

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

response = retrieverQueryEngine.query("이 웹페이지에서 전하고 싶은 말은 무엇인가요? 한국어로 대답해 주세요.")

print(response)

▶ 실행 결과


Requirement already satisfied: llama-index-readers-web in ./env/lib/python3.10/site-packages (0.1.18)
Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (4.12.3)
Requirement already satisfied: spider-client<0.0.12,>=0.0.11 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.0.11)
Requirement already satisfied: aiohttp<4.0.0,>=3.9.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (3.9.5)
Requirement already satisfied: urllib3>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2.2.1)
Requirement already satisfied: html2text<2021.0.0,>=2020.1.16 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2020.1.16)
Requirement already satisfied: chromedriver-autoinstaller<0.7.0,>=0.6.3 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.6.4)
Requirement already satisfied: playwright<2.0,>=1.30 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (1.44.0)
Requirement already satisfied: requests<3.0.0,>=2.31.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2.32.3)
Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.10.43)
Requirement already satisfied: newspaper3k<0.3.0,>=0.2.8 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.2.8)
Requirement already satisfied: selenium<5.0.0,>=4.17.2 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (4.21.0)
Requirement already satisfied: aiosignal>=1.1.2 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.3.1)
Requirement already satisfied: attrs>=17.3.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (23.2.0)
Requirement already satisfied: yarl<2.0,>=1.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.9.4)
Requirement already satisfied: async-timeout<5.0,>=4.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (4.0.3)
Requirement already satisfied: multidict<7.0,>=4.5 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (6.0.5)
Requirement already satisfied: frozenlist>=1.1.1 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.4.1)
Requirement already satisfied: soupsieve>1.2 in ./env/lib/python3.10/site-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-web) (2.5)
Requirement already satisfied: packaging>=23.1 in ./env/lib/python3.10/site-packages (from chromedriver-autoinstaller<0.7.0,>=0.6.3->llama-index-readers-web) (23.2)
Requirement already satisfied: PyYAML>=6.0.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (6.0.1)
Requirement already satisfied: typing-inspect>=0.8.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.9.0)
Requirement already satisfied: openai>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.33.0)
Requirement already satisfied: typing-extensions>=4.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.12.2)
Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (8.3.0)
Requirement already satisfied: tiktoken>=0.3.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.7.0)
Requirement already satisfied: nltk<4.0.0,>=3.8.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.8.1)
Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.6.0)
Requirement already satisfied: pillow>=9.0.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (10.3.0)
Requirement already satisfied: dataclasses-json in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.6.6)
Requirement already satisfied: numpy in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.26.4)
Requirement already satisfied: deprecated>=1.2.9.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.2.14)
Requirement already satisfied: pandas in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.2.2)
Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.66.4)
Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.18 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.1.19)
Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.0.30)
Requirement already satisfied: wrapt in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.16.0)
Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.8)
Requirement already satisfied: httpx in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.27.0)
Requirement already satisfied: networkx>=3.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.3)
Requirement already satisfied: fsspec>=2023.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.6.0)
Requirement already satisfied: tldextract>=2.0.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (5.1.2)
Requirement already satisfied: feedparser>=5.2.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (6.0.11)
Requirement already satisfied: feedfinder2>=0.0.4 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.0.4)
Requirement already satisfied: jieba3k>=0.35.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.35.1)
Requirement already satisfied: lxml>=3.6.0 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (5.2.2)
Requirement already satisfied: tinysegmenter==0.3 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.3)
Requirement already satisfied: cssselect>=0.9.2 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.2.0)
Requirement already satisfied: python-dateutil>=2.5.3 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (2.9.0.post0)
Requirement already satisfied: greenlet==3.0.3 in ./env/lib/python3.10/site-packages (from playwright<2.0,>=1.30->llama-index-readers-web) (3.0.3)
Requirement already satisfied: pyee==11.1.0 in ./env/lib/python3.10/site-packages (from playwright<2.0,>=1.30->llama-index-readers-web) (11.1.0)
Requirement already satisfied: charset-normalizer<4,>=2 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.3.2)
Requirement already satisfied: certifi>=2017.4.17 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (2024.6.2)
Requirement already satisfied: idna<4,>=2.5 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.7)
Requirement already satisfied: trio-websocket~=0.9 in ./env/lib/python3.10/site-packages (from selenium<5.0.0,>=4.17.2->llama-index-readers-web) (0.11.1)
Requirement already satisfied: trio~=0.17 in ./env/lib/python3.10/site-packages (from selenium<5.0.0,>=4.17.2->llama-index-readers-web) (0.25.1)
Requirement already satisfied: six in ./env/lib/python3.10/site-packages (from feedfinder2>=0.0.4->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.16.0)
Requirement already satisfied: sgmllib3k in ./env/lib/python3.10/site-packages (from feedparser>=5.2.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.0.0)
Requirement already satisfied: pydantic>=1.10 in ./env/lib/python3.10/site-packages (from llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.7.3)
Requirement already satisfied: sniffio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.3.1)
Requirement already satisfied: httpcore==1.* in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.5)
Requirement already satisfied: anyio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.4.0)
Requirement already satisfied: h11<0.15,>=0.13 in ./env/lib/python3.10/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.14.0)
Requirement already satisfied: click in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (8.1.7)
Requirement already satisfied: joblib in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.4.2)
Requirement already satisfied: regex>=2021.8.3 in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.5.15)
Requirement already satisfied: distro<2,>=1.7.0 in ./env/lib/python3.10/site-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.9.0)
Requirement already satisfied: requests-file>=1.4 in ./env/lib/python3.10/site-packages (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (2.1.0)
Requirement already satisfied: filelock>=3.0.8 in ./env/lib/python3.10/site-packages (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (3.14.0)
Requirement already satisfied: exceptiongroup in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.2.1)
Requirement already satisfied: sortedcontainers in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (2.4.0)
Requirement already satisfied: outcome in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.3.0.post0)
Requirement already satisfied: wsproto>=0.14 in ./env/lib/python3.10/site-packages (from trio-websocket~=0.9->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.2.0)
Requirement already satisfied: mypy-extensions>=0.3.0 in ./env/lib/python3.10/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.0)
Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in ./env/lib/python3.10/site-packages (from urllib3>=1.1.0->llama-index-readers-web) (1.7.1)
Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in ./env/lib/python3.10/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.21.3)
Requirement already satisfied: pytz>=2020.1 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.1)
Requirement already satisfied: tzdata>=2022.7 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.1)
Requirement already satisfied: annotated-types>=0.4.0 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.7.0)
Requirement already satisfied: pydantic-core==2.18.4 in ./env/lib/python3.10/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.18.4)
이 웹페이지에서는 인공 일반 지능(AGI)에 대한 계획 및 그 이상에 대한 논의가 이루어지고 있습니다.

Requirement already satisfied: llama-index-readers-web in ./env/lib/python3.10/site-packages (0.1.18)

Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (4.12.3)

Requirement already satisfied: spider-client<0.0.12,>=0.0.11 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.0.11)

Requirement already satisfied: aiohttp<4.0.0,>=3.9.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (3.9.5)

Requirement already satisfied: urllib3>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2.2.1)

Requirement already satisfied: html2text<2021.0.0,>=2020.1.16 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2020.1.16)

Requirement already satisfied: chromedriver-autoinstaller<0.7.0,>=0.6.3 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.6.4)

Requirement already satisfied: playwright<2.0,>=1.30 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (1.44.0)

Requirement already satisfied: requests<3.0.0,>=2.31.0 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (2.32.3)

Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.10.43)

Requirement already satisfied: newspaper3k<0.3.0,>=0.2.8 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (0.2.8)

Requirement already satisfied: selenium<5.0.0,>=4.17.2 in ./env/lib/python3.10/site-packages (from llama-index-readers-web) (4.21.0)

Requirement already satisfied: aiosignal>=1.1.2 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.3.1)

Requirement already satisfied: attrs>=17.3.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (23.2.0)

Requirement already satisfied: yarl<2.0,>=1.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.9.4)

Requirement already satisfied: async-timeout<5.0,>=4.0 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (4.0.3)

Requirement already satisfied: multidict<7.0,>=4.5 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (6.0.5)

Requirement already satisfied: frozenlist>=1.1.1 in ./env/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.4.1)

Requirement already satisfied: soupsieve>1.2 in ./env/lib/python3.10/site-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-web) (2.5)

Requirement already satisfied: packaging>=23.1 in ./env/lib/python3.10/site-packages (from chromedriver-autoinstaller<0.7.0,>=0.6.3->llama-index-readers-web) (23.2)

Requirement already satisfied: PyYAML>=6.0.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (6.0.1)

Requirement already satisfied: typing-inspect>=0.8.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.9.0)

Requirement already satisfied: openai>=1.1.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.33.0)

Requirement already satisfied: typing-extensions>=4.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.12.2)

Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (8.3.0)

Requirement already satisfied: tiktoken>=0.3.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.7.0)

Requirement already satisfied: nltk<4.0.0,>=3.8.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.8.1)

Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.6.0)

Requirement already satisfied: pillow>=9.0.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (10.3.0)

Requirement already satisfied: dataclasses-json in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.6.6)

Requirement already satisfied: numpy in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.26.4)

Requirement already satisfied: deprecated>=1.2.9.3 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.2.14)

Requirement already satisfied: pandas in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.2.2)

Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.66.4)

Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.18 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.1.19)

Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.0.30)

Requirement already satisfied: wrapt in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.16.0)

Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.8)

Requirement already satisfied: httpx in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.27.0)

Requirement already satisfied: networkx>=3.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.3)

Requirement already satisfied: fsspec>=2023.5.0 in ./env/lib/python3.10/site-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.6.0)

Requirement already satisfied: tldextract>=2.0.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (5.1.2)

Requirement already satisfied: feedparser>=5.2.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (6.0.11)

Requirement already satisfied: feedfinder2>=0.0.4 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.0.4)

Requirement already satisfied: jieba3k>=0.35.1 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.35.1)

Requirement already satisfied: lxml>=3.6.0 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (5.2.2)

Requirement already satisfied: tinysegmenter==0.3 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (0.3)

Requirement already satisfied: cssselect>=0.9.2 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.2.0)

Requirement already satisfied: python-dateutil>=2.5.3 in ./env/lib/python3.10/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (2.9.0.post0)

Requirement already satisfied: greenlet==3.0.3 in ./env/lib/python3.10/site-packages (from playwright<2.0,>=1.30->llama-index-readers-web) (3.0.3)

Requirement already satisfied: pyee==11.1.0 in ./env/lib/python3.10/site-packages (from playwright<2.0,>=1.30->llama-index-readers-web) (11.1.0)

Requirement already satisfied: charset-normalizer<4,>=2 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.3.2)

Requirement already satisfied: certifi>=2017.4.17 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (2024.6.2)

Requirement already satisfied: idna<4,>=2.5 in ./env/lib/python3.10/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.7)

Requirement already satisfied: trio-websocket~=0.9 in ./env/lib/python3.10/site-packages (from selenium<5.0.0,>=4.17.2->llama-index-readers-web) (0.11.1)

Requirement already satisfied: trio~=0.17 in ./env/lib/python3.10/site-packages (from selenium<5.0.0,>=4.17.2->llama-index-readers-web) (0.25.1)

Requirement already satisfied: six in ./env/lib/python3.10/site-packages (from feedfinder2>=0.0.4->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.16.0)

Requirement already satisfied: sgmllib3k in ./env/lib/python3.10/site-packages (from feedparser>=5.2.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.0.0)

Requirement already satisfied: pydantic>=1.10 in ./env/lib/python3.10/site-packages (from llamaindex-py-client<0.2.0,>=0.1.18->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2.7.3)

Requirement already satisfied: sniffio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.3.1)

Requirement already satisfied: httpcore==1.* in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.5)

Requirement already satisfied: anyio in ./env/lib/python3.10/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (4.4.0)

Requirement already satisfied: h11<0.15,>=0.13 in ./env/lib/python3.10/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (0.14.0)

Requirement already satisfied: click in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (8.1.7)

Requirement already satisfied: joblib in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.4.2)

Requirement already satisfied: regex>=2021.8.3 in ./env/lib/python3.10/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.5.15)

Requirement already satisfied: distro<2,>=1.7.0 in ./env/lib/python3.10/site-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.9.0)

Requirement already satisfied: requests-file>=1.4 in ./env/lib/python3.10/site-packages (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (2.1.0)

Requirement already satisfied: filelock>=3.0.8 in ./env/lib/python3.10/site-packages (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (3.14.0)

Requirement already satisfied: exceptiongroup in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.2.1)

Requirement already satisfied: sortedcontainers in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (2.4.0)

Requirement already satisfied: outcome in ./env/lib/python3.10/site-packages (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.3.0.post0)

Requirement already satisfied: wsproto>=0.14 in ./env/lib/python3.10/site-packages (from trio-websocket~=0.9->selenium<5.0.0,>=4.17.2->llama-index-readers-web) (1.2.0)

Requirement already satisfied: mypy-extensions>=0.3.0 in ./env/lib/python3.10/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (1.0.0)

Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in ./env/lib/python3.10/site-packages (from urllib3>=1.1.0->llama-index-readers-web) (1.7.1)

Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in ./env/lib/python3.10/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (3.21.3)

Requirement already satisfied: pytz>=2020.1 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.1)

Requirement already satisfied: tzdata>=2022.7 in ./env/lib/python3.10/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-readers-web) (2024.1)

이 웹페이지에서는 인공 일반 지능(AGI)에 대한 계획 및 그 이상에 대한 논의가 이루어지고 있습니다.

[PYTHON/LLAMA-INDEX] Settings 클래스 : embed_model 정적 변수를 사용해 허깅 페이스 임베딩(HuggingFaceEmbeddings) 커스텀 설정하기

■ Settings 클래스의 embed_model 정적 변수를 사용해 허깅 페이스 임베딩(HuggingFaceEmbeddings)을 커스텀 설정하는 방법을 보여준다. ▶ main.py


import os

from langchain_huggingface    import HuggingFaceEmbeddings
from llama_index.core         import Settings, SimpleDirectoryReader, GPTVectorStoreIndex
from llama_index.llms.openai  import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(
    model       = "gpt-4", # 모델명
    temperature = 0.1      # 온도
)

Settings.max_input_size    = 4096 # LLM 입력의 최대 토큰 수
Settings.num_output        = 256  # LLM 출력의 토큰 수
Settings.max_chunk_overlap = 20   # 청크 오버랩의 최대 토큰 수

Settings.embed_model = HuggingFaceEmbeddings(model_name = "bongsoo/moco-sentencedistilbertV2.1")

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            : {answer3}")

import os

from langchain_huggingface import HuggingFaceEmbeddings

from llama_index.core import Settings, SimpleDirectoryReader, GPTVectorStoreIndex

from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(

model = "gpt-4", # 모델명

temperature = 0.1 # 온도

)

Settings.max_input_size = 4096 # LLM 입력의 최대 토큰 수

Settings.num_output = 256 # LLM 출력의 토큰 수

Settings.max_chunk_overlap = 20 # 청크 오버랩의 최대 토큰 수

Settings.embed_model = HuggingFaceEmbeddings(model_name = "bongsoo/moco-sentencedistilbertV2.1")

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 : {answer3}")

▶ 실행 결과


미코의 소꿉친구 이름     : 미코의 소꿉친구의 이름은 료입니다.
울프 코퍼레이션 CEO 이름 : 울프 박사입니다.
미코의 성격은            : 문맥에 따르면 미코는 끈기 있고 용감한 성격을 가진 것으로 보입니다. 그녀는 상황이 엄중한 데도 굴하지 않고 울프 박사와 싸우었으며, 그의 약점을 찾아 그를 이길 수 있었습니다.

미코의 소꿉친구 이름 : 미코의 소꿉친구의 이름은 료입니다.

울프 코퍼레이션 CEO 이름 : 울프 박사입니다.

미코의 성격은 : 문맥에 따르면 미코는 끈기 있고 용감한 성격을 가진 것으로 보입니다. 그녀는 상황이 엄중한 데도 굴하지 않고 울프 박사와 싸우었으며, 그의 약점을 찾아 그를 이길 수 있었습니다.

▶

[PYTHON/LLAMA-INDEX] Settings 클래스 : embed_model 정적 변수를 사용해 OPENAI 임베딩(OpenAIEmbedding) 커스텀 설정하기

■ Settings 클래스의 embed_model 정적 변수를 사용해 OPENAI 임베딩(OpenAIEmbedding)을 커스텀 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core              import Settings, SimpleDirectoryReader, GPTVectorStoreIndex
from llama_index.llms.openai       import OpenAI
from llama_index.embeddings.openai import OpenAIEmbedding

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(
    model       = "gpt-3.5-turbo", # 모델명
    temperature = 0.1              # 온도
)

Settings.max_input_size    = 4096 # LLM 입력의 최대 토큰 수
Settings.num_output        = 256  # LLM 출력의 토큰 수
Settings.max_chunk_overlap = 20   # 청크 오버랩의 최대 토큰 수

Settings.embed_model = OpenAIEmbedding(
    model            = "text-embedding-3-small", # 임베드 모델명
    embed_batch_size = 100                       # 임베드 배치 크기
)

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            : {answer3}")

import os

from llama_index.core import Settings, SimpleDirectoryReader, GPTVectorStoreIndex

from llama_index.llms.openai import OpenAI

from llama_index.embeddings.openai import OpenAIEmbedding

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(

model = "gpt-3.5-turbo", # 모델명

temperature = 0.1 # 온도

)

Settings.max_input_size = 4096 # LLM 입력의 최대 토큰 수

Settings.num_output = 256 # LLM 출력의 토큰 수

Settings.max_chunk_overlap = 20 # 청크 오버랩의 최대 토큰 수

Settings.embed_model = OpenAIEmbedding(

model = "text-embedding-3-small", # 임베드 모델명

embed_batch_size = 100 # 임베드 배치 크기

)

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 : {answer3}")

▶ 실행 결과


미코의 소꿉친구 이름     : 료
울프 코퍼레이션 CEO 이름 : 울프 코퍼레이션의 CEO 이름은 교활한 울프 박사입니다.
미코의 성격은            : Determined and resourceful.

미코의 소꿉친구 이름 : 료

울프 코퍼레이션 CEO 이름 : 울프 코퍼레이션의 CEO 이름은 교활한 울프 박사입니다.

미코의 성격은 : Determined and resourceful.

▶ requirements.txt

[PYTHON/LLAMA-INDEX] Settings 클래스 : max_input_size/num_output/max_chunk_overlap 정적 변수를 사용해 청크 분할 규칙 커스텀 설정하기

■ Settings 클래스의 max_input_size/num_output/max_chunk_overlap 정적 변수를 사용해 청크 분할 규칙을 커스텀 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core        import Settings, SimpleDirectoryReader, GPTVectorStoreIndex
from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(
    model       = "gpt-3.5-turbo", # 모델명
    temperature = 0.1              # 온도
)

Settings.max_input_size    = 4096 # LLM 입력의 최대 토큰 수
Settings.num_output        = 256  # LLM 출력의 토큰 수
Settings.max_chunk_overlap = 20   # 청크 오버랩의 최대 토큰 수

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            :\n{answer3}")

import os

from llama_index.core import Settings, SimpleDirectoryReader, GPTVectorStoreIndex

from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(

model = "gpt-3.5-turbo", # 모델명

temperature = 0.1 # 온도

)

Settings.max_input_size = 4096 # LLM 입력의 최대 토큰 수

Settings.num_output = 256 # LLM 출력의 토큰 수

Settings.max_chunk_overlap = 20 # 청크 오버랩의 최대 토큰 수

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 :\n{answer3}")

▶ 실행 결과


미코의 소꿉친구 이름     : 료
울프 코퍼레이션 CEO 이름 : 교활한 울프 박사
미코의 성격은            :
미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 능력을 보여줍니다. 이후 울프 코퍼레이션이 무너진 후에는 용서하고 우정을 회복하는 따뜻하고 이해심 깊은 성품을 보여주며, 새로운 도전에 대한 열정과 결의를 갖고 있습니다.

미코의 소꿉친구 이름 : 료

울프 코퍼레이션 CEO 이름 : 교활한 울프 박사

미코의 성격은 :

미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 능력을 보여줍니다. 이후 울프 코퍼레이션이 무너진 후에는 용서하고 우정을 회복하는 따뜻하고 이해심 깊은 성품을 보여주며, 새로운 도전에 대한 열정과 결의를 갖고 있습니다.

▶

[PYTHON/LLAMA-INDEX] Settings 클래스 : max_input_size/num_output/max_chunk_overlap 정적 변수를 사용해 청크 분할 규칙 커스텀 설정하기

■ Settings 클래스의 llm 정적 변수를 사용해 LLM을 커스텀 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core        import Settings, SimpleDirectoryReader, GPTVectorStoreIndex
from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(
    model       = "gpt-3.5-turbo", # 모델명
    temperature = 0.1              # 온도
)

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            :\n{answer3}")

import os

from llama_index.core import Settings, SimpleDirectoryReader, GPTVectorStoreIndex

from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(

model = "gpt-3.5-turbo", # 모델명

temperature = 0.1 # 온도

)

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 :\n{answer3}")

▶ 실행 결과


미코의 소꿉친구 이름     : 료
울프 코퍼레이션 CEO 이름 : 교활한 울프 박사
미코의 성격은            :
미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 능력을 보여줍니다. 이후 울프 코퍼레이션이 무너진 후에는 용서하고 우정을 회복하는 따뜻하고 이해심 깊은 성품을 보여주며, 새로운 도전에 대한 열정과 결의를 갖고 있습니다.

미코의 소꿉친구 이름 : 료

울프 코퍼레이션 CEO 이름 : 교활한 울프 박사

미코의 성격은 :

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpointer==2.4
langchain==0.2.3
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.75
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.32.0
orjson==3.10.3
packaging==23.2
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.1
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpointer==2.4

langchain==0.2.3

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.75

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.32.0

orjson==3.10.3

packaging==23.2

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.1

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

[PYTHON/LLAMA-INDEX] Settings 클래스 : llm 정적 변수를 사용해 LLM 커스텀 설정하기

■ Settings 클래스의 llm 정적 변수를 사용해 LLM을 커스텀 설정하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core        import Settings, SimpleDirectoryReader, GPTVectorStoreIndex
from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(model = "gpt-3.5-turbo", temperature = 0.1) # 모델명, 온도

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            :\n{answer3}")

import os

from llama_index.core import Settings, SimpleDirectoryReader, GPTVectorStoreIndex

from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

Settings.llm = OpenAI(model = "gpt-3.5-turbo", temperature = 0.1) # 모델명, 온도

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 :\n{answer3}")

▶ 실행 결과


미코의 소꿉친구 이름     : 료
울프 코퍼레이션 CEO 이름 : 교활한 울프 박사
미코의 성격은            :
미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 능력을 보여줍니다. 이후 울프 코퍼레이션이 무너진 후에는 용서하고 우정을 회복하는 따뜻하고 이해심 깊은 성품을 보여주며, 새로운 도전에 대한 열정과 결의를 갖고 있습니다.

미코의 소꿉친구 이름 : 료

울프 코퍼레이션 CEO 이름 : 교활한 울프 박사

미코의 성격은 :

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpointer==2.4
langchain==0.2.3
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.75
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.32.0
orjson==3.10.3
packaging==23.2
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.1
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpointer==2.4

langchain==0.2.3

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.75

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.32.0

orjson==3.10.3

packaging==23.2

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.1

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

[PYTHON/LLAMA-INDEX] VectorStoreIndex 클래스 : insert 메소드를 사용해 문서를 벡터 저장소 인덱스에 추가하기

■ VectorStoreIndex 클래스의 insert 메소드를 사용해 문서를 벡터 저장소 인덱스에 추가하는 방법을 보여준다. ▶ main.py


import os

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents([], model = "gpt-3.5-instruct")

for document in documentList:
    vectorStoreIndex.insert(document)

import os

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents([], model = "gpt-3.5-instruct")

for document in documentList:

vectorStoreIndex.insert(document)

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.32.0
packaging==24.0
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.1
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.32.0

packaging==24.0

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.1

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

※ pip install

[PYTHON/LLAMA-INDEX] GPTVectorStoreIndex 클래스 : from_documents 정적 메소드를 사용해 VectorStoreIndex 객체 만들기

■ GPTVectorStoreIndex 클래스의 from_documents 정적 메소드를 사용해 VectorStoreIndex 객체를 만드는 방법을 보여준다. ▶ main.py


import os

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList, model = "gpt-3.5-instruct")

import os

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList, model = "gpt-3.5-instruct")

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.32.0
packaging==24.0
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.1
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.32.0

packaging==24.0

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.1

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

※ pip install openai

[PYTHON/LLAMA-INDEX] Document 클래스 : 수동으로 문서 작성하기

■ Document 클래스를 사용해 수동으로 문서를 작성하는 방법을 보여준다. ▶ main.py


from llama_index.core import Document

textList = [
    "This is the first document. It contains some sample text.",
    "This is the second document. It also contains some sample text.",
    "This is the third document with different content."
]

documentList = [Document(text = text) for text in textList]

print(documentList)

from llama_index.core import Document

textList = [

"This is the first document. It contains some sample text.",

"This is the second document. It also contains some sample text.",

"This is the third document with different content."

]

documentList = [Document(text = text) for text in textList]

print(documentList)

▶ 실행 결과


[
    Document(
        id_                          ='40b4f5f2-c8be-404a-a0e9-18eedf1916f7',
        embedding                    = None,
        metadata                     = {},
        excluded_embed_metadata_keys = [],
        excluded_llm_metadata_keys   = [],
        relationships                = {},
        text                         = 'This is the first document. It contains some sample text.',
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           ='\n'
    ),
    Document
    (
        id_                          = '28b47184-b448-41ce-a6fe-b3df69286081',
        embedding                    = None,
        metadata                     = {},
        excluded_embed_metadata_keys = [],
        excluded_llm_metadata_keys   = [],
        relationships                = {},
        text                         = 'This is the second document. It also contains some sample text.',
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           = '\n'
    ),
    Document
    (
        id_                          = 'c60f0ec7-db70-4a7f-8936-7ddeed886c94',
        embedding                    = None,
        metadata                     = {},
        excluded_embed_metadata_keys = [],
        excluded_llm_metadata_keys   = [],
        relationships                = {},
        text                         = 'This is the third document with different content.',
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           = '\n'
    )
]

[

Document(

id_ ='40b4f5f2-c8be-404a-a0e9-18eedf1916f7',

embedding = None,

metadata = {},

excluded_embed_metadata_keys = [],

excluded_llm_metadata_keys = [],

relationships = {},

text = 'This is the first document. It contains some sample text.',

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator ='\n'

Document

(

id_ = '28b47184-b448-41ce-a6fe-b3df69286081',

embedding = None,

metadata = {},

excluded_embed_metadata_keys = [],

excluded_llm_metadata_keys = [],

relationships = {},

text = 'This is the second document. It also contains some sample text.',

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator = '\n'

Document

(

id_ = 'c60f0ec7-db70-4a7f-8936-7ddeed886c94',

embedding = None,

metadata = {},

excluded_embed_metadata_keys = [],

excluded_llm_metadata_keys = [],

relationships = {},

text = 'This is the third document with different content.',

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator = '\n'

)

]

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.33.0
packaging==24.0
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.2
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.33.0

packaging==24.0

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.2

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

※ pip install

[PYTHON/LLAMA-INDEX] SimpleDirectoryReader 클래스 : load_data 메소드를 사용해 문서 로드하기

■ SimpleDirectoryReader 클래스의 load_data 메소드를 사용해 문서를 로드하는 방법을 보여준다. ▶ main.py


from llama_index.core import SimpleDirectoryReader

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

print(documentList)

from llama_index.core import SimpleDirectoryReader

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

print(documentList)

▶ 실행 결과


[
    Document(
        id_       = 'd8ba06fd-16ed-4ab1-b2b9-4ebd77a18a51',
        embedding = None,
        metadata  = {
            'file_path'          : '/home/king/data/akazukin1.txt',
            'file_name'          : 'akazukin1.txt',
            'file_type'          : 'text/plain',
            'file_size'          : 686,
            'creation_date'      : '2024-06-07',
            'last_modified_date' : '2024-01-16'
        },
        excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        excluded_llm_metadata_keys   = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        relationships                = {},
        text                         = "제1장: 데이터 프론트\n\n밤이 되면 반짝이는 네오 도쿄. 고층 빌딩이 늘어서고, 네온사인이 거리를 수놓는다. 그 거리에서 빨간 두건을 쓴 소녀 미코는 불법 데이터 카우리아를 운반하는 배달원으로 일하고 있었다. 그녀는 어머니가 병에 걸려 치료비를 벌기 위해 데이터카우리아에 몸을 던지고 있었다.\n\n그러던 어느 날, 미코는 중요한 데이터를 운반하는 임무를 맡게 된다. 그 데이터에는 거대 기업 '울프 코퍼레이션'의 시민에 대한 악랄한 지배를 폭로하는 정보가 담겨 있었다. 그녀는 데이터를 받아 목적지로 향한다.\n",
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           = '\n'
    ),
    Document
    (
        id_       = '8897a682-d01d-407b-881a-6d1a7517158c',
        embedding = None,
        metadata  = {
            'file_path'          : '/home/king/data/akazukin2.txt',
            'file_name'          : 'akazukin2.txt',
            'file_type'          : 'text/plain',
            'file_size'          : 364,
            'creation_date'      : '2024-06-07',
            'last_modified_date' : '2024-01-16'
        },
        excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        excluded_llm_metadata_keys   = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        relationships                = {},
        text                         = "제2장: 울프 코퍼레이션의 함정\n\n미코는 목적지인 술집 '할머니의 집'으로 향하는 길에 울프 코퍼레이션의 요원들에게 쫓기게 된다. 그들은 '빨간 망토'라는 데이터 카우리아에 대한 소문을 듣고 데이터를 탈취하려 했다. 미코는 교묘하게 요원들을 흩뿌리고 술집에 도착한다.\n",
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           = '\n'
    ),
    Document(
        id_       ='1be7ff28-02e3-4ec9-9d49-be61eb108f0d',
        embedding = None,
        metadata  = {
            'file_path'          : '/home/king/data/akazukin3.txt',
            'file_name'          : 'akazukin3.txt',
            'file_type'          : 'text/plain',
            'file_size'          : 619,
            'creation_date'      : '2024-06-07',
            'last_modified_date' : '2024-01-16'
        },
        excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        excluded_llm_metadata_keys   = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],
        relationships                = {},
        text                         = "제3장: 배신과 재회\n\n술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단의 일원이었다. 하지만 료는 미코에게 배신감을 느꼈고, 그녀가 데이터 카우리아에 몸을 던진 것에 화가 났다.\n\n그럼에도 불구하고 미코는 료에게 데이터를 건네며 울프 코퍼레이션에 대한 반격을 믿기로 한다. 두 사람은 함께 울프 코퍼레이션의 음모를 밝혀내고 시민들을 구하기로 결심한다.\n",
        start_char_idx               = None,
        end_char_idx                 = None,
        text_template                = '{metadata_str}\n\n{content}',
        metadata_template            = '{key}: {value}',
        metadata_seperator           = '\n'
    ),
    ...
]

[

Document(

id_ = 'd8ba06fd-16ed-4ab1-b2b9-4ebd77a18a51',

embedding = None,

metadata = {

'file_path' : '/home/king/data/akazukin1.txt',

'file_name' : 'akazukin1.txt',

'file_type' : 'text/plain',

'file_size' : 686,

'creation_date' : '2024-06-07',

'last_modified_date' : '2024-01-16'

excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

excluded_llm_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

relationships = {},

text = "제1장: 데이터 프론트\n\n밤이 되면 반짝이는 네오 도쿄. 고층 빌딩이 늘어서고, 네온사인이 거리를 수놓는다. 그 거리에서 빨간 두건을 쓴 소녀 미코는 불법 데이터 카우리아를 운반하는 배달원으로 일하고 있었다. 그녀는 어머니가 병에 걸려 치료비를 벌기 위해 데이터카우리아에 몸을 던지고 있었다.\n\n그러던 어느 날, 미코는 중요한 데이터를 운반하는 임무를 맡게 된다. 그 데이터에는 거대 기업 '울프 코퍼레이션'의 시민에 대한 악랄한 지배를 폭로하는 정보가 담겨 있었다. 그녀는 데이터를 받아 목적지로 향한다.\n",

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator = '\n'

Document

(

id_ = '8897a682-d01d-407b-881a-6d1a7517158c',

embedding = None,

metadata = {

'file_path' : '/home/king/data/akazukin2.txt',

'file_name' : 'akazukin2.txt',

'file_type' : 'text/plain',

'file_size' : 364,

'creation_date' : '2024-06-07',

'last_modified_date' : '2024-01-16'

excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

excluded_llm_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

relationships = {},

text = "제2장: 울프 코퍼레이션의 함정\n\n미코는 목적지인 술집 '할머니의 집'으로 향하는 길에 울프 코퍼레이션의 요원들에게 쫓기게 된다. 그들은 '빨간 망토'라는 데이터 카우리아에 대한 소문을 듣고 데이터를 탈취하려 했다. 미코는 교묘하게 요원들을 흩뿌리고 술집에 도착한다.\n",

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator = '\n'

Document(

id_ ='1be7ff28-02e3-4ec9-9d49-be61eb108f0d',

embedding = None,

metadata = {

'file_path' : '/home/king/data/akazukin3.txt',

'file_name' : 'akazukin3.txt',

'file_type' : 'text/plain',

'file_size' : 619,

'creation_date' : '2024-06-07',

'last_modified_date' : '2024-01-16'

excluded_embed_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

excluded_llm_metadata_keys = ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'],

relationships = {},

text = "제3장: 배신과 재회\n\n술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단의 일원이었다. 하지만 료는 미코에게 배신감을 느꼈고, 그녀가 데이터 카우리아에 몸을 던진 것에 화가 났다.\n\n그럼에도 불구하고 미코는 료에게 데이터를 건네며 울프 코퍼레이션에 대한 반격을 믿기로 한다. 두 사람은 함께 울프 코퍼레이션의 음모를 밝혀내고 시민들을 구하기로 결심한다.\n",

start_char_idx = None,

end_char_idx = None,

text_template = '{metadata_str}\n\n{content}',

metadata_template = '{key}: {value}',

metadata_seperator = '\n'

...

]

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
certifi==2024.6.2
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.6
Deprecated==1.2.14
dirtyjson==1.0.8
distro==1.9.0
exceptiongroup==1.2.1
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
llama-index==0.10.43
llama-index-agent-openai==0.2.7
llama-index-cli==0.1.12
llama-index-core==0.10.43
llama-index-embeddings-openai==0.1.10
llama-index-indices-managed-llama-cloud==0.1.6
llama-index-legacy==0.9.48
llama-index-llms-openai==0.1.22
llama-index-multi-modal-llms-openai==0.1.6
llama-index-program-openai==0.1.6
llama-index-question-gen-openai==0.1.3
llama-index-readers-file==0.1.23
llama-index-readers-llama-parse==0.1.4
llama-parse==0.4.4
llamaindex-py-client==0.1.19
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.3
nltk==3.8.1
numpy==1.26.4
openai==1.32.0
packaging==24.0
pandas==2.2.2
pillow==10.3.0
pydantic==2.7.3
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.30
striprtf==0.0.26
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.1
tzdata==2024.1
urllib3==2.2.1
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

certifi==2024.6.2

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.6

Deprecated==1.2.14

dirtyjson==1.0.8

distro==1.9.0

exceptiongroup==1.2.1

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

llama-index==0.10.43

llama-index-agent-openai==0.2.7

llama-index-cli==0.1.12

llama-index-core==0.10.43

llama-index-embeddings-openai==0.1.10

llama-index-indices-managed-llama-cloud==0.1.6

llama-index-legacy==0.9.48

llama-index-llms-openai==0.1.22

llama-index-multi-modal-llms-openai==0.1.6

llama-index-program-openai==0.1.6

llama-index-question-gen-openai==0.1.3

llama-index-readers-file==0.1.23

llama-index-readers-llama-parse==0.1.4

llama-parse==0.4.4

llamaindex-py-client==0.1.19

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

networkx==3.3

nltk==3.8.1

numpy==1.26.4

openai==1.32.0

packaging==24.0

pandas==2.2.2

pillow==10.3.0

pydantic==2.7.3

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.30

striprtf==0.0.26

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.1

tzdata==2024.1

urllib3==2.2.1

wrapt==1.16.0

yarl==1.9.4

※ pip

[PYTHON/LLAMA-INDEX] load_index_from_storage 함수 : 라마 인덱스 데이터 로드하기

■ load_index_from_storage 함수를 사용해 라마 인덱스 데이터를 로드하는 방법을 보여준다. ▶ main.py


import os
import logging
import sys

from llama_index.core import StorageContext, load_index_from_storage

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

logging.basicConfig(stream = sys.stdout, level = logging.DEBUG, force = True)

storageContext = StorageContext.from_defaults(persist_dir = "./storage") # 라마 인덱스 데이터 저장 디렉토리

vectorStoreIndex = load_index_from_storage(storageContext)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?"      )
answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")
answer3 = retrieverQueryEngine.query("미코의 성격은?"               )

print(f"미코의 소꿉친구 이름     : {answer1}")
print(f"울프 코퍼레이션 CEO 이름 : {answer2}")
print(f"미코의 성격은            :\n{answer3}")

import os

import logging

import sys

from llama_index.core import StorageContext, load_index_from_storage

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

logging.basicConfig(stream = sys.stdout, level = logging.DEBUG, force = True)

storageContext = StorageContext.from_defaults(persist_dir = "./storage") # 라마 인덱스 데이터 저장 디렉토리

vectorStoreIndex = load_index_from_storage(storageContext)

retrieverQueryEngine = vectorStoreIndex.as_query_engine()

answer1 = retrieverQueryEngine.query("미코의 소꿉친구 이름은?" )

answer2 = retrieverQueryEngine.query("울프 코퍼레이션의 CEO 이름은?")

answer3 = retrieverQueryEngine.query("미코의 성격은?" )

print(f"미코의 소꿉친구 이름 : {answer1}")

print(f"울프 코퍼레이션 CEO 이름 : {answer2}")

print(f"미코의 성격은 :\n{answer3}")

▶ 실행 결과


DEBUG:llama_index.core.storage.kvstore.simple_kvstore:Loading llama_index.core.storage.kvstore.simple_kvstore from ./storage/docstore.json.
DEBUG:fsspec.local:open file: /home/king/testproject/storage/docstore.json
DEBUG:llama_index.core.storage.kvstore.simple_kvstore:Loading llama_index.core.storage.kvstore.simple_kvstore from ./storage/index_store.json.
DEBUG:fsspec.local:open file: /home/king/testproject/storage/index_store.json
DEBUG:llama_index.core.graph_stores.simple:Loading llama_index.core.graph_stores.simple from ./storage/graph_store.json.
DEBUG:fsspec.local:open file: /home/king/testproject/storage/graph_store.json
DEBUG:fsspec.local:open file: /home/king/testproject/storage/property_graph_store.json
DEBUG:llama_index.core.vector_stores.simple:Loading llama_index.core.vector_stores.simple from ./storage/default__vector_store.json.
DEBUG:fsspec.local:open file: /home/king/testproject/storage/default__vector_store.json
DEBUG:llama_index.core.vector_stores.simple:Loading llama_index.core.vector_stores.simple from ./storage/image__vector_store.json.
DEBUG:fsspec.local:open file: /home/king/testproject/storage/image__vector_store.json
INFO:llama_index.core.indices.loading:Loading all indices.
DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False
DEBUG:httpx:load_verify_locations cafile='/home/king/testproject/env/lib/python3.10/site-packages/certifi/cacert.pem'
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dca32290>, 'json_data': {'input': ['미코의 소꿉친구 이름은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings
DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=60.0 socket_options=None
DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dca12920>
DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x7f64dca487c0> server_hostname='api.openai.com' timeout=60.0
DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dca12950>
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:40 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'21'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999992'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_c5a0bcd32220edb07c65d55653e4273e'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=nkTy3ivlAbzUzytgZssBYptgFk626l2QUmcDwRqOBOQ-1717730260-1.0.1.1-aihbxzcYBco35zEj4LVmdM4DzPGM8Lua2fR2e4md2EE7a3xz2fc6xnpjuvt4tKh4kNEACsmt2c_N1VcucFWWhA; path=/; expires=Fri, 07-Jun-24 03:47:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=XsPmspdYh5sKEaMuaNifrM4F4325_RdDLq.NuSnWwAY-1717730260652-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d8fab5b305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers([('date', 'Fri, 07 Jun 2024 03:17:40 GMT'), ('content-type', 'application/json'), ('transfer-encoding', 'chunked'), ('connection', 'keep-alive'), ('access-control-allow-origin', '*'), ('openai-model', 'text-embedding-ada-002'), ('openai-organization', 'user-89spyw5fyioq1wqsb6pwacpu'), ('openai-processing-ms', '21'), ('openai-version', '2020-10-01'), ('strict-transport-security', 'max-age=15724800; includeSubDomains'), ('x-ratelimit-limit-requests', '3000'), ('x-ratelimit-limit-tokens', '1000000'), ('x-ratelimit-remaining-requests', '2999'), ('x-ratelimit-remaining-tokens', '999992'), ('x-ratelimit-reset-requests', '20ms'), ('x-ratelimit-reset-tokens', '0s'), ('x-request-id', 'req_c5a0bcd32220edb07c65d55653e4273e'), ('cf-cache-status', 'DYNAMIC'), ('set-cookie', '__cf_bm=nkTy3ivlAbzUzytgZssBYptgFk626l2QUmcDwRqOBOQ-1717730260-1.0.1.1-aihbxzcYBco35zEj4LVmdM4DzPGM8Lua2fR2e4md2EE7a3xz2fc6xnpjuvt4tKh4kNEACsmt2c_N1VcucFWWhA; path=/; expires=Fri, 07-Jun-24 03:47:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('set-cookie', '_cfuvid=XsPmspdYh5sKEaMuaNifrM4F4325_RdDLq.NuSnWwAY-1717730260652-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('server', 'cloudflare'), ('cf-ray', '88fd7d8fab5b305b-ICN'), ('content-encoding', 'gzip'), ('alt-svc', 'h3=":443"; ma=86400')])
DEBUG:openai._base_client:request_id: req_c5a0bcd32220edb07c65d55653e4273e
DEBUG:llama_index.core.indices.utils:> Top 2 nodes:
> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score:             0.826677] 제5장: 결전의 순간

미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신...
> [Node 5792c637-74de-45e3-81c1-62dc41c8a21d] [Similarity score:             0.825153] 제3장: 배신과 재회

술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단...
DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False
DEBUG:httpx:load_verify_locations cafile='/home/king/testproject/env/lib/python3.10/site-packages/certifi/cacert.pem'
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': "Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin3.txt\n\n제3장: 배신과 재회\n\n술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단의 일원이었다. 하지만 료는 미코에게 배신감을 느꼈고, 그녀가 데이터 카우리아에 몸을 던진 것에 화가 났다.\n\n그럼에도 불구하고 미코는 료에게 데이터를 건네며 울프 코퍼레이션에 대한 반격을 믿기로 한다. 두 사람은 함께 울프 코퍼레이션의 음모를 밝혀내고 시민들을 구하기로 결심한다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 미코의 소꿉친구 이름은?\nAnswer: "}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions
DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=60.0 socket_options=None
DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dc262e90>
DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x7f64deaded40> server_hostname='api.openai.com' timeout=60.0
DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dc262e60>
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:41 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'351'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9999'), (b'x-ratelimit-remaining-tokens', b'59574'), (b'x-ratelimit-reset-requests', b'8.64s'), (b'x-ratelimit-reset-tokens', b'426ms'), (b'x-request-id', b'req_b7a589d97b4ac1b84eb33c0bd0ad6516'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=7W1TFceQ4JA91P4W4IEwUndOyl6hdphUI1s9GQyNFWY-1717730261-1.0.1.1-rF0gCMlakrxaKJhqQRMFvBwavlxGsM3NssNFOjsHrtZXj1bGjBg4vAmHVfEWOn5iNPJQDLq_eToIR..PET92eg; path=/; expires=Fri, 07-Jun-24 03:47:41 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=hejal5DunMH9lrgAeXGl.YwKMuMyrbNt9oPfru30h8U-1717730261361-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d915ec3327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers([('date', 'Fri, 07 Jun 2024 03:17:41 GMT'), ('content-type', 'application/json'), ('transfer-encoding', 'chunked'), ('connection', 'keep-alive'), ('openai-organization', 'user-89spyw5fyioq1wqsb6pwacpu'), ('openai-processing-ms', '351'), ('openai-version', '2020-10-01'), ('strict-transport-security', 'max-age=15724800; includeSubDomains'), ('x-ratelimit-limit-requests', '10000'), ('x-ratelimit-limit-tokens', '60000'), ('x-ratelimit-remaining-requests', '9999'), ('x-ratelimit-remaining-tokens', '59574'), ('x-ratelimit-reset-requests', '8.64s'), ('x-ratelimit-reset-tokens', '426ms'), ('x-request-id', 'req_b7a589d97b4ac1b84eb33c0bd0ad6516'), ('cf-cache-status', 'DYNAMIC'), ('set-cookie', '__cf_bm=7W1TFceQ4JA91P4W4IEwUndOyl6hdphUI1s9GQyNFWY-1717730261-1.0.1.1-rF0gCMlakrxaKJhqQRMFvBwavlxGsM3NssNFOjsHrtZXj1bGjBg4vAmHVfEWOn5iNPJQDLq_eToIR..PET92eg; path=/; expires=Fri, 07-Jun-24 03:47:41 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('set-cookie', '_cfuvid=hejal5DunMH9lrgAeXGl.YwKMuMyrbNt9oPfru30h8U-1717730261361-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('server', 'cloudflare'), ('cf-ray', '88fd7d915ec3327d-ICN'), ('content-encoding', 'gzip'), ('alt-svc', 'h3=":443"; ma=86400')])
DEBUG:openai._base_client:request_id: req_b7a589d97b4ac1b84eb33c0bd0ad6516
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dbcd2a70>, 'json_data': {'input': ['울프 코퍼레이션의 CEO 이름은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:41 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'19'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999989'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_c73e85835cca8078a00ad87f1f811785'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d958b43305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:41 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'text-embedding-ada-002', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '19', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '3000', 'x-ratelimit-limit-tokens': '1000000', 'x-ratelimit-remaining-requests': '2999', 'x-ratelimit-remaining-tokens': '999989', 'x-ratelimit-reset-requests': '20ms', 'x-ratelimit-reset-tokens': '0s', 'x-request-id': 'req_c73e85835cca8078a00ad87f1f811785', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d958b43305b-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})
DEBUG:openai._base_client:request_id: req_c73e85835cca8078a00ad87f1f811785
DEBUG:llama_index.core.indices.utils:> Top 2 nodes:
> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score:             0.83732] 제5장: 결전의 순간

미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신...
> [Node 4362bb3e-1179-44c9-9ccc-7d951cdf5a96] [Similarity score:             0.803833] 제7장: 새로운 시작

울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 ...
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': 'Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin7.txt\n\n제7장: 새로운 시작\n\n울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 시작한다. 그들은 스스로의 힘으로 미래의 네오 도쿄를 더 나은 도시로 바꾸어 나갈 것을 다짐한다. 이것은 미코와 료, 그리고 전뇌 빨간 망토의 새로운 모험의 시작이었다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 울프 코퍼레이션의 CEO 이름은?\nAnswer: '}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'515'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9998'), (b'x-ratelimit-remaining-tokens', b'59611'), (b'x-ratelimit-reset-requests', b'16.417s'), (b'x-ratelimit-reset-tokens', b'389ms'), (b'x-request-id', b'req_3de085ba97e1b1a4d47604411822c429'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d96ee49327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:42 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '515', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '10000', 'x-ratelimit-limit-tokens': '60000', 'x-ratelimit-remaining-requests': '9998', 'x-ratelimit-remaining-tokens': '59611', 'x-ratelimit-reset-requests': '16.417s', 'x-ratelimit-reset-tokens': '389ms', 'x-request-id': 'req_3de085ba97e1b1a4d47604411822c429', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d96ee49327d-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})
DEBUG:openai._base_client:request_id: req_3de085ba97e1b1a4d47604411822c429
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dca33e20>, 'json_data': {'input': ['미코의 성격은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'19'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999994'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_4531c9ce2feaa047646ee067c556b2a9'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d9bfceb305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:42 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'text-embedding-ada-002', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '19', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '3000', 'x-ratelimit-limit-tokens': '1000000', 'x-ratelimit-remaining-requests': '2999', 'x-ratelimit-remaining-tokens': '999994', 'x-ratelimit-reset-requests': '20ms', 'x-ratelimit-reset-tokens': '0s', 'x-request-id': 'req_4531c9ce2feaa047646ee067c556b2a9', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d9bfceb305b-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})
DEBUG:openai._base_client:request_id: req_4531c9ce2feaa047646ee067c556b2a9
DEBUG:llama_index.core.indices.utils:> Top 2 nodes:
> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score:             0.825045] 제5장: 결전의 순간

미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신...
> [Node 4362bb3e-1179-44c9-9ccc-7d951cdf5a96] [Similarity score:             0.818595] 제7장: 새로운 시작

울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 ...
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': 'Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin7.txt\n\n제7장: 새로운 시작\n\n울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 시작한다. 그들은 스스로의 힘으로 미래의 네오 도쿄를 더 나은 도시로 바꾸어 나갈 것을 다짐한다. 이것은 미코와 료, 그리고 전뇌 빨간 망토의 새로운 모험의 시작이었다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 미코의 성격은?\nAnswer: '}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:45 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'2368'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9997'), (b'x-ratelimit-remaining-tokens', b'59616'), (b'x-ratelimit-reset-requests', b'23.966s'), (b'x-ratelimit-reset-tokens', b'384ms'), (b'x-request-id', b'req_d84a9069d17d013d58ebdbf190f7b793'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d9d9fde327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:45 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '2368', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '10000', 'x-ratelimit-limit-tokens': '60000', 'x-ratelimit-remaining-requests': '9997', 'x-ratelimit-remaining-tokens': '59616', 'x-ratelimit-reset-requests': '23.966s', 'x-ratelimit-reset-tokens': '384ms', 'x-request-id': 'req_d84a9069d17d013d58ebdbf190f7b793', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d9d9fde327d-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})
DEBUG:openai._base_client:request_id: req_d84a9069d17d013d58ebdbf190f7b793
미코의 소꿉친구 이름     : 료
울프 코퍼레이션 CEO 이름 : 교활한 울프 박사
미코의 성격은            :
미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 등 지능적이고 창의적인 면을 보여준다. 이후 울프 코퍼레이션이 무너지고 나서는 료와의 우정을 회복하고 함께 미래를 향해 나아가는 모습을 보여주어 따뜻하고 헌신적인 성향을 보여준다.

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

DEBUG:llama_index.core.storage.kvstore.simple_kvstore:Loading llama_index.core.storage.kvstore.simple_kvstore from ./storage/docstore.json.

DEBUG:fsspec.local:open file: /home/king/testproject/storage/docstore.json

DEBUG:llama_index.core.storage.kvstore.simple_kvstore:Loading llama_index.core.storage.kvstore.simple_kvstore from ./storage/index_store.json.

DEBUG:fsspec.local:open file: /home/king/testproject/storage/index_store.json

DEBUG:llama_index.core.graph_stores.simple:Loading llama_index.core.graph_stores.simple from ./storage/graph_store.json.

DEBUG:fsspec.local:open file: /home/king/testproject/storage/graph_store.json

DEBUG:fsspec.local:open file: /home/king/testproject/storage/property_graph_store.json

DEBUG:llama_index.core.vector_stores.simple:Loading llama_index.core.vector_stores.simple from ./storage/default__vector_store.json.

DEBUG:fsspec.local:open file: /home/king/testproject/storage/default__vector_store.json

DEBUG:llama_index.core.vector_stores.simple:Loading llama_index.core.vector_stores.simple from ./storage/image__vector_store.json.

DEBUG:fsspec.local:open file: /home/king/testproject/storage/image__vector_store.json

INFO:llama_index.core.indices.loading:Loading all indices.

DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False

DEBUG:httpx:load_verify_locations cafile='/home/king/testproject/env/lib/python3.10/site-packages/certifi/cacert.pem'

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dca32290>, 'json_data': {'input': ['미코의 소꿉친구 이름은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings

DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=60.0 socket_options=None

DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dca12920>

DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x7f64dca487c0> server_hostname='api.openai.com' timeout=60.0

DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dca12950>

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:40 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'21'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999992'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_c5a0bcd32220edb07c65d55653e4273e'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=nkTy3ivlAbzUzytgZssBYptgFk626l2QUmcDwRqOBOQ-1717730260-1.0.1.1-aihbxzcYBco35zEj4LVmdM4DzPGM8Lua2fR2e4md2EE7a3xz2fc6xnpjuvt4tKh4kNEACsmt2c_N1VcucFWWhA; path=/; expires=Fri, 07-Jun-24 03:47:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=XsPmspdYh5sKEaMuaNifrM4F4325_RdDLq.NuSnWwAY-1717730260652-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d8fab5b305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers([('date', 'Fri, 07 Jun 2024 03:17:40 GMT'), ('content-type', 'application/json'), ('transfer-encoding', 'chunked'), ('connection', 'keep-alive'), ('access-control-allow-origin', '*'), ('openai-model', 'text-embedding-ada-002'), ('openai-organization', 'user-89spyw5fyioq1wqsb6pwacpu'), ('openai-processing-ms', '21'), ('openai-version', '2020-10-01'), ('strict-transport-security', 'max-age=15724800; includeSubDomains'), ('x-ratelimit-limit-requests', '3000'), ('x-ratelimit-limit-tokens', '1000000'), ('x-ratelimit-remaining-requests', '2999'), ('x-ratelimit-remaining-tokens', '999992'), ('x-ratelimit-reset-requests', '20ms'), ('x-ratelimit-reset-tokens', '0s'), ('x-request-id', 'req_c5a0bcd32220edb07c65d55653e4273e'), ('cf-cache-status', 'DYNAMIC'), ('set-cookie', '__cf_bm=nkTy3ivlAbzUzytgZssBYptgFk626l2QUmcDwRqOBOQ-1717730260-1.0.1.1-aihbxzcYBco35zEj4LVmdM4DzPGM8Lua2fR2e4md2EE7a3xz2fc6xnpjuvt4tKh4kNEACsmt2c_N1VcucFWWhA; path=/; expires=Fri, 07-Jun-24 03:47:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('set-cookie', '_cfuvid=XsPmspdYh5sKEaMuaNifrM4F4325_RdDLq.NuSnWwAY-1717730260652-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('server', 'cloudflare'), ('cf-ray', '88fd7d8fab5b305b-ICN'), ('content-encoding', 'gzip'), ('alt-svc', 'h3=":443"; ma=86400')])

DEBUG:openai._base_client:request_id: req_c5a0bcd32220edb07c65d55653e4273e

DEBUG:llama_index.core.indices.utils:> Top 2 nodes:

> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score: 0.826677] 제5장: 결전의 순간

미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신...

> [Node 5792c637-74de-45e3-81c1-62dc41c8a21d] [Similarity score: 0.825153] 제3장: 배신과 재회

술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단...

DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False

DEBUG:httpx:load_verify_locations cafile='/home/king/testproject/env/lib/python3.10/site-packages/certifi/cacert.pem'

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': "Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin3.txt\n\n제3장: 배신과 재회\n\n술집 '할머니의 집'에서 미코는 데이터를 받을 사람인 료를 기다리고 있었다. 료는 그녀의 어릴 적 친구이자 그 역시 울프 코퍼레이션과 싸우는 해커 집단의 일원이었다. 하지만 료는 미코에게 배신감을 느꼈고, 그녀가 데이터 카우리아에 몸을 던진 것에 화가 났다.\n\n그럼에도 불구하고 미코는 료에게 데이터를 건네며 울프 코퍼레이션에 대한 반격을 믿기로 한다. 두 사람은 함께 울프 코퍼레이션의 음모를 밝혀내고 시민들을 구하기로 결심한다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 미코의 소꿉친구 이름은?\nAnswer: "}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions

DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=60.0 socket_options=None

DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dc262e90>

DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x7f64deaded40> server_hostname='api.openai.com' timeout=60.0

DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x7f64dc262e60>

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:41 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'351'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9999'), (b'x-ratelimit-remaining-tokens', b'59574'), (b'x-ratelimit-reset-requests', b'8.64s'), (b'x-ratelimit-reset-tokens', b'426ms'), (b'x-request-id', b'req_b7a589d97b4ac1b84eb33c0bd0ad6516'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=7W1TFceQ4JA91P4W4IEwUndOyl6hdphUI1s9GQyNFWY-1717730261-1.0.1.1-rF0gCMlakrxaKJhqQRMFvBwavlxGsM3NssNFOjsHrtZXj1bGjBg4vAmHVfEWOn5iNPJQDLq_eToIR..PET92eg; path=/; expires=Fri, 07-Jun-24 03:47:41 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=hejal5DunMH9lrgAeXGl.YwKMuMyrbNt9oPfru30h8U-1717730261361-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d915ec3327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers([('date', 'Fri, 07 Jun 2024 03:17:41 GMT'), ('content-type', 'application/json'), ('transfer-encoding', 'chunked'), ('connection', 'keep-alive'), ('openai-organization', 'user-89spyw5fyioq1wqsb6pwacpu'), ('openai-processing-ms', '351'), ('openai-version', '2020-10-01'), ('strict-transport-security', 'max-age=15724800; includeSubDomains'), ('x-ratelimit-limit-requests', '10000'), ('x-ratelimit-limit-tokens', '60000'), ('x-ratelimit-remaining-requests', '9999'), ('x-ratelimit-remaining-tokens', '59574'), ('x-ratelimit-reset-requests', '8.64s'), ('x-ratelimit-reset-tokens', '426ms'), ('x-request-id', 'req_b7a589d97b4ac1b84eb33c0bd0ad6516'), ('cf-cache-status', 'DYNAMIC'), ('set-cookie', '__cf_bm=7W1TFceQ4JA91P4W4IEwUndOyl6hdphUI1s9GQyNFWY-1717730261-1.0.1.1-rF0gCMlakrxaKJhqQRMFvBwavlxGsM3NssNFOjsHrtZXj1bGjBg4vAmHVfEWOn5iNPJQDLq_eToIR..PET92eg; path=/; expires=Fri, 07-Jun-24 03:47:41 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('set-cookie', '_cfuvid=hejal5DunMH9lrgAeXGl.YwKMuMyrbNt9oPfru30h8U-1717730261361-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), ('server', 'cloudflare'), ('cf-ray', '88fd7d915ec3327d-ICN'), ('content-encoding', 'gzip'), ('alt-svc', 'h3=":443"; ma=86400')])

DEBUG:openai._base_client:request_id: req_b7a589d97b4ac1b84eb33c0bd0ad6516

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dbcd2a70>, 'json_data': {'input': ['울프 코퍼레이션의 CEO 이름은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:41 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'19'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999989'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_c73e85835cca8078a00ad87f1f811785'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d958b43305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:41 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'text-embedding-ada-002', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '19', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '3000', 'x-ratelimit-limit-tokens': '1000000', 'x-ratelimit-remaining-requests': '2999', 'x-ratelimit-remaining-tokens': '999989', 'x-ratelimit-reset-requests': '20ms', 'x-ratelimit-reset-tokens': '0s', 'x-request-id': 'req_c73e85835cca8078a00ad87f1f811785', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d958b43305b-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})

DEBUG:openai._base_client:request_id: req_c73e85835cca8078a00ad87f1f811785

DEBUG:llama_index.core.indices.utils:> Top 2 nodes:

> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score: 0.83732] 제5장: 결전의 순간

> [Node 4362bb3e-1179-44c9-9ccc-7d951cdf5a96] [Similarity score: 0.803833] 제7장: 새로운 시작

울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 ...

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': 'Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin7.txt\n\n제7장: 새로운 시작\n\n울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 시작한다. 그들은 스스로의 힘으로 미래의 네오 도쿄를 더 나은 도시로 바꾸어 나갈 것을 다짐한다. 이것은 미코와 료, 그리고 전뇌 빨간 망토의 새로운 모험의 시작이었다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 울프 코퍼레이션의 CEO 이름은?\nAnswer: '}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'515'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9998'), (b'x-ratelimit-remaining-tokens', b'59611'), (b'x-ratelimit-reset-requests', b'16.417s'), (b'x-ratelimit-reset-tokens', b'389ms'), (b'x-request-id', b'req_3de085ba97e1b1a4d47604411822c429'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d96ee49327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:42 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '515', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '10000', 'x-ratelimit-limit-tokens': '60000', 'x-ratelimit-remaining-requests': '9998', 'x-ratelimit-remaining-tokens': '59611', 'x-ratelimit-reset-requests': '16.417s', 'x-ratelimit-reset-tokens': '389ms', 'x-request-id': 'req_3de085ba97e1b1a4d47604411822c429', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d96ee49327d-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})

DEBUG:openai._base_client:request_id: req_3de085ba97e1b1a4d47604411822c429

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/embeddings', 'files': None, 'post_parser': <function Embeddings.create.<locals>.parser at 0x7f64dca33e20>, 'json_data': {'input': ['미코의 성격은?'], 'model': 'text-embedding-ada-002', 'encoding_format': 'base64'}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/embeddings

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'openai-model', b'text-embedding-ada-002'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'19'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'3000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-remaining-requests', b'2999'), (b'x-ratelimit-remaining-tokens', b'999994'), (b'x-ratelimit-reset-requests', b'20ms'), (b'x-ratelimit-reset-tokens', b'0s'), (b'x-request-id', b'req_4531c9ce2feaa047646ee067c556b2a9'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d9bfceb305b-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/embeddings "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:42 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'access-control-allow-origin': '*', 'openai-model': 'text-embedding-ada-002', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '19', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '3000', 'x-ratelimit-limit-tokens': '1000000', 'x-ratelimit-remaining-requests': '2999', 'x-ratelimit-remaining-tokens': '999994', 'x-ratelimit-reset-requests': '20ms', 'x-ratelimit-reset-tokens': '0s', 'x-request-id': 'req_4531c9ce2feaa047646ee067c556b2a9', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d9bfceb305b-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})

DEBUG:openai._base_client:request_id: req_4531c9ce2feaa047646ee067c556b2a9

DEBUG:llama_index.core.indices.utils:> Top 2 nodes:

> [Node a28f295b-4029-4d54-9d62-ae2b8413062a] [Similarity score: 0.825045] 제5장: 결전의 순간

> [Node 4362bb3e-1179-44c9-9ccc-7d951cdf5a96] [Similarity score: 0.818595] 제7장: 새로운 시작

DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': "You are an expert Q&A system that is trusted around the world.\nAlways answer the query using the provided context information, and not prior knowledge.\nSome rules to follow:\n1. Never directly reference the given context in your answer.\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines."}, {'role': 'user', 'content': 'Context information is below.\n---------------------\nfile_path: /home/king/data/akazukin5.txt\n\n제5장: 결전의 순간\n\n미코와 료는 마침내 울프 코퍼레이션의 최상층에 도착해 CEO인 교활한 울프 박사와 대면한다. 울프 박사는 시민을 지배하려는 사악한 야망을 드러내며 자신의 압도적인 힘을 과시한다. 하지만 미코와 료는 서로를 도와가며 울프 박사와 싸우고 그의 약점을 찾아낸다.\n\nfile_path: /home/king/data/akazukin7.txt\n\n제7장: 새로운 시작\n\n울프 코퍼레이션이 무너진 후, 미코와 료는 서로의 과거를 용서하고 다시 우정을 회복한다. 미코는 데이터카우리아를 그만두고 료와 함께 새로운 길을 걷기 시작한다. 그들은 스스로의 힘으로 미래의 네오 도쿄를 더 나은 도시로 바꾸어 나갈 것을 다짐한다. 이것은 미코와 료, 그리고 전뇌 빨간 망토의 새로운 모험의 시작이었다.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: 미코의 성격은?\nAnswer: '}], 'model': 'gpt-3.5-turbo', 'stream': False, 'temperature': 0.1}}

DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions

DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_headers.complete

DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:send_request_body.complete

DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Fri, 07 Jun 2024 03:17:45 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-89spyw5fyioq1wqsb6pwacpu'), (b'openai-processing-ms', b'2368'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'60000'), (b'x-ratelimit-remaining-requests', b'9997'), (b'x-ratelimit-remaining-tokens', b'59616'), (b'x-ratelimit-reset-requests', b'23.966s'), (b'x-ratelimit-reset-tokens', b'384ms'), (b'x-request-id', b'req_d84a9069d17d013d58ebdbf190f7b793'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'88fd7d9d9fde327d-ICN'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')])

INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"

DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>

DEBUG:httpcore.http11:receive_response_body.complete

DEBUG:httpcore.http11:response_closed.started

DEBUG:httpcore.http11:response_closed.complete

DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers({'date': 'Fri, 07 Jun 2024 03:17:45 GMT', 'content-type': 'application/json', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'openai-organization': 'user-89spyw5fyioq1wqsb6pwacpu', 'openai-processing-ms': '2368', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15724800; includeSubDomains', 'x-ratelimit-limit-requests': '10000', 'x-ratelimit-limit-tokens': '60000', 'x-ratelimit-remaining-requests': '9997', 'x-ratelimit-remaining-tokens': '59616', 'x-ratelimit-reset-requests': '23.966s', 'x-ratelimit-reset-tokens': '384ms', 'x-request-id': 'req_d84a9069d17d013d58ebdbf190f7b793', 'cf-cache-status': 'DYNAMIC', 'server': 'cloudflare', 'cf-ray': '88fd7d9d9fde327d-ICN', 'content-encoding': 'gzip', 'alt-svc': 'h3=":443"; ma=86400'})

DEBUG:openai._base_client:request_id: req_d84a9069d17d013d58ebdbf190f7b793

미코의 소꿉친구 이름 : 료

울프 코퍼레이션 CEO 이름 : 교활한 울프 박사

미코의 성격은 :

미코는 울프 박사와의 결전에서 용감하고 결의가 강한 모습을 보여주며, 료와의 협력을 통해 약점을 찾아내는 등 지능적이고 창의적인 면을 보여준다. 이후 울프 코퍼레이션이 무너지고 나서는 료와의 우정을 회복하고 함께 미래를 향해 나아가는 모습을 보여주어 따뜻하고 헌신적인 성향을 보여준다.

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.5.14
frozenlist==1.4.1
fsspec==2024.6.0
greenlet==3.0.3
idna==3.7
langchain==0.0.181
llama-index==0.6.12
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numexpr==2.10.0
numpy==1.26.4
openai==0.28.0
openapi-schema-pydantic==1.2.4
packaging==24.0
pandas==2.2.2
pydantic==1.10.15
python-dateutil==2.9.0.post0
pytz==2024.1
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
six==1.16.0
SQLAlchemy==2.0.24
tenacity==8.3.0
tiktoken==0.7.0
tqdm==4.66.4
typing-inspect==0.8.0
typing_extensions==4.5.0
tzdata==2024.1
urllib3==1.26.18
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.5.14

frozenlist==1.4.1

fsspec==2024.6.0

greenlet==3.0.3

idna==3.7

langchain==0.0.181

llama-index==0.6.12

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numexpr==2.10.0

numpy==1.26.4

openai==0.28.0

openapi-schema-pydantic==1.2.4

packaging==24.0

pandas==2.2.2

pydantic==1.10.15

python-dateutil==2.9.0.post0

pytz==2024.1

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

six==1.16.0

SQLAlchemy==2.0.24

tenacity==8.3.0

tiktoken==0.7.0

tqdm==4.66.4

typing-inspect==0.8.0

typing_extensions==4.5.0

tzdata==2024.1

urllib3==1.26.18

yarl==1.9.4

※ pip

[PYTHON/LLAMA-INDEX] VectorStoreIndex 클래스 : storage_context 변수를 사용해 라마 인덱스 데이터 저장하기

■ VectorStoreIndex 클래스의 storage_context 변수를 사용해 라마 인덱스 데이터를 저장하는 방법을 보여준다. ▶ main.py


import os
import logging
import sys

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

logging.basicConfig(stream = sys.stdout, level = logging.DEBUG, force = True)

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList, model = "gpt-3.5-instruct")

vectorStoreIndex.storage_context.persist()

import os

import logging

import sys

from llama_index.core import SimpleDirectoryReader, GPTVectorStoreIndex

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

logging.basicConfig(stream = sys.stdout, level = logging.DEBUG, force = True)

simpleDirectoryReader = SimpleDirectoryReader(input_dir = "/home/king/data")

documentList = simpleDirectoryReader.load_data()

vectorStoreIndex = GPTVectorStoreIndex.from_documents(documentList, model = "gpt-3.5-instruct")

vectorStoreIndex.storage_context.persist()

※ 실행 디렉토리에서 storage 디렉토리가 생성되고 인덱스

[PYTHON/LLAMA-INDEX] 라마 인덱스 질의 응답 사용하기

■ 라마 인덱스에서 질의 응답을 사용하는 방법을 보여준다. ※ 첨부 파일 data.zip 파일을 다운로드 받아 적당한 위치에 압축을 풀고 main.py의 SimpleDirectoryReader 객체를