■ MultiQueryRetriever 클래스의 invoke 메소드를 사용하는 방법을 보여준다.
※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다.
▶ main.py
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
import logging from dotenv import load_dotenv from langchain_community.document_loaders import WebBaseLoader from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_openai import OpenAIEmbeddings from langchain_chroma import Chroma from langchain_openai import ChatOpenAI from langchain.retrievers.multi_query import MultiQueryRetriever logging.basicConfig() logger = logging.getLogger("langchain.retrievers.multi_query") logger.setLevel(logging.INFO) load_dotenv() webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/") documentList = webBaseLoader.load() recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0) splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList) openAIEmbeddings = OpenAIEmbeddings() chroma = Chroma.from_documents(documents = splitDocumentList, embedding = openAIEmbeddings) vectorStoreRetriever = chroma.as_retriever() chatOpenAI = ChatOpenAI(temperature = 0) multiQueryRetriever = MultiQueryRetriever.from_llm(retriever = vectorStoreRetriever, llm = chatOpenAI) question = "What are the approaches to Task Decomposition?" responseDocumentList = multiQueryRetriever.invoke(question) print(len(responseDocumentList)) print(responseDocumentList[0].page_content) """ 5 Task decomposition can be done (1) by LLM with simple prompting like "Steps for XYZ.\n1.", "What are the subgoals for achieving XYZ?", (2) by using task-specific instructions; e.g. "Write a story outline." for writing a novel, or (3) with human inputs. """ |
▶ requirements.txt
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
aiohappyeyeballs==2.3.5 aiohttp==3.10.3 aiosignal==1.3.1 annotated-types==0.7.0 anyio==4.4.0 asgiref==3.8.1 async-timeout==4.0.3 attrs==24.2.0 backoff==2.2.1 bcrypt==4.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 build==1.2.1 cachetools==5.4.0 certifi==2024.7.4 charset-normalizer==3.3.2 chroma-hnswlib==0.7.6 chromadb==0.5.5 click==8.1.7 coloredlogs==15.0.1 dataclasses-json==0.6.7 Deprecated==1.2.14 distro==1.9.0 exceptiongroup==1.2.2 fastapi==0.112.0 filelock==3.15.4 flatbuffers==24.3.25 frozenlist==1.4.1 fsspec==2024.6.1 google-auth==2.33.0 googleapis-common-protos==1.63.2 greenlet==3.0.3 grpcio==1.65.4 h11==0.14.0 httpcore==1.0.5 httptools==0.6.1 httpx==0.27.0 huggingface-hub==0.24.5 humanfriendly==10.0 idna==3.7 importlib_metadata==8.0.0 importlib_resources==6.4.0 jiter==0.5.0 jsonpatch==1.33 jsonpointer==3.0.0 kubernetes==30.1.0 langchain==0.2.12 langchain-chroma==0.1.2 langchain-community==0.2.11 langchain-core==0.2.29 langchain-openai==0.1.21 langchain-text-splitters==0.2.2 langsmith==0.1.98 markdown-it-py==3.0.0 marshmallow==3.21.3 mdurl==0.1.2 mmh3==4.1.0 monotonic==1.6 mpmath==1.3.0 multidict==6.0.5 mypy-extensions==1.0.0 numpy==1.26.4 oauthlib==3.2.2 onnxruntime==1.18.1 openai==1.40.3 opentelemetry-api==1.26.0 opentelemetry-exporter-otlp-proto-common==1.26.0 opentelemetry-exporter-otlp-proto-grpc==1.26.0 opentelemetry-instrumentation==0.47b0 opentelemetry-instrumentation-asgi==0.47b0 opentelemetry-instrumentation-fastapi==0.47b0 opentelemetry-proto==1.26.0 opentelemetry-sdk==1.26.0 opentelemetry-semantic-conventions==0.47b0 opentelemetry-util-http==0.47b0 orjson==3.10.7 overrides==7.7.0 packaging==24.1 posthog==3.5.0 protobuf==4.25.4 pyasn1==0.6.0 pyasn1_modules==0.4.0 pydantic==2.8.2 pydantic_core==2.20.1 Pygments==2.18.0 PyPika==0.48.9 pyproject_hooks==1.1.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 PyYAML==6.0.2 regex==2024.7.24 requests==2.32.3 requests-oauthlib==2.0.0 rich==13.7.1 rsa==4.9 shellingham==1.5.4 six==1.16.0 sniffio==1.3.1 soupsieve==2.5 SQLAlchemy==2.0.32 starlette==0.37.2 sympy==1.13.1 tenacity==8.5.0 tiktoken==0.7.0 tokenizers==0.20.0 tomli==2.0.1 tqdm==4.66.5 typer==0.12.3 typing-inspect==0.9.0 typing_extensions==4.12.2 urllib3==2.2.2 uvicorn==0.30.5 uvloop==0.19.0 watchfiles==0.23.0 websocket-client==1.8.0 websockets==12.0 wrapt==1.16.0 yarl==1.9.4 zipp==3.19.2 |
※ pip install python-dotenv langchain-community langchain-openai langchain-chroma bs4 명령을 실행했다.