■ MapReduceDocumentsChain 클래스를 사용해 MAP-REDUCE 요약 체인을 만드는 방법을 보여준다.
▶ main.py
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import os from langchain_community.document_loaders import WebBaseLoader from langchain_openai import ChatOpenAI from langchain_core.prompts import PromptTemplate from langchain.chains.llm import LLMChain from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains import ReduceDocumentsChain, MapReduceDocumentsChain from langchain_text_splitters import CharacterTextSplitter os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>" # 문서 리스트를 로드한다. webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/") documentList = webBaseLoader.load() # 분할 문서 리스트를 만든다. characterTextSplitter = CharacterTextSplitter.from_tiktoken_encoder(chunk_size = 1000, chunk_overlap = 0) splitDocumentList = characterTextSplitter.split_documents(documentList) # LLM 모델을 설정한다. chatOpenAI = ChatOpenAI(temperature = 0) # MAP 프롬프트 템플리트를 설정한다. mapPromptTemplateString = """The following is a set of documents {docs} Based on this list of docs, please identify the main themes Helpful Answer : """ mapPromptTemplate = PromptTemplate.from_template(mapPromptTemplateString) # MAP LLM 체인을 설정한다. mapLLMChain = LLMChain(llm = chatOpenAI, prompt = mapPromptTemplate) # REDUCE 프롬프트 템플리트를 설정한다. reducePromptTemplateString = """The following is set of summaries : {docs} Take these and distill it into a final, consolidated summary of the main themes. Helpful Answer : """ reducePromptTemplate = PromptTemplate.from_template(reducePromptTemplateString) # REDUCE LLM 체인을 설정한다. reduceLLMChain = LLMChain(llm = chatOpenAI, prompt = reducePromptTemplate) # SUTFF 문서 체인을 설정한다. stuffDocumentsChain = StuffDocumentsChain(llm_chain = reduceLLMChain, document_variable_name = "docs") # REDUCE 문서 체인을 설정한다. reduceDocumentsChain = ReduceDocumentsChain( combine_documents_chain = stuffDocumentsChain, collapse_documents_chain = stuffDocumentsChain, token_max = 4000 ) # MAP REDUCE 문서 체인을 설정한다. mapReduceDocumentsChain = MapReduceDocumentsChain( llm_chain = mapLLMChain, reduce_documents_chain = reduceDocumentsChain, document_variable_name = "docs", return_intermediate_steps = False, ) # MAP REDUCE 문서 체인을 실행한다. resultDictionary = mapReduceDocumentsChain.invoke(splitDocumentList) print(resultDictionary["output_text"]) """ The main themes identified in the set of documents provided revolve around the utilization of large language models (LLMs) in autonomous agents for reasoning, problem-solving, and task execution. Key topics include feedback mechanisms, planning proficiency, reinforcement learning, neuro-symbolic architectures, tool augmentation, AI task delegation, chemistry tools, scientific research capabilities, generative agents, and prompt engineering. These themes highlight the diverse applications and potential of LLMs in various domains, emphasizing their role in enhancing performance and efficiency in complex tasks. """ |
▶ requirements.txt
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
aiohttp==3.9.5 aiosignal==1.3.1 annotated-types==0.7.0 anyio==4.4.0 async-timeout==4.0.3 attrs==23.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 certifi==2024.6.2 charset-normalizer==3.3.2 dataclasses-json==0.6.7 distro==1.9.0 exceptiongroup==1.2.1 frozenlist==1.4.1 greenlet==3.0.3 h11==0.14.0 httpcore==1.0.5 httpx==0.27.0 idna==3.7 jsonpatch==1.33 jsonpointer==3.0.0 langchain==0.2.5 langchain-community==0.2.5 langchain-core==0.2.7 langchain-openai==0.1.8 langchain-text-splitters==0.2.1 langsmith==0.1.77 marshmallow==3.21.3 multidict==6.0.5 mypy-extensions==1.0.0 numpy==1.26.4 openai==1.34.0 orjson==3.10.5 packaging==24.1 pydantic==2.7.4 pydantic_core==2.18.4 PyYAML==6.0.1 regex==2024.5.15 requests==2.32.3 sniffio==1.3.1 soupsieve==2.5 SQLAlchemy==2.0.30 tenacity==8.3.0 tiktoken==0.7.0 tqdm==4.66.4 typing-inspect==0.9.0 typing_extensions==4.12.2 urllib3==2.2.1 yarl==1.9.4 |
※ pip install langchain langchain-community langchain-openai bs4 명령을 실행했다.