DOCUMENT LOADER Archives

[PYTHON/LANGCHAIN] GenericLoader 클래스 : 커스텀 로더 만들기

■ GenericLoader 클래스를 사용해 커스텀 로더를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.document_loaders              import Blob
from typing                                       import Iterator
from langchain_core.documents                     import Document
from langchain_community.document_loaders.generic import GenericLoader
from typing                                       import Any

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

class CustomLoader(GenericLoader):
    @staticmethod
    def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:
        """Override this method to associate a default parser with the class."""
        return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):
    print(document)

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.generic import GenericLoader

from typing import Any

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

class CustomLoader(GenericLoader):

@staticmethod

def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:

"""Override this method to associate a default parser with the class."""

return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):

print(document)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain langchain-community tqdm 명령을

[PYTHON/LANGCHAIN] GenericLoader 클래스 : from_filesystem 메소드에서 path/glob/show_progress/parser 인자를 사용해 GenericLoader 객체 만들기

■ GenericLoader 클래스의 from_filesystem 메소드에서 path/glob/show_progress/parser 인자를 사용해 GenericLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.document_loaders              import Blob
from typing                                       import Iterator
from langchain_core.documents                     import Document
from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True, parser = CustomBlobParser())

generator = genericLoader.lazy_load()

for index, document in enumerate(generator):
    print(document)

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True, parser = CustomBlobParser())

generator = genericLoader.lazy_load()

for index, document in enumerate(generator):

print(document)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install

[PYTHON/LANGCHAIN] FileSystemBlobLoader 클래스 : 생성자에서 path/glob/show_progress 인자를 사용해 FileSystemBlobLoader 객체 만들기

■ FileSystemBlobLoader 클래스의 생성자에서 path/glob/show_progress 인자를 사용해 FileSystemBlobLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders                   import BaseBlobParser
from langchain_core.document_loaders                   import Blob
from typing                                            import Iterator
from langchain_core.documents                          import Document
from langchain_community.document_loaders.blob_loaders import FileSystemBlobLoader

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

fileSystemBlobLoader = FileSystemBlobLoader(path = ".", glob = "*.txt", show_progress = True)
customBlobParser     = CustomBlobParser()

for blob in fileSystemBlobLoader.yield_blobs():
    for document in customBlobParser.lazy_parse(blob):
        print(document)
        break

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.blob_loaders import FileSystemBlobLoader

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

fileSystemBlobLoader = FileSystemBlobLoader(path = ".", glob = "*.txt", show_progress = True)

customBlobParser = CustomBlobParser()

for blob in fileSystemBlobLoader.yield_blobs():

for document in customBlobParser.lazy_parse(blob):

print(document)

break

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain

[PYTHON/LANGCHAIN] Blob 클래스 : as_string 메소드를 사용해 문자열 구하기

■ Blob 클래스의 as_string 메소드를 사용해 문자열을 구하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

string = blob.as_string()

print(string)

"""
meow meow1
meow meow2
meow meow3
"""

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

string = blob.as_string()

print(string)

"""

meow meow1

meow meow2

meow meow3

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다.

[PYTHON/LANGCHAIN] Blob 클래스 : as_bytes 메소드를 사용해 bytes 객체 구하기

■ Blob 클래스의 as_bytes 메소드를 사용해 bytes 객체를 구하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

bytes1 = blob.as_bytes()

print(bytes1)

"""
b'meow meow1\r\nmeow meow2\r\nmeow meow3'
"""

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

bytes1 = blob.as_bytes()

print(bytes1)

"""

b'meow meow1\r\nmeow meow2\r\nmeow meow3'

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을

[PYTHON/LANGCHAIN] Blob 클래스 : encoding/source/metadata 속성 사용하기

■ Blob 클래스의 encoding/source/metadata 속성을 사용하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

print(blob.encoding)
print(blob.source)
print(blob.metadata)

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

print(blob.encoding)

print(blob.source)

print(blob.metadata)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다. meow.txt

[PYTHON/LANGCHAIN] Blob 클래스 : from_path 정적 메소드를 사용해 파일 로드하기

■ Blob 클래스의 from_path 정적 메소드를 사용해 파일을 로드하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을

[PYTHON/LANGCHAIN] BaseBlobParser 클래스 : 커스텀 BLOB 파서 만들기

■ BaseBlobParser 클래스를 사용해 커스텀 BLOB 파서를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import BaseBlobParser
from langchain_core.document_loaders import Blob
from typing                          import Iterator
from langchain_core.documents        import Document

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

blob1 = Blob.from_path("meow.txt")

customBlobParser = CustomBlobParser()

generator1 = customBlobParser.lazy_parse(blob1)

documentList1 = list(generator1)

for document in documentList1:
    print(document)
print("-" * 50)

blob2 = Blob(data = b"some data from memory\nmeow")

generator2 = customBlobParser.lazy_parse(blob2)

documentList2 = list(generator2)

for document in documentList2:
    print(document)

"""
page_content='meow meow1' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 2, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 3, 'source': './meow.txt'}
--------------------------------------------------
page_content='some data from memory' metadata={'line_number': 1, 'source': None}
page_content='meow' metadata={'line_number': 2, 'source': None}
"""

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

blob1 = Blob.from_path("meow.txt")

customBlobParser = CustomBlobParser()

generator1 = customBlobParser.lazy_parse(blob1)

documentList1 = list(generator1)

for document in documentList1:

print(document)

print("-" * 50)

blob2 = Blob(data = b"some data from memory\nmeow")

generator2 = customBlobParser.lazy_parse(blob2)

documentList2 = list(generator2)

for document in documentList2:

print(document)

"""

page_content='meow meow1' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 2, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 3, 'source': './meow.txt'}

--------------------------------------------------

page_content='some data from memory' metadata={'line_number': 1, 'source': None}

page_content='meow' metadata={'line_number': 2, 'source': None}

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다.

[PYTHON/LANGCHAIN] BaseLoader 클래스 : 커스텀 문서 로더 만들기

■ BaseLoader 클래스를 사용해 커스텀 문서 로더를 만드는 방법을 보여준다. ▶ main.py


import aiofiles
import asyncio

from langchain_core.document_loaders import BaseLoader
from typing                          import Iterator
from langchain_core.documents        import Document
from typing                          import AsyncIterator

class CustomDocumentLoader(BaseLoader):
    """An example document loader that reads a file line by line."""

    def __init__(self, filePath : str) -> None:
        """Initialize the loader with a file path.

        Args :
            file_path: The path to the file to load.
        """
        self.filePath = filePath

    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader that reads a file line by line.

        When you're implementing lazy load methods, you should use a generator to yield documents one by one.
        """
        with open(self.filePath, encoding = "utf-8") as textIOWrapper:
            lineNumber = 0
            for line in textIOWrapper:
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : self.filePath}
                )
                lineNumber += 1

    # alazy_load는 선택 사항이다.
    # 구현을 생략하면 lazy_load에 위임하는 기본 구현이 사용된다!
    async def alazy_load(self) -> AsyncIterator[Document]:
        """An async lazy loader that reads a file line by line."""

        async with aiofiles.open(self.filePath, encoding = "utf-8") as asyncTextIOWrapper:
            lineNumber = 0
            async for line in asyncTextIOWrapper:
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : self.filePath}
                )
                lineNumber += 1

with open("./meow.txt", "w", encoding = "utf-8") as textIOWrapper:
    fileContent = "meow meow1\nmeow meow2\nmeow meow3"
    textIOWrapper.write(fileContent)

customDocumentLoader = CustomDocumentLoader("./meow.txt")

for document in customDocumentLoader.lazy_load():
    print(document)
print("-" * 50)

documentList = customDocumentLoader.load()
for document in documentList:
    print(document)
print("-" * 50)

async def main():
    async for document in customDocumentLoader.alazy_load():
        print(document)
    print("-" * 50)

asyncio.run(main())

"""
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
"""

import aiofiles

import asyncio

from langchain_core.document_loaders import BaseLoader

from typing import Iterator

from langchain_core.documents import Document

from typing import AsyncIterator

class CustomDocumentLoader(BaseLoader):

"""An example document loader that reads a file line by line."""

def __init__(self, filePath : str) -> None:

"""Initialize the loader with a file path.

Args :

file_path: The path to the file to load.

"""

self.filePath = filePath

def lazy_load(self) -> Iterator[Document]:

"""A lazy loader that reads a file line by line.

When you're implementing lazy load methods, you should use a generator to yield documents one by one.

"""

with open(self.filePath, encoding = "utf-8") as textIOWrapper:

lineNumber = 0

for line in textIOWrapper:

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : self.filePath}

)

lineNumber += 1

# alazy_load는 선택 사항이다.

# 구현을 생략하면 lazy_load에 위임하는 기본 구현이 사용된다!

async def alazy_load(self) -> AsyncIterator[Document]:

"""An async lazy loader that reads a file line by line."""

async with aiofiles.open(self.filePath, encoding = "utf-8") as asyncTextIOWrapper:

lineNumber = 0

async for line in asyncTextIOWrapper:

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : self.filePath}

)

lineNumber += 1

with open("./meow.txt", "w", encoding = "utf-8") as textIOWrapper:

fileContent = "meow meow1\nmeow meow2\nmeow meow3"

textIOWrapper.write(fileContent)

customDocumentLoader = CustomDocumentLoader("./meow.txt")

for document in customDocumentLoader.lazy_load():

print(document)

print("-" * 50)

documentList = customDocumentLoader.load()

for document in documentList:

print(document)

print("-" * 50)

async def main():

async for document in customDocumentLoader.alazy_load():

print(document)

print("-" * 50)

asyncio.run(main())

"""

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

"""

▶ requirements.txt


aiofiles==24.1.0
aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.11
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiofiles==24.1.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.11

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install aiofiles langchain 명령을

[PYTHON/LANGCHAIN] TextLoader 클래스 : 복수 텍스트 파일 문서 로드하기

■ TextLoader 클래스를 사용해 복수 텍스트 파일 문서를 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

TextLoaderList = [
    TextLoader("paul_graham_essay.txt" , encoding = "utf-8"),
    TextLoader("state_of_the_union.txt", encoding = "utf-8")
]

documentList = []

for textLoader in TextLoaderList:
    documentList.extend(textLoader.load())

from langchain_community.document_loaders import TextLoader

TextLoaderList = [

TextLoader("paul_graham_essay.txt" , encoding = "utf-8"),

TextLoader("state_of_the_union.txt", encoding = "utf-8")

]

documentList = []

for textLoader in TextLoaderList:

documentList.extend(textLoader.load())

※ pip install langchain-community 명령을 실행했다.

[PYTHON/LANGCHAIN] TextLoader 클래스 : 생성자에서 encoding 인자를 사용해 UTF-8 인코딩 파일 로드하기

■ TextLoader 클래스의 생성자에서 encoding 인자를 사용해 UTF-8 인코딩 파일을 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("paul_graham_essay.txt" , encoding = "utf-8")

from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("paul_graham_essay.txt" , encoding = "utf-8")

※ pip install langchain-community

[PYTHON/LANGCHAIN] Docx2txtLoader 클래스 : load 메소드를 사용해 MS WORD 문서 로드하기

■ Docx2txtLoader 클래스의 load 메소드를 사용해 MS WORD 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import Docx2txtLoader

docx2txtLoader = Docx2txtLoader("sample.docx")

documentList = docx2txtLoader.load()

document = documentList[0]

print(document.page_content)

from langchain_community.document_loaders import Docx2txtLoader

docx2txtLoader = Docx2txtLoader("sample.docx")

documentList = docx2txtLoader.load()

document = documentList[0]

print(document.page_content)

▶ requirements.txt


aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.3.2
dataclasses-json==0.6.7
docx2txt==0.8
frozenlist==1.4.1
greenlet==3.1.0
h11==0.14.0
httpcore==1.0.5
httpx==0.27.2
idna==3.8
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.16
langchain-community==0.2.16
langchain-core==0.2.39
langchain-text-splitters==0.2.4
langsmith==0.1.118
marshmallow==3.22.0
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.7
packaging==24.1
pydantic==2.9.1
pydantic_core==2.23.3
PyYAML==6.0.2
requests==2.32.3
sniffio==1.3.1
SQLAlchemy==2.0.34
tenacity==8.5.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.11.1

aiohappyeyeballs==2.4.0

aiohttp==3.10.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.3.2

dataclasses-json==0.6.7

docx2txt==0.8

frozenlist==1.4.1

greenlet==3.1.0

h11==0.14.0

httpcore==1.0.5

httpx==0.27.2

idna==3.8

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.16

langchain-community==0.2.16

langchain-core==0.2.39

langchain-text-splitters==0.2.4

langsmith==0.1.118

marshmallow==3.22.0

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.7

packaging==24.1

pydantic==2.9.1

pydantic_core==2.23.3

PyYAML==6.0.2

requests==2.32.3

sniffio==1.3.1

SQLAlchemy==2.0.34

tenacity==8.5.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.11.1

※ pip install langchain-community

[PYTHON/LANGCHAIN] UnstructuredURLLoader 클래스 : 생성자에서 urls 인자를 사용해 UnstructuredURLLoader 객체 만들기

■ UnstructuredURLLoader 클래스의 생성자에서 urls 인자를 사용해 UnstructuredURLLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import UnstructuredURLLoader

urlList = [
    "https://n.news.naver.com/news/articles/092/0002307222?sid=105",
    "https://n.news.naver.com/news/articles/052/0001944792?sid=105"
]

unstructuredURLLoader = UnstructuredURLLoader(urls = urlList)

documentList = unstructuredURLLoader.load()

document = documentList[0]

print(document.page_content)

from langchain_community.document_loaders import UnstructuredURLLoader

urlList = [

"https://n.news.naver.com/news/articles/092/0002307222?sid=105",

"https://n.news.naver.com/news/articles/052/0001944792?sid=105"

]

unstructuredURLLoader = UnstructuredURLLoader(urls = urlList)

documentList = unstructuredURLLoader.load()

document = documentList[0]

print(document.page_content)

▶ requirements.txt


aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
attrs==24.2.0
backoff==2.2.1
beautifulsoup4==4.12.3
certifi==2024.8.30
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
colorama==0.4.6
dataclasses-json==0.6.7
deepdiff==8.0.1
emoji==2.12.1
filetype==1.2.0
frozenlist==1.4.1
greenlet==3.1.0
h11==0.14.0
httpcore==1.0.5
httpx==0.27.2
idna==3.8
joblib==1.4.2
jsonpatch==1.33
jsonpath-python==1.0.6
jsonpointer==3.0.0
langchain==0.2.16
langchain-community==0.2.16
langchain-core==0.2.39
langchain-text-splitters==0.2.4
langdetect==1.0.9
langsmith==0.1.118
lxml==5.3.0
marshmallow==3.22.0
multidict==6.1.0
mypy-extensions==1.0.0
nest-asyncio==1.6.0
nltk==3.9.1
numpy==1.26.4
olefile==0.47
orderly-set==5.2.2
orjson==3.10.7
packaging==24.1
psutil==6.0.0
pydantic==2.9.1
pydantic_core==2.23.3
pypdf==4.3.1
python-dateutil==2.9.0.post0
python-iso639==2024.4.27
python-magic==0.4.27
python-oxmsg==0.0.1
PyYAML==6.0.2
rapidfuzz==3.9.7
regex==2024.9.11
requests==2.32.3
requests-toolbelt==1.0.0
six==1.16.0
sniffio==1.3.1
soupsieve==2.6
SQLAlchemy==2.0.34
tabulate==0.9.0
tenacity==8.5.0
tqdm==4.66.5
typing-inspect==0.9.0
typing_extensions==4.12.2
unstructured==0.15.10
unstructured-client==0.25.8
urllib3==2.2.2
wrapt==1.16.0
yarl==1.11.1

aiohappyeyeballs==2.4.0

aiohttp==3.10.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

attrs==24.2.0

backoff==2.2.1

beautifulsoup4==4.12.3

certifi==2024.8.30

chardet==5.2.0

charset-normalizer==3.3.2

click==8.1.7

colorama==0.4.6

dataclasses-json==0.6.7

deepdiff==8.0.1

emoji==2.12.1

filetype==1.2.0

frozenlist==1.4.1

greenlet==3.1.0

h11==0.14.0

httpcore==1.0.5

httpx==0.27.2

idna==3.8

joblib==1.4.2

jsonpatch==1.33

jsonpath-python==1.0.6

jsonpointer==3.0.0

langchain==0.2.16

langchain-community==0.2.16

langchain-core==0.2.39

langchain-text-splitters==0.2.4

langdetect==1.0.9

langsmith==0.1.118

lxml==5.3.0

marshmallow==3.22.0

multidict==6.1.0

mypy-extensions==1.0.0

nest-asyncio==1.6.0

nltk==3.9.1

numpy==1.26.4

olefile==0.47

orderly-set==5.2.2

orjson==3.10.7

packaging==24.1

psutil==6.0.0

pydantic==2.9.1

pydantic_core==2.23.3

pypdf==4.3.1

python-dateutil==2.9.0.post0

python-iso639==2024.4.27

python-magic==0.4.27

python-oxmsg==0.0.1

PyYAML==6.0.2

rapidfuzz==3.9.7

regex==2024.9.11

requests==2.32.3

requests-toolbelt==1.0.0

six==1.16.0

sniffio==1.3.1

soupsieve==2.6

SQLAlchemy==2.0.34

tabulate==0.9.0

tenacity==8.5.0

tqdm==4.66.5

typing-inspect==0.9.0

typing_extensions==4.12.2

unstructured==0.15.10

unstructured-client==0.25.8

urllib3==2.2.2

wrapt==1.16.0

yarl==1.11.1

※ pip install langchain-community

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : invoke 메소드 사용하기 2

■ MultiQueryRetriever 클래스의 invoke 메소드를 사용하는 방법을 보여준다. 2 ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ main.py


from dotenv import load_dotenv
from typing import List

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter
from langchain_openai                     import OpenAIEmbeddings
from langchain_chroma                     import Chroma
from langchain_core.prompts               import PromptTemplate
from langchain_openai                     import ChatOpenAI
from langchain_core.output_parsers        import BaseOutputParser
from langchain.retrievers.multi_query     import MultiQueryRetriever

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitdocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitdocumentList, embedding = openAIEmbeddings)

promptTemplate = PromptTemplate(
    input_variables = ["question"],
    template        = """You are an AI language model assistant.
Your task is to generate five different versions of the given user question to retrieve relevant documents from a vector database.
By generating multiple perspectives on the user question, your goal is to help the user overcome some of the limitations of the distance-based similarity search.
Provide these alternative questions separated by newlines.
Original question : {question}"""
)

chatOpenAI = ChatOpenAI(temperature = 0)

class LineListOutputParser(BaseOutputParser[List[str]]):
    """Output parser for a list of lines."""

    def parse(self, text : str) -> List[str]:
        lineList = text.strip().split("\n")
        return list(filter(None, lineList))

lineListOutputParser = LineListOutputParser()

runnableSequence = promptTemplate | chatOpenAI | lineListOutputParser

rultiQueryRetriever = MultiQueryRetriever(retriever = chroma.as_retriever(), llm_chain = runnableSequence, parser_key = "lines")

responseDocumentList = rultiQueryRetriever.invoke("What does the course say about regression?")

print(len(responseDocumentList))

from dotenv import load_dotenv

from typing import List

from langchain_community.document_loaders import WebBaseLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

from langchain_openai import OpenAIEmbeddings

from langchain_chroma import Chroma

from langchain_core.prompts import PromptTemplate

from langchain_openai import ChatOpenAI

from langchain_core.output_parsers import BaseOutputParser

from langchain.retrievers.multi_query import MultiQueryRetriever

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitdocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitdocumentList, embedding = openAIEmbeddings)

promptTemplate = PromptTemplate(

input_variables = ["question"],

template = """You are an AI language model assistant.

Your task is to generate five different versions of the given user question to retrieve relevant documents from a vector database.

By generating multiple perspectives on the user question, your goal is to help the user overcome some of the limitations of the distance-based similarity search.

Provide these alternative questions separated by newlines.

Original question : {question}"""

)

chatOpenAI = ChatOpenAI(temperature = 0)

class LineListOutputParser(BaseOutputParser[List[str]]):

"""Output parser for a list of lines."""

def parse(self, text : str) -> List[str]:

lineList = text.strip().split("\n")

return list(filter(None, lineList))

lineListOutputParser = LineListOutputParser()

runnableSequence = promptTemplate | chatOpenAI | lineListOutputParser

rultiQueryRetriever = MultiQueryRetriever(retriever = chroma.as_retriever(), llm_chain = runnableSequence, parser_key = "lines")

responseDocumentList = rultiQueryRetriever.invoke("What does the course say about regression?")

print(len(responseDocumentList))

▶ requirements.txt

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : invoke 메소드 사용하기 1

■ MultiQueryRetriever 클래스의 invoke 메소드를 사용하는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ main.py


import logging

from dotenv import load_dotenv

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter
from langchain_openai                     import OpenAIEmbeddings
from langchain_chroma                     import Chroma
from langchain_openai                     import ChatOpenAI
from langchain.retrievers.multi_query     import MultiQueryRetriever

logging.basicConfig()

logger = logging.getLogger("langchain.retrievers.multi_query")

logger.setLevel(logging.INFO)

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitDocumentList, embedding = openAIEmbeddings)

vectorStoreRetriever = chroma.as_retriever()

chatOpenAI = ChatOpenAI(temperature = 0)

multiQueryRetriever = MultiQueryRetriever.from_llm(retriever = vectorStoreRetriever, llm = chatOpenAI)

question = "What are the approaches to Task Decomposition?"

responseDocumentList = multiQueryRetriever.invoke(question)

print(len(responseDocumentList))

print(responseDocumentList[0].page_content)

"""
5
Task decomposition can be done (1) by LLM with simple prompting like "Steps for XYZ.\n1.", "What are the subgoals for achieving XYZ?", (2) by using task-specific instructions; e.g. "Write a story outline." for writing a novel, or (3) with human inputs.
"""

import logging

from dotenv import load_dotenv

from langchain_community.document_loaders import WebBaseLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

from langchain_openai import OpenAIEmbeddings

from langchain_chroma import Chroma

from langchain_openai import ChatOpenAI

from langchain.retrievers.multi_query import MultiQueryRetriever

logging.basicConfig()

logger = logging.getLogger("langchain.retrievers.multi_query")

logger.setLevel(logging.INFO)

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitDocumentList, embedding = openAIEmbeddings)

vectorStoreRetriever = chroma.as_retriever()

chatOpenAI = ChatOpenAI(temperature = 0)

multiQueryRetriever = MultiQueryRetriever.from_llm(retriever = vectorStoreRetriever, llm = chatOpenAI)

question = "What are the approaches to Task Decomposition?"

responseDocumentList = multiQueryRetriever.invoke(question)

print(len(responseDocumentList))

print(responseDocumentList[0].page_content)

"""

Task decomposition can be done (1) by LLM with simple prompting like "Steps for XYZ.\n1.", "What are the subgoals for achieving XYZ?", (2) by using task-specific instructions; e.g. "Write a story outline." for writing a novel, or (3) with human inputs.

"""

▶ requirements.txt


aiohappyeyeballs==2.3.5
aiohttp==3.10.3
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
backoff==2.2.1
bcrypt==4.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
build==1.2.1
cachetools==5.4.0
certifi==2024.7.4
charset-normalizer==3.3.2
chroma-hnswlib==0.7.6
chromadb==0.5.5
click==8.1.7
coloredlogs==15.0.1
dataclasses-json==0.6.7
Deprecated==1.2.14
distro==1.9.0
exceptiongroup==1.2.2
fastapi==0.112.0
filelock==3.15.4
flatbuffers==24.3.25
frozenlist==1.4.1
fsspec==2024.6.1
google-auth==2.33.0
googleapis-common-protos==1.63.2
greenlet==3.0.3
grpcio==1.65.4
h11==0.14.0
httpcore==1.0.5
httptools==0.6.1
httpx==0.27.0
huggingface-hub==0.24.5
humanfriendly==10.0
idna==3.7
importlib_metadata==8.0.0
importlib_resources==6.4.0
jiter==0.5.0
jsonpatch==1.33
jsonpointer==3.0.0
kubernetes==30.1.0
langchain==0.2.12
langchain-chroma==0.1.2
langchain-community==0.2.11
langchain-core==0.2.29
langchain-openai==0.1.21
langchain-text-splitters==0.2.2
langsmith==0.1.98
markdown-it-py==3.0.0
marshmallow==3.21.3
mdurl==0.1.2
mmh3==4.1.0
monotonic==1.6
mpmath==1.3.0
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
oauthlib==3.2.2
onnxruntime==1.18.1
openai==1.40.3
opentelemetry-api==1.26.0
opentelemetry-exporter-otlp-proto-common==1.26.0
opentelemetry-exporter-otlp-proto-grpc==1.26.0
opentelemetry-instrumentation==0.47b0
opentelemetry-instrumentation-asgi==0.47b0
opentelemetry-instrumentation-fastapi==0.47b0
opentelemetry-proto==1.26.0
opentelemetry-sdk==1.26.0
opentelemetry-semantic-conventions==0.47b0
opentelemetry-util-http==0.47b0
orjson==3.10.7
overrides==7.7.0
packaging==24.1
posthog==3.5.0
protobuf==4.25.4
pyasn1==0.6.0
pyasn1_modules==0.4.0
pydantic==2.8.2
pydantic_core==2.20.1
Pygments==2.18.0
PyPika==0.48.9
pyproject_hooks==1.1.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
PyYAML==6.0.2
regex==2024.7.24
requests==2.32.3
requests-oauthlib==2.0.0
rich==13.7.1
rsa==4.9
shellingham==1.5.4
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.32
starlette==0.37.2
sympy==1.13.1
tenacity==8.5.0
tiktoken==0.7.0
tokenizers==0.20.0
tomli==2.0.1
tqdm==4.66.5
typer==0.12.3
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
uvicorn==0.30.5
uvloop==0.19.0
watchfiles==0.23.0
websocket-client==1.8.0
websockets==12.0
wrapt==1.16.0
yarl==1.9.4
zipp==3.19.2

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

aiohappyeyeballs==2.3.5

aiohttp==3.10.3

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

asgiref==3.8.1

async-timeout==4.0.3

attrs==24.2.0

backoff==2.2.1

bcrypt==4.2.0

beautifulsoup4==4.12.3

bs4==0.0.2

build==1.2.1

cachetools==5.4.0

certifi==2024.7.4

charset-normalizer==3.3.2

chroma-hnswlib==0.7.6

chromadb==0.5.5

click==8.1.7

coloredlogs==15.0.1

dataclasses-json==0.6.7

Deprecated==1.2.14

distro==1.9.0

exceptiongroup==1.2.2

fastapi==0.112.0

filelock==3.15.4

flatbuffers==24.3.25

frozenlist==1.4.1

fsspec==2024.6.1

google-auth==2.33.0

googleapis-common-protos==1.63.2

greenlet==3.0.3

grpcio==1.65.4

h11==0.14.0

httpcore==1.0.5

httptools==0.6.1

httpx==0.27.0

huggingface-hub==0.24.5

humanfriendly==10.0

idna==3.7

importlib_metadata==8.0.0

importlib_resources==6.4.0

jiter==0.5.0

jsonpatch==1.33

jsonpointer==3.0.0

kubernetes==30.1.0

langchain==0.2.12

langchain-chroma==0.1.2

langchain-community==0.2.11

langchain-core==0.2.29

langchain-openai==0.1.21

langchain-text-splitters==0.2.2

langsmith==0.1.98

markdown-it-py==3.0.0

marshmallow==3.21.3

mdurl==0.1.2

mmh3==4.1.0

monotonic==1.6

mpmath==1.3.0

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

oauthlib==3.2.2

onnxruntime==1.18.1

openai==1.40.3

opentelemetry-api==1.26.0

opentelemetry-exporter-otlp-proto-common==1.26.0

opentelemetry-exporter-otlp-proto-grpc==1.26.0

opentelemetry-instrumentation==0.47b0

opentelemetry-instrumentation-asgi==0.47b0

opentelemetry-instrumentation-fastapi==0.47b0

opentelemetry-proto==1.26.0

opentelemetry-sdk==1.26.0

opentelemetry-semantic-conventions==0.47b0

opentelemetry-util-http==0.47b0

orjson==3.10.7

overrides==7.7.0

packaging==24.1

posthog==3.5.0

protobuf==4.25.4

pyasn1==0.6.0

pyasn1_modules==0.4.0

pydantic==2.8.2

pydantic_core==2.20.1

Pygments==2.18.0

PyPika==0.48.9

pyproject_hooks==1.1.0

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

PyYAML==6.0.2

regex==2024.7.24

requests==2.32.3

requests-oauthlib==2.0.0

rich==13.7.1

rsa==4.9

shellingham==1.5.4

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.32

starlette==0.37.2

sympy==1.13.1

tenacity==8.5.0

tiktoken==0.7.0

tokenizers==0.20.0

tomli==2.0.1

tqdm==4.66.5

typer==0.12.3

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

uvicorn==0.30.5

uvloop==0.19.0

watchfiles==0.23.0

websocket-client==1.8.0

websockets==12.0

wrapt==1.16.0

yarl==1.9.4

zipp==3.19.2

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : 생성자에서 retriever/llm_chain/parser_key 인자를 사용해 MultiQueryRetriever 객체 만들기

■ MultiQueryRetriever 클래스의 생성자에서 retriever/llm_chain/parser_key 인자를 사용해 MultiQueryRetriever 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ 예제

[PYTHON/LANGCHAIN] from_llm 함수 : retriever/llm 인자를 사용해 multiQueryRetriever 객체 만들기

■ from_llm 함수의 retriever/llm 인자를 사용해 multiQueryRetriever 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ 예제 코드

[PYTHON/LANGCHAIN] from_documents 함수 : documents/embedding 인자를 사용해 메모리 내 벡터 저장소를 위한 Chroma 객체 만들기

■ from_documents 함수의 documents/embedding 인자를 사용해 메모리 내 벡터 저장소를 위한 Chroma 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env

[PYTHON/LANGCHAIN] WebBaseLoader 클래스 : 생성자에서 web_paths/bs_kwargs 인자를 사용해 웹 문서 로더 만들기

■ WebBaseLoader 클래스의 생성자에서 web_paths/bs_kwargs 인자를 사용해 웹 문서 로더를 만드는 방법을 보여준다. ▶ main.py


import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(
    web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),
    bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))
)

import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(

web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),

bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))

)

[PYTHON/LANGCHAIN] WebBaseLoader 클래스 : load 메소드를 사용해 웹 문서 로드하기

■ WebBaseLoader 클래스의 load 메소드를 사용해 웹 문서를 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

[PYTHON/LANGCHAIN] WebBaseLoader 클래스 : 웹 문서 로더 만들기

■ WebBaseLoader 클래스를 사용해 웹 문서 로더를 만드는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

[PYTHON/LANGCHAIN] TextLoader 클래스 : load 메소드를 사용해 텍스트 파일 로드하기

■ TextLoader 클래스의 load 메소드를 사용해 텍스트 파일을 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("state_of_the_union.txt")

documentList = textLoader.load()

from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("state_of_the_union.txt")

documentList = textLoader.load()

[PYTHON/LANGCHAIN] TextLoader 클래스 : 텍스트 로더 만들기

■ TextLoader 클래스를 사용해 텍스트 로더를 만드는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("state_of_the_union.txt")

from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("state_of_the_union.txt")

[PYTHON/LANGCHAIN] GenericLoader 클래스 : 커스텀 일반 로더 만들기

■ GenericLoader 클래스를 사용해 커스텀 일반 로더를 만드는 방법을 보여준다. ▶ main.py


from typing import Iterator

from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.documents                     import Document
from langchain_core.document_loaders              import Blob
from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):
    """각 라인에서 문서를 생성하는 간단한 파서이다."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Blob을 한 줄씩 문서로 구문 분석한다."""

        lineNumber = 0

        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(page_content = line, metadata = {"line_number" : lineNumber, "source" : blob.source})

from typing import Any

class CustomGenericLoader(GenericLoader):
    @staticmethod
    def get_parser(**kwargs: Any) -> BaseBlobParser:
        """기본 파서를 클래스와 연결하려면 이 메서드를 재정의한다."""
        return CustomBlobParser()

customGenericLoader = CustomGenericLoader.from_filesystem(path = ".", glob = "*.md")

for index, document in enumerate(customGenericLoader.lazy_load()):
    if index < 5:
        print(document)

"""
page_content='# 제목 1단계\r\n' metadata={'line_number': 1, 'source': 'header.md'}
page_content='## 제목 2단계  \r\n' metadata={'line_number': 2, 'source': 'header.md'}
page_content='### 제목 3단계\r\n' metadata={'line_number': 3, 'source': 'header.md'}
page_content='#### 제목 4단계\r\n' metadata={'line_number': 4, 'source': 'header.md'}
page_content='##### 제목 5단계\r\n' metadata={'line_number': 5, 'source': 'header.md'}
"""

from typing import Iterator

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.documents import Document

from langchain_core.document_loaders import Blob

from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):

"""각 라인에서 문서를 생성하는 간단한 파서이다."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Blob을 한 줄씩 문서로 구문 분석한다."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(page_content = line, metadata = {"line_number" : lineNumber, "source" : blob.source})

from typing import Any

class CustomGenericLoader(GenericLoader):

@staticmethod

def get_parser(**kwargs: Any) -> BaseBlobParser:

"""기본 파서를 클래스와 연결하려면 이 메서드를 재정의한다."""

return CustomBlobParser()

customGenericLoader = CustomGenericLoader.from_filesystem(path = ".", glob = "*.md")

for index, document in enumerate(customGenericLoader.lazy_load()):

if index < 5:

print(document)

"""

page_content='# 제목 1단계\r\n' metadata={'line_number': 1, 'source': 'header.md'}

page_content='## 제목 2단계 \r\n' metadata={'line_number': 2, 'source': 'header.md'}

page_content='### 제목 3단계\r\n' metadata={'line_number': 3, 'source': 'header.md'}

page_content='#### 제목 4단계\r\n' metadata={'line_number': 4, 'source': 'header.md'}

page_content='##### 제목 5단계\r\n' metadata={'line_number': 5, 'source': 'header.md'}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install langchain-community 명령을 실행했다.

[PYTHON/LANGCHAIN] GenericLoader 클래스 : lazy_load 메소드를 사용해 문서 로드하기

■ GenericLoader 클래스의 lazy_load 메소드를 사용해 문서를 로드하는 방법을 보여준다. ▶ main.py


from typing import Iterator

from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.documents                     import Document
from langchain_core.document_loaders              import Blob
from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):
    """각 라인에서 문서를 생성하는 간단한 파서이다."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Blob을 한 줄씩 문서로 구문 분석한다."""

        lineNumber = 0

        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(page_content = line, metadata = {"line_number" : lineNumber, "source" : blob.source})

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.md", parser = CustomBlobParser())

for index, document in enumerate(genericLoader.lazy_load()):
    if index < 5:
        print(document)

"""
page_content='# 제목 1단계\r\n' metadata={'line_number': 1, 'source': 'header.md'}
page_content='## 제목 2단계  \r\n' metadata={'line_number': 2, 'source': 'header.md'}
page_content='### 제목 3단계\r\n' metadata={'line_number': 3, 'source': 'header.md'}
page_content='#### 제목 4단계\r\n' metadata={'line_number': 4, 'source': 'header.md'}
page_content='##### 제목 5단계\r\n' metadata={'line_number': 5, 'source': 'header.md'}
"""

from typing import Iterator

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.documents import Document

from langchain_core.document_loaders import Blob

from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):

"""각 라인에서 문서를 생성하는 간단한 파서이다."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Blob을 한 줄씩 문서로 구문 분석한다."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(page_content = line, metadata = {"line_number" : lineNumber, "source" : blob.source})

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.md", parser = CustomBlobParser())

for index, document in enumerate(genericLoader.lazy_load()):

if index < 5:

print(document)

"""

page_content='# 제목 1단계\r\n' metadata={'line_number': 1, 'source': 'header.md'}

page_content='## 제목 2단계 \r\n' metadata={'line_number': 2, 'source': 'header.md'}

page_content='### 제목 3단계\r\n' metadata={'line_number': 3, 'source': 'header.md'}

page_content='#### 제목 4단계\r\n' metadata={'line_number': 4, 'source': 'header.md'}

page_content='##### 제목 5단계\r\n' metadata={'line_number': 5, 'source': 'header.md'}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install langchain-community 명령을 실행했다.