[PYTHON/LANGCHAIN] GenericLoader 클래스 : 커스텀 로더 만들기

■ GenericLoader 클래스를 사용해 커스텀 로더를 만드는 방법을 보여준다.

▶ main.py


from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.document_loaders              import Blob
from typing                                       import Iterator
from langchain_core.documents                     import Document
from langchain_community.document_loaders.generic import GenericLoader
from typing                                       import Any

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

class CustomLoader(GenericLoader):
    @staticmethod
    def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:
        """Override this method to associate a default parser with the class."""
        return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):
    print(document)

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.generic import GenericLoader

from typing import Any

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

class CustomLoader(GenericLoader):

@staticmethod

def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:

"""Override this method to associate a default parser with the class."""

return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):

print(document)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain langchain-community tqdm 명령을 실행했다.