DOCUMENT Archives - icodebroker

[PYTHON/LANGCHAIN] GenericLoader 클래스 : 커스텀 로더 만들기

■ GenericLoader 클래스를 사용해 커스텀 로더를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.document_loaders              import Blob
from typing                                       import Iterator
from langchain_core.documents                     import Document
from langchain_community.document_loaders.generic import GenericLoader
from typing                                       import Any

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

class CustomLoader(GenericLoader):
    @staticmethod
    def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:
        """Override this method to associate a default parser with the class."""
        return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):
    print(document)

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.generic import GenericLoader

from typing import Any

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

class CustomLoader(GenericLoader):

@staticmethod

def get_parser(**keywordArgumentArray : Any) -> BaseBlobParser:

"""Override this method to associate a default parser with the class."""

return CustomBlobParser()

customLoader = CustomLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True)

generator = customLoader.lazy_load()

for index, document in enumerate(generator):

print(document)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain langchain-community tqdm 명령을

[PYTHON/LANGCHAIN] GenericLoader 클래스 : from_filesystem 메소드에서 path/glob/show_progress/parser 인자를 사용해 GenericLoader 객체 만들기

■ GenericLoader 클래스의 from_filesystem 메소드에서 path/glob/show_progress/parser 인자를 사용해 GenericLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders              import BaseBlobParser
from langchain_core.document_loaders              import Blob
from typing                                       import Iterator
from langchain_core.documents                     import Document
from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True, parser = CustomBlobParser())

generator = genericLoader.lazy_load()

for index, document in enumerate(generator):
    print(document)

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.generic import GenericLoader

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

genericLoader = GenericLoader.from_filesystem(path = ".", glob = "*.txt", show_progress = True, parser = CustomBlobParser())

generator = genericLoader.lazy_load()

for index, document in enumerate(generator):

print(document)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install

[PYTHON/LANGCHAIN] FileSystemBlobLoader 클래스 : 생성자에서 path/glob/show_progress 인자를 사용해 FileSystemBlobLoader 객체 만들기

■ FileSystemBlobLoader 클래스의 생성자에서 path/glob/show_progress 인자를 사용해 FileSystemBlobLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders                   import BaseBlobParser
from langchain_core.document_loaders                   import Blob
from typing                                            import Iterator
from langchain_core.documents                          import Document
from langchain_community.document_loaders.blob_loaders import FileSystemBlobLoader

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

fileSystemBlobLoader = FileSystemBlobLoader(path = ".", glob = "*.txt", show_progress = True)
customBlobParser     = CustomBlobParser()

for blob in fileSystemBlobLoader.yield_blobs():
    for document in customBlobParser.lazy_parse(blob):
        print(document)
        break

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

from langchain_community.document_loaders.blob_loaders import FileSystemBlobLoader

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

fileSystemBlobLoader = FileSystemBlobLoader(path = ".", glob = "*.txt", show_progress = True)

customBlobParser = CustomBlobParser()

for blob in fileSystemBlobLoader.yield_blobs():

for document in customBlobParser.lazy_parse(blob):

print(document)

break

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
colorama==0.4.6
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-community==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.35
tenacity==9.0.0
tqdm==4.67.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

colorama==0.4.6

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-community==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.35

tenacity==9.0.0

tqdm==4.67.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain

[PYTHON/LANGCHAIN] Blob 클래스 : as_string 메소드를 사용해 문자열 구하기

■ Blob 클래스의 as_string 메소드를 사용해 문자열을 구하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

string = blob.as_string()

print(string)

"""
meow meow1
meow meow2
meow meow3
"""

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

string = blob.as_string()

print(string)

"""

meow meow1

meow meow2

meow meow3

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다.

[PYTHON/LANGCHAIN] Blob 클래스 : as_bytes 메소드를 사용해 bytes 객체 구하기

■ Blob 클래스의 as_bytes 메소드를 사용해 bytes 객체를 구하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

bytes1 = blob.as_bytes()

print(bytes1)

"""
b'meow meow1\r\nmeow meow2\r\nmeow meow3'
"""

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

bytes1 = blob.as_bytes()

print(bytes1)

"""

b'meow meow1\r\nmeow meow2\r\nmeow meow3'

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을

[PYTHON/LANGCHAIN] Blob 클래스 : encoding/source/metadata 속성 사용하기

■ Blob 클래스의 encoding/source/metadata 속성을 사용하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

print(blob.encoding)
print(blob.source)
print(blob.metadata)

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

print(blob.encoding)

print(blob.source)

print(blob.metadata)

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다. meow.txt

[PYTHON/LANGCHAIN] Blob 클래스 : from_path 정적 메소드를 사용해 파일 로드하기

■ Blob 클래스의 from_path 정적 메소드를 사용해 파일을 로드하는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

from langchain_core.document_loaders import Blob

blob = Blob.from_path("meow.txt", metadata = {"foo" : "bar"})

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을

[PYTHON/LANGCHAIN] BaseBlobParser 클래스 : 커스텀 BLOB 파서 만들기

■ BaseBlobParser 클래스를 사용해 커스텀 BLOB 파서를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.document_loaders import BaseBlobParser
from langchain_core.document_loaders import Blob
from typing                          import Iterator
from langchain_core.documents        import Document

class CustomBlobParser(BaseBlobParser):
    """A simple parser that creates a document from each line."""

    def lazy_parse(self, blob : Blob) -> Iterator[Document]:
        """Parse a blob into a document line by line."""
        lineNumber = 0
        with blob.as_bytes_io() as bufferedReader:
            for line in bufferedReader:
                lineNumber += 1
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : blob.source}
                )

blob1 = Blob.from_path("meow.txt")

customBlobParser = CustomBlobParser()

generator1 = customBlobParser.lazy_parse(blob1)

documentList1 = list(generator1)

for document in documentList1:
    print(document)
print("-" * 50)

blob2 = Blob(data = b"some data from memory\nmeow")

generator2 = customBlobParser.lazy_parse(blob2)

documentList2 = list(generator2)

for document in documentList2:
    print(document)

"""
page_content='meow meow1' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 2, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 3, 'source': './meow.txt'}
--------------------------------------------------
page_content='some data from memory' metadata={'line_number': 1, 'source': None}
page_content='meow' metadata={'line_number': 2, 'source': None}
"""

from langchain_core.document_loaders import BaseBlobParser

from langchain_core.document_loaders import Blob

from typing import Iterator

from langchain_core.documents import Document

class CustomBlobParser(BaseBlobParser):

"""A simple parser that creates a document from each line."""

def lazy_parse(self, blob : Blob) -> Iterator[Document]:

"""Parse a blob into a document line by line."""

lineNumber = 0

with blob.as_bytes_io() as bufferedReader:

for line in bufferedReader:

lineNumber += 1

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : blob.source}

)

blob1 = Blob.from_path("meow.txt")

customBlobParser = CustomBlobParser()

generator1 = customBlobParser.lazy_parse(blob1)

documentList1 = list(generator1)

for document in documentList1:

print(document)

print("-" * 50)

blob2 = Blob(data = b"some data from memory\nmeow")

generator2 = customBlobParser.lazy_parse(blob2)

documentList2 = list(generator2)

for document in documentList2:

print(document)

"""

page_content='meow meow1' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 2, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 3, 'source': './meow.txt'}

--------------------------------------------------

page_content='some data from memory' metadata={'line_number': 1, 'source': None}

page_content='meow' metadata={'line_number': 2, 'source': None}

"""

▶ requirements.txt


aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.12
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.12

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install langchain 명령을 실행했다.

[PYTHON/LANGCHAIN] BaseLoader 클래스 : 커스텀 문서 로더 만들기

■ BaseLoader 클래스를 사용해 커스텀 문서 로더를 만드는 방법을 보여준다. ▶ main.py


import aiofiles
import asyncio

from langchain_core.document_loaders import BaseLoader
from typing                          import Iterator
from langchain_core.documents        import Document
from typing                          import AsyncIterator

class CustomDocumentLoader(BaseLoader):
    """An example document loader that reads a file line by line."""

    def __init__(self, filePath : str) -> None:
        """Initialize the loader with a file path.

        Args :
            file_path: The path to the file to load.
        """
        self.filePath = filePath

    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader that reads a file line by line.

        When you're implementing lazy load methods, you should use a generator to yield documents one by one.
        """
        with open(self.filePath, encoding = "utf-8") as textIOWrapper:
            lineNumber = 0
            for line in textIOWrapper:
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : self.filePath}
                )
                lineNumber += 1

    # alazy_load는 선택 사항이다.
    # 구현을 생략하면 lazy_load에 위임하는 기본 구현이 사용된다!
    async def alazy_load(self) -> AsyncIterator[Document]:
        """An async lazy loader that reads a file line by line."""

        async with aiofiles.open(self.filePath, encoding = "utf-8") as asyncTextIOWrapper:
            lineNumber = 0
            async for line in asyncTextIOWrapper:
                yield Document(
                    page_content = line.strip(),
                    metadata     = {"line_number" : lineNumber, "source" : self.filePath}
                )
                lineNumber += 1

with open("./meow.txt", "w", encoding = "utf-8") as textIOWrapper:
    fileContent = "meow meow1\nmeow meow2\nmeow meow3"
    textIOWrapper.write(fileContent)

customDocumentLoader = CustomDocumentLoader("./meow.txt")

for document in customDocumentLoader.lazy_load():
    print(document)
print("-" * 50)

documentList = customDocumentLoader.load()
for document in documentList:
    print(document)
print("-" * 50)

async def main():
    async for document in customDocumentLoader.alazy_load():
        print(document)
    print("-" * 50)

asyncio.run(main())

"""
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}
page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}
page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}
--------------------------------------------------
"""

import aiofiles

import asyncio

from langchain_core.document_loaders import BaseLoader

from typing import Iterator

from langchain_core.documents import Document

from typing import AsyncIterator

class CustomDocumentLoader(BaseLoader):

"""An example document loader that reads a file line by line."""

def __init__(self, filePath : str) -> None:

"""Initialize the loader with a file path.

Args :

file_path: The path to the file to load.

"""

self.filePath = filePath

def lazy_load(self) -> Iterator[Document]:

"""A lazy loader that reads a file line by line.

When you're implementing lazy load methods, you should use a generator to yield documents one by one.

"""

with open(self.filePath, encoding = "utf-8") as textIOWrapper:

lineNumber = 0

for line in textIOWrapper:

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : self.filePath}

)

lineNumber += 1

# alazy_load는 선택 사항이다.

# 구현을 생략하면 lazy_load에 위임하는 기본 구현이 사용된다!

async def alazy_load(self) -> AsyncIterator[Document]:

"""An async lazy loader that reads a file line by line."""

async with aiofiles.open(self.filePath, encoding = "utf-8") as asyncTextIOWrapper:

lineNumber = 0

async for line in asyncTextIOWrapper:

yield Document(

page_content = line.strip(),

metadata = {"line_number" : lineNumber, "source" : self.filePath}

)

lineNumber += 1

with open("./meow.txt", "w", encoding = "utf-8") as textIOWrapper:

fileContent = "meow meow1\nmeow meow2\nmeow meow3"

textIOWrapper.write(fileContent)

customDocumentLoader = CustomDocumentLoader("./meow.txt")

for document in customDocumentLoader.lazy_load():

print(document)

print("-" * 50)

documentList = customDocumentLoader.load()

for document in documentList:

print(document)

print("-" * 50)

async def main():

async for document in customDocumentLoader.alazy_load():

print(document)

print("-" * 50)

asyncio.run(main())

"""

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

page_content='meow meow1' metadata={'line_number': 0, 'source': './meow.txt'}

page_content='meow meow2' metadata={'line_number': 1, 'source': './meow.txt'}

page_content='meow meow3' metadata={'line_number': 2, 'source': './meow.txt'}

--------------------------------------------------

"""

▶ requirements.txt


aiofiles==24.1.0
aiohappyeyeballs==2.4.3
aiohttp==3.11.7
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.4.0
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.27.2
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.8
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.145
multidict==6.1.0
numpy==1.26.4
orjson==3.10.11
packaging==24.2
propcache==0.2.0
pydantic==2.10.1
pydantic_core==2.27.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
SQLAlchemy==2.0.36
tenacity==9.0.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.0

aiofiles==24.1.0

aiohappyeyeballs==2.4.3

aiohttp==3.11.7

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.4.0

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.27.2

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.8

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.145

multidict==6.1.0

numpy==1.26.4

orjson==3.10.11

packaging==24.2

propcache==0.2.0

pydantic==2.10.1

pydantic_core==2.27.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

SQLAlchemy==2.0.36

tenacity==9.0.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.0

※ pip install aiofiles langchain 명령을

[PYTHON/LANGCHAIN] Document 클래스 : page_content 속성을 사용해 Document 객체 만들기

■ Document 클래스의 page_content 속성을 사용해 Document 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_core.documents import Document

documentList = [
    Document(page_content = "Dogs are great companions, known for their loyalty and friendliness."),
    Document(page_content = "Cats are independent pets that often enjoy their own space."         )
]

from langchain_core.documents import Document

documentList = [

Document(page_content = "Dogs are great companions, known for their loyalty and friendliness."),

Document(page_content = "Cats are independent pets that often enjoy their own space." )

]

[PYTHON/LANGCHAIN] TextLoader 클래스 : 복수 텍스트 파일 문서 로드하기

■ TextLoader 클래스를 사용해 복수 텍스트 파일 문서를 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

TextLoaderList = [
    TextLoader("paul_graham_essay.txt" , encoding = "utf-8"),
    TextLoader("state_of_the_union.txt", encoding = "utf-8")
]

documentList = []

for textLoader in TextLoaderList:
    documentList.extend(textLoader.load())

from langchain_community.document_loaders import TextLoader

TextLoaderList = [

TextLoader("paul_graham_essay.txt" , encoding = "utf-8"),

TextLoader("state_of_the_union.txt", encoding = "utf-8")

]

documentList = []

for textLoader in TextLoaderList:

documentList.extend(textLoader.load())

※ pip install langchain-community 명령을 실행했다.

[PYTHON/LANGCHAIN] TextLoader 클래스 : 생성자에서 encoding 인자를 사용해 UTF-8 인코딩 파일 로드하기

■ TextLoader 클래스의 생성자에서 encoding 인자를 사용해 UTF-8 인코딩 파일을 로드하는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("paul_graham_essay.txt" , encoding = "utf-8")

from langchain_community.document_loaders import TextLoader

textLoader = TextLoader("paul_graham_essay.txt" , encoding = "utf-8")

※ pip install langchain-community

[PYTHON/LANGCHAIN] Docx2txtLoader 클래스 : load 메소드를 사용해 MS WORD 문서 로드하기

■ Docx2txtLoader 클래스의 load 메소드를 사용해 MS WORD 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import Docx2txtLoader

docx2txtLoader = Docx2txtLoader("sample.docx")

documentList = docx2txtLoader.load()

document = documentList[0]

print(document.page_content)

from langchain_community.document_loaders import Docx2txtLoader

docx2txtLoader = Docx2txtLoader("sample.docx")

documentList = docx2txtLoader.load()

document = documentList[0]

print(document.page_content)

▶ requirements.txt


aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
attrs==24.2.0
certifi==2024.8.30
charset-normalizer==3.3.2
dataclasses-json==0.6.7
docx2txt==0.8
frozenlist==1.4.1
greenlet==3.1.0
h11==0.14.0
httpcore==1.0.5
httpx==0.27.2
idna==3.8
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.16
langchain-community==0.2.16
langchain-core==0.2.39
langchain-text-splitters==0.2.4
langsmith==0.1.118
marshmallow==3.22.0
multidict==6.1.0
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.7
packaging==24.1
pydantic==2.9.1
pydantic_core==2.23.3
PyYAML==6.0.2
requests==2.32.3
sniffio==1.3.1
SQLAlchemy==2.0.34
tenacity==8.5.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.11.1

aiohappyeyeballs==2.4.0

aiohttp==3.10.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

attrs==24.2.0

certifi==2024.8.30

charset-normalizer==3.3.2

dataclasses-json==0.6.7

docx2txt==0.8

frozenlist==1.4.1

greenlet==3.1.0

h11==0.14.0

httpcore==1.0.5

httpx==0.27.2

idna==3.8

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.16

langchain-community==0.2.16

langchain-core==0.2.39

langchain-text-splitters==0.2.4

langsmith==0.1.118

marshmallow==3.22.0

multidict==6.1.0

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.7

packaging==24.1

pydantic==2.9.1

pydantic_core==2.23.3

PyYAML==6.0.2

requests==2.32.3

sniffio==1.3.1

SQLAlchemy==2.0.34

tenacity==8.5.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.11.1

※ pip install langchain-community

[PYTHON/LANGCHAIN] UnstructuredURLLoader 클래스 : 생성자에서 urls 인자를 사용해 UnstructuredURLLoader 객체 만들기

■ UnstructuredURLLoader 클래스의 생성자에서 urls 인자를 사용해 UnstructuredURLLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import UnstructuredURLLoader

urlList = [
    "https://n.news.naver.com/news/articles/092/0002307222?sid=105",
    "https://n.news.naver.com/news/articles/052/0001944792?sid=105"
]

unstructuredURLLoader = UnstructuredURLLoader(urls = urlList)

documentList = unstructuredURLLoader.load()

document = documentList[0]

print(document.page_content)

from langchain_community.document_loaders import UnstructuredURLLoader

urlList = [

"https://n.news.naver.com/news/articles/092/0002307222?sid=105",

"https://n.news.naver.com/news/articles/052/0001944792?sid=105"

]

unstructuredURLLoader = UnstructuredURLLoader(urls = urlList)

documentList = unstructuredURLLoader.load()

document = documentList[0]

print(document.page_content)

▶ requirements.txt


aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
attrs==24.2.0
backoff==2.2.1
beautifulsoup4==4.12.3
certifi==2024.8.30
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
colorama==0.4.6
dataclasses-json==0.6.7
deepdiff==8.0.1
emoji==2.12.1
filetype==1.2.0
frozenlist==1.4.1
greenlet==3.1.0
h11==0.14.0
httpcore==1.0.5
httpx==0.27.2
idna==3.8
joblib==1.4.2
jsonpatch==1.33
jsonpath-python==1.0.6
jsonpointer==3.0.0
langchain==0.2.16
langchain-community==0.2.16
langchain-core==0.2.39
langchain-text-splitters==0.2.4
langdetect==1.0.9
langsmith==0.1.118
lxml==5.3.0
marshmallow==3.22.0
multidict==6.1.0
mypy-extensions==1.0.0
nest-asyncio==1.6.0
nltk==3.9.1
numpy==1.26.4
olefile==0.47
orderly-set==5.2.2
orjson==3.10.7
packaging==24.1
psutil==6.0.0
pydantic==2.9.1
pydantic_core==2.23.3
pypdf==4.3.1
python-dateutil==2.9.0.post0
python-iso639==2024.4.27
python-magic==0.4.27
python-oxmsg==0.0.1
PyYAML==6.0.2
rapidfuzz==3.9.7
regex==2024.9.11
requests==2.32.3
requests-toolbelt==1.0.0
six==1.16.0
sniffio==1.3.1
soupsieve==2.6
SQLAlchemy==2.0.34
tabulate==0.9.0
tenacity==8.5.0
tqdm==4.66.5
typing-inspect==0.9.0
typing_extensions==4.12.2
unstructured==0.15.10
unstructured-client==0.25.8
urllib3==2.2.2
wrapt==1.16.0
yarl==1.11.1

aiohappyeyeballs==2.4.0

aiohttp==3.10.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

attrs==24.2.0

backoff==2.2.1

beautifulsoup4==4.12.3

certifi==2024.8.30

chardet==5.2.0

charset-normalizer==3.3.2

click==8.1.7

colorama==0.4.6

dataclasses-json==0.6.7

deepdiff==8.0.1

emoji==2.12.1

filetype==1.2.0

frozenlist==1.4.1

greenlet==3.1.0

h11==0.14.0

httpcore==1.0.5

httpx==0.27.2

idna==3.8

joblib==1.4.2

jsonpatch==1.33

jsonpath-python==1.0.6

jsonpointer==3.0.0

langchain==0.2.16

langchain-community==0.2.16

langchain-core==0.2.39

langchain-text-splitters==0.2.4

langdetect==1.0.9

langsmith==0.1.118

lxml==5.3.0

marshmallow==3.22.0

multidict==6.1.0

mypy-extensions==1.0.0

nest-asyncio==1.6.0

nltk==3.9.1

numpy==1.26.4

olefile==0.47

orderly-set==5.2.2

orjson==3.10.7

packaging==24.1

psutil==6.0.0

pydantic==2.9.1

pydantic_core==2.23.3

pypdf==4.3.1

python-dateutil==2.9.0.post0

python-iso639==2024.4.27

python-magic==0.4.27

python-oxmsg==0.0.1

PyYAML==6.0.2

rapidfuzz==3.9.7

regex==2024.9.11

requests==2.32.3

requests-toolbelt==1.0.0

six==1.16.0

sniffio==1.3.1

soupsieve==2.6

SQLAlchemy==2.0.34

tabulate==0.9.0

tenacity==8.5.0

tqdm==4.66.5

typing-inspect==0.9.0

typing_extensions==4.12.2

unstructured==0.15.10

unstructured-client==0.25.8

urllib3==2.2.2

wrapt==1.16.0

yarl==1.11.1

※ pip install langchain-community

[PYTHON/PYPDF] PDF 파일 분할하기

■ PDF 파일을 분할하는 방법을 보여준다. ▶ main.py


import os

from PyPDF2 import PdfReader
from PyPDF2 import PdfWriter

def splitPDFFile(sourceFilePath, targetDirectoryPath, pageCountPerFile):
    pdfReader      = PdfReader(sourceFilePath)
    totalPageCount = len(pdfReader.pages)
    if not os.path.exists(targetDirectoryPath):
        os.makedirs(targetDirectoryPath)
    fileNameWithoutExtension = os.path.splitext(os.path.basename(sourceFilePath))[0]
    for i in range(0, totalPageCount, pageCountPerFile):
        pdfWriter = PdfWriter()
        for page in range(i, min(i + pageCountPerFile, totalPageCount)):
            pdfWriter.add_page(pdfReader.pages[page])
        targetFileName = f"{fileNameWithoutExtension}_part_{i//pageCountPerFile + 1}.pdf"
        targetFilePath = os.path.join(targetDirectoryPath, targetFileName)
        with open(targetFilePath, "wb") as bufferedWriter:
            pdfWriter.write(bufferedWriter)
        print(f"Created : {targetFileName}")
    print("PDF splitting completed.")

sourceFilePath      = "d:\\Practical C# Charts and Graphics.pdf"
targetDirectoryPath = "d:\\Practical C# Charts and Graphics"
pageCountPerFile    = 100

splitPDFFile(sourceFilePath, targetDirectoryPath, pageCountPerFile)

import os

from PyPDF2 import PdfReader

from PyPDF2 import PdfWriter

def splitPDFFile(sourceFilePath, targetDirectoryPath, pageCountPerFile):

pdfReader = PdfReader(sourceFilePath)

totalPageCount = len(pdfReader.pages)

if not os.path.exists(targetDirectoryPath):

os.makedirs(targetDirectoryPath)

fileNameWithoutExtension = os.path.splitext(os.path.basename(sourceFilePath))[0]

for i in range(0, totalPageCount, pageCountPerFile):

pdfWriter = PdfWriter()

for page in range(i, min(i + pageCountPerFile, totalPageCount)):

pdfWriter.add_page(pdfReader.pages[page])

targetFileName = f"{fileNameWithoutExtension}_part_{i//pageCountPerFile + 1}.pdf"

targetFilePath = os.path.join(targetDirectoryPath, targetFileName)

with open(targetFilePath, "wb") as bufferedWriter:

pdfWriter.write(bufferedWriter)

print(f"Created : {targetFileName}")

print("PDF splitting completed.")

sourceFilePath = "d:\\Practical C# Charts and Graphics.pdf"

targetDirectoryPath = "d:\\Practical C# Charts and Graphics"

pageCountPerFile = 100

splitPDFFile(sourceFilePath, targetDirectoryPath, pageCountPerFile)

▶ requirements.txt


PyPDF2==3.0.1

PyPDF2==3.0.1

※ pip install pypdf2 명령을 실행했다.

[PYTHON/PYPDF] pypdf2 패키지 설치하기

■ pypdf2 패키지를 설치하는 방법을 보여준다. 1. 명령 프롬프트를 실행한다. 2. 명령 프롬프트에서 아래 명령을 실행한다. ▶ 실행 명령


pip install pypdf2

pip install pypdf2

[C#/WPF/.NET8] Table 엘리먼트 : FlowDocument 엘리먼트에서 테이블 만들기 (C#)

■ Table 엘리먼트를 사용해 FlowDocument 엘리먼트에서 테이블을 만드는 방법을 보여준다. ▶ MainWindow.xaml


<Window x:Class="TestProject.MainWindow"
    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
    xmlns:local="clr-namespace:TestProject"
    Width="800"
    Height="600"
    Title="TestProject"
    FontFamily="나눔고딕코딩"
    FontSize="16">
    <FlowDocumentReader>
        <FlowDocument Name="flowDocument" />
    </FlowDocumentReader>
</Window>

<Window x:Class="TestProject.MainWindow"

xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"

xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"

xmlns:local="clr-namespace:TestProject"

Width="800"

Height="600"

Title="TestProject"

FontFamily="나눔고딕코딩"

FontSize="16">

</FlowDocumentReader>

</Window>

▶ MainWindow.xaml.cs


using System.Windows;
using System.Windows.Documents;
using System.Windows.Media;

namespace TestProject;

/// <summary>
/// 메인 윈도우
/// </summary>
public partial class MainWindow : Window
{
    //////////////////////////////////////////////////////////////////////////////////////////////////// Constructor
    ////////////////////////////////////////////////////////////////////////////////////////// Public

    #region 생성자 - MainWindow()

    /// <summary>
    /// 생성자
    /// </summary>
    public MainWindow()
    {
        InitializeComponent();

        TableColumn tableColumn1 = new TableColumn { Width = GridLength.Auto };
        TableColumn tableColumn2 = new TableColumn { Width = GridLength.Auto };
        TableColumn tableColumn3 = new TableColumn { Width = GridLength.Auto };

        Paragraph headerCellParagraph1 = new Paragraph(new Run("Header 1"));
        Paragraph headerCellParagraph2 = new Paragraph(new Run("Header 2"));
        Paragraph headerCellParagraph3 = new Paragraph(new Run("Header 3"));

        TableCell headerTableCell1 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };
        TableCell headerTableCell2 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };
        TableCell headerTableCell3 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };

        headerTableCell1.Blocks.Add(headerCellParagraph1);
        headerTableCell2.Blocks.Add(headerCellParagraph2);
        headerTableCell3.Blocks.Add(headerCellParagraph3);

        TableRow headerTableRow = new TableRow();

        headerTableRow.Cells.Add(headerTableCell1);
        headerTableRow.Cells.Add(headerTableCell2);
        headerTableRow.Cells.Add(headerTableCell3);

        Paragraph dataCellParagraph1 = new Paragraph(new Run("Row 1, Cell 1"));
        Paragraph dataCellParagraph2 = new Paragraph(new Run("Row 1, Cell 2"));
        Paragraph dataCellParagraph3 = new Paragraph(new Run("Row 1, Cell 3"));

        TableCell dataTableCell1 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };
        TableCell dataTableCell2 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };
        TableCell dataTableCell3 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

        dataTableCell1.Blocks.Add(dataCellParagraph1);
        dataTableCell2.Blocks.Add(dataCellParagraph2);
        dataTableCell3.Blocks.Add(dataCellParagraph3);

        TableRow dataTableRow1 = new TableRow();

        dataTableRow1.Cells.Add(dataTableCell1);
        dataTableRow1.Cells.Add(dataTableCell2);
        dataTableRow1.Cells.Add(dataTableCell3);


        Paragraph dataCellParagraph4 = new Paragraph(new Run("Row 2, Cell 1"));
        Paragraph dataCellParagraph5 = new Paragraph(new Run("Row 2, Cell 2"));
        Paragraph dataCellParagraph6 = new Paragraph(new Run("Row 2, Cell 3"));

        TableCell dataTableCell4 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };
        TableCell dataTableCell5 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };
        TableCell dataTableCell6 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

        dataTableCell4.Blocks.Add(dataCellParagraph4);
        dataTableCell5.Blocks.Add(dataCellParagraph5);
        dataTableCell6.Blocks.Add(dataCellParagraph6);

        TableRow dataTableRow2 = new TableRow();

        dataTableRow2.Cells.Add(dataTableCell4);
        dataTableRow2.Cells.Add(dataTableCell5);
        dataTableRow2.Cells.Add(dataTableCell6);

        TableRowGroup tableRowGroup = new TableRowGroup();

        tableRowGroup.Rows.Add(headerTableRow);
        tableRowGroup.Rows.Add(dataTableRow1 );
        tableRowGroup.Rows.Add(dataTableRow2 );

        Table table = new Table();

        table.BorderThickness = new Thickness(1);
        table.BorderBrush     = Brushes.Black;
        table.CellSpacing     = 3;

        table.Columns.Add(tableColumn1);
        table.Columns.Add(tableColumn2);
        table.Columns.Add(tableColumn3);

        table.RowGroups.Add(tableRowGroup);

        this.flowDocument.Blocks.Add(table);
    }

    #endregion
}

100

101

102

103

104

105

106

107

using System.Windows;

using System.Windows.Documents;

using System.Windows.Media;

namespace TestProject;

/// <summary>

/// 메인 윈도우

/// </summary>

public partial class MainWindow : Window

{

//////////////////////////////////////////////////////////////////////////////////////////////////// Constructor

////////////////////////////////////////////////////////////////////////////////////////// Public

#region 생성자 - MainWindow()

/// <summary>

/// 생성자

/// </summary>

public MainWindow()

{

InitializeComponent();

TableColumn tableColumn1 = new TableColumn { Width = GridLength.Auto };

TableColumn tableColumn2 = new TableColumn { Width = GridLength.Auto };

TableColumn tableColumn3 = new TableColumn { Width = GridLength.Auto };

Paragraph headerCellParagraph1 = new Paragraph(new Run("Header 1"));

Paragraph headerCellParagraph2 = new Paragraph(new Run("Header 2"));

Paragraph headerCellParagraph3 = new Paragraph(new Run("Header 3"));

TableCell headerTableCell1 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };

TableCell headerTableCell2 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };

TableCell headerTableCell3 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3), Background = Brushes.LightGray, TextAlignment = TextAlignment.Center };

headerTableCell1.Blocks.Add(headerCellParagraph1);

headerTableCell2.Blocks.Add(headerCellParagraph2);

headerTableCell3.Blocks.Add(headerCellParagraph3);

TableRow headerTableRow = new TableRow();

headerTableRow.Cells.Add(headerTableCell1);

headerTableRow.Cells.Add(headerTableCell2);

headerTableRow.Cells.Add(headerTableCell3);

Paragraph dataCellParagraph1 = new Paragraph(new Run("Row 1, Cell 1"));

Paragraph dataCellParagraph2 = new Paragraph(new Run("Row 1, Cell 2"));

Paragraph dataCellParagraph3 = new Paragraph(new Run("Row 1, Cell 3"));

TableCell dataTableCell1 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

TableCell dataTableCell2 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

TableCell dataTableCell3 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

dataTableCell1.Blocks.Add(dataCellParagraph1);

dataTableCell2.Blocks.Add(dataCellParagraph2);

dataTableCell3.Blocks.Add(dataCellParagraph3);

TableRow dataTableRow1 = new TableRow();

dataTableRow1.Cells.Add(dataTableCell1);

dataTableRow1.Cells.Add(dataTableCell2);

dataTableRow1.Cells.Add(dataTableCell3);

Paragraph dataCellParagraph4 = new Paragraph(new Run("Row 2, Cell 1"));

Paragraph dataCellParagraph5 = new Paragraph(new Run("Row 2, Cell 2"));

Paragraph dataCellParagraph6 = new Paragraph(new Run("Row 2, Cell 3"));

TableCell dataTableCell4 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

TableCell dataTableCell5 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

TableCell dataTableCell6 = new TableCell { BorderThickness = new Thickness(1), BorderBrush = Brushes.Black, Padding = new Thickness(3) };

dataTableCell4.Blocks.Add(dataCellParagraph4);

dataTableCell5.Blocks.Add(dataCellParagraph5);

dataTableCell6.Blocks.Add(dataCellParagraph6);

TableRow dataTableRow2 = new TableRow();

dataTableRow2.Cells.Add(dataTableCell4);

dataTableRow2.Cells.Add(dataTableCell5);

dataTableRow2.Cells.Add(dataTableCell6);

TableRowGroup tableRowGroup = new TableRowGroup();

tableRowGroup.Rows.Add(headerTableRow);

tableRowGroup.Rows.Add(dataTableRow1 );

tableRowGroup.Rows.Add(dataTableRow2 );

Table table = new Table();

table.BorderThickness = new Thickness(1);

table.BorderBrush = Brushes.Black;

table.CellSpacing = 3;

table.Columns.Add(tableColumn1);

table.Columns.Add(tableColumn2);

table.Columns.Add(tableColumn3);

table.RowGroups.Add(tableRowGroup);

this.flowDocument.Blocks.Add(table);

}

#endregion

}

TestProject.zip

[C#/WPF/.NET8] Table 엘리먼트 : FlowDocument 엘리먼트에서 테이블 만들기 (XAML)

■ Table 엘리먼트를 사용해 FlowDocument 엘리먼트에서 테이블을 만드는 방법을 보여준다. ▶ MainWindow.xaml


<Window x:Class="TestProject.MainWindow"
    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
    xmlns:local="clr-namespace:TestProject"
    Width="800"
    Height="600"
    Title="TestProject"
    FontFamily="나눔고딕코딩"
    FontSize="16">
    <FlowDocumentReader>
        <FlowDocument>
            <Table
                BorderThickness="1"
                BorderBrush="Black"
                CellSpacing="3">
                <Table.Columns>
                    <TableColumn Width="Auto" />
                    <TableColumn Width="Auto" />
                    <TableColumn Width="Auto" />
                </Table.Columns>
                <TableRowGroup>
                    <TableRow>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3"
                            Background="LightGray"
                            TextAlignment="Center">
                            <Paragraph>Header 1</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Background="LightGray"
                            Padding="3"
                            TextAlignment="Center">
                            <Paragraph>Header 2</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Background="LightGray"
                            Padding="3"
                            TextAlignment="Center">
                            <Paragraph>Header 3</Paragraph>
                        </TableCell>
                    </TableRow>
                    <TableRow>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 1, Cell 1</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 1, Cell 2</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 1, Cell 3</Paragraph>
                        </TableCell>
                    </TableRow>
                    <TableRow>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 2, Cell 1</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 2, Cell 2</Paragraph>
                        </TableCell>
                        <TableCell
                            BorderThickness="1"
                            BorderBrush="Black"
                            Padding="3">
                            <Paragraph>Row 2, Cell 3</Paragraph>
                        </TableCell>
                    </TableRow>
                </TableRowGroup>
            </Table>
        </FlowDocument>
    </FlowDocumentReader>
</Window>

<Window x:Class="TestProject.MainWindow"

xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"

xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"

xmlns:local="clr-namespace:TestProject"

Width="800"

Height="600"

Title="TestProject"

FontFamily="나눔고딕코딩"

FontSize="16">

<Table

BorderThickness="1"

BorderBrush="Black"

CellSpacing="3">

<Table.Columns>

</Table.Columns>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3"

Background="LightGray"

TextAlignment="Center">

<Paragraph>Header 1</Paragraph>

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Background="LightGray"

Padding="3"

TextAlignment="Center">

<Paragraph>Header 2</Paragraph>

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Background="LightGray"

Padding="3"

TextAlignment="Center">

<Paragraph>Header 3</Paragraph>

</TableCell>

</TableRow>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

</TableRow>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

<TableCell

BorderThickness="1"

BorderBrush="Black"

Padding="3">

</TableCell>

</TableRow>

</TableRowGroup>

</Table>

</FlowDocument>

</FlowDocumentReader>

</Window>

TestProject.zip

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : invoke 메소드 사용하기 2

■ MultiQueryRetriever 클래스의 invoke 메소드를 사용하는 방법을 보여준다. 2 ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ main.py


from dotenv import load_dotenv
from typing import List

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter
from langchain_openai                     import OpenAIEmbeddings
from langchain_chroma                     import Chroma
from langchain_core.prompts               import PromptTemplate
from langchain_openai                     import ChatOpenAI
from langchain_core.output_parsers        import BaseOutputParser
from langchain.retrievers.multi_query     import MultiQueryRetriever

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitdocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitdocumentList, embedding = openAIEmbeddings)

promptTemplate = PromptTemplate(
    input_variables = ["question"],
    template        = """You are an AI language model assistant.
Your task is to generate five different versions of the given user question to retrieve relevant documents from a vector database.
By generating multiple perspectives on the user question, your goal is to help the user overcome some of the limitations of the distance-based similarity search.
Provide these alternative questions separated by newlines.
Original question : {question}"""
)

chatOpenAI = ChatOpenAI(temperature = 0)

class LineListOutputParser(BaseOutputParser[List[str]]):
    """Output parser for a list of lines."""

    def parse(self, text : str) -> List[str]:
        lineList = text.strip().split("\n")
        return list(filter(None, lineList))

lineListOutputParser = LineListOutputParser()

runnableSequence = promptTemplate | chatOpenAI | lineListOutputParser

rultiQueryRetriever = MultiQueryRetriever(retriever = chroma.as_retriever(), llm_chain = runnableSequence, parser_key = "lines")

responseDocumentList = rultiQueryRetriever.invoke("What does the course say about regression?")

print(len(responseDocumentList))

from dotenv import load_dotenv

from typing import List

from langchain_community.document_loaders import WebBaseLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

from langchain_openai import OpenAIEmbeddings

from langchain_chroma import Chroma

from langchain_core.prompts import PromptTemplate

from langchain_openai import ChatOpenAI

from langchain_core.output_parsers import BaseOutputParser

from langchain.retrievers.multi_query import MultiQueryRetriever

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitdocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitdocumentList, embedding = openAIEmbeddings)

promptTemplate = PromptTemplate(

input_variables = ["question"],

template = """You are an AI language model assistant.

Your task is to generate five different versions of the given user question to retrieve relevant documents from a vector database.

By generating multiple perspectives on the user question, your goal is to help the user overcome some of the limitations of the distance-based similarity search.

Provide these alternative questions separated by newlines.

Original question : {question}"""

)

chatOpenAI = ChatOpenAI(temperature = 0)

class LineListOutputParser(BaseOutputParser[List[str]]):

"""Output parser for a list of lines."""

def parse(self, text : str) -> List[str]:

lineList = text.strip().split("\n")

return list(filter(None, lineList))

lineListOutputParser = LineListOutputParser()

runnableSequence = promptTemplate | chatOpenAI | lineListOutputParser

rultiQueryRetriever = MultiQueryRetriever(retriever = chroma.as_retriever(), llm_chain = runnableSequence, parser_key = "lines")

responseDocumentList = rultiQueryRetriever.invoke("What does the course say about regression?")

print(len(responseDocumentList))

▶ requirements.txt

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : invoke 메소드 사용하기 1

■ MultiQueryRetriever 클래스의 invoke 메소드를 사용하는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ main.py


import logging

from dotenv import load_dotenv

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter
from langchain_openai                     import OpenAIEmbeddings
from langchain_chroma                     import Chroma
from langchain_openai                     import ChatOpenAI
from langchain.retrievers.multi_query     import MultiQueryRetriever

logging.basicConfig()

logger = logging.getLogger("langchain.retrievers.multi_query")

logger.setLevel(logging.INFO)

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitDocumentList, embedding = openAIEmbeddings)

vectorStoreRetriever = chroma.as_retriever()

chatOpenAI = ChatOpenAI(temperature = 0)

multiQueryRetriever = MultiQueryRetriever.from_llm(retriever = vectorStoreRetriever, llm = chatOpenAI)

question = "What are the approaches to Task Decomposition?"

responseDocumentList = multiQueryRetriever.invoke(question)

print(len(responseDocumentList))

print(responseDocumentList[0].page_content)

"""
5
Task decomposition can be done (1) by LLM with simple prompting like "Steps for XYZ.\n1.", "What are the subgoals for achieving XYZ?", (2) by using task-specific instructions; e.g. "Write a story outline." for writing a novel, or (3) with human inputs.
"""

import logging

from dotenv import load_dotenv

from langchain_community.document_loaders import WebBaseLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

from langchain_openai import OpenAIEmbeddings

from langchain_chroma import Chroma

from langchain_openai import ChatOpenAI

from langchain.retrievers.multi_query import MultiQueryRetriever

logging.basicConfig()

logger = logging.getLogger("langchain.retrievers.multi_query")

logger.setLevel(logging.INFO)

load_dotenv()

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

openAIEmbeddings = OpenAIEmbeddings()

chroma = Chroma.from_documents(documents = splitDocumentList, embedding = openAIEmbeddings)

vectorStoreRetriever = chroma.as_retriever()

chatOpenAI = ChatOpenAI(temperature = 0)

multiQueryRetriever = MultiQueryRetriever.from_llm(retriever = vectorStoreRetriever, llm = chatOpenAI)

question = "What are the approaches to Task Decomposition?"

responseDocumentList = multiQueryRetriever.invoke(question)

print(len(responseDocumentList))

print(responseDocumentList[0].page_content)

"""

Task decomposition can be done (1) by LLM with simple prompting like "Steps for XYZ.\n1.", "What are the subgoals for achieving XYZ?", (2) by using task-specific instructions; e.g. "Write a story outline." for writing a novel, or (3) with human inputs.

"""

▶ requirements.txt


aiohappyeyeballs==2.3.5
aiohttp==3.10.3
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
backoff==2.2.1
bcrypt==4.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
build==1.2.1
cachetools==5.4.0
certifi==2024.7.4
charset-normalizer==3.3.2
chroma-hnswlib==0.7.6
chromadb==0.5.5
click==8.1.7
coloredlogs==15.0.1
dataclasses-json==0.6.7
Deprecated==1.2.14
distro==1.9.0
exceptiongroup==1.2.2
fastapi==0.112.0
filelock==3.15.4
flatbuffers==24.3.25
frozenlist==1.4.1
fsspec==2024.6.1
google-auth==2.33.0
googleapis-common-protos==1.63.2
greenlet==3.0.3
grpcio==1.65.4
h11==0.14.0
httpcore==1.0.5
httptools==0.6.1
httpx==0.27.0
huggingface-hub==0.24.5
humanfriendly==10.0
idna==3.7
importlib_metadata==8.0.0
importlib_resources==6.4.0
jiter==0.5.0
jsonpatch==1.33
jsonpointer==3.0.0
kubernetes==30.1.0
langchain==0.2.12
langchain-chroma==0.1.2
langchain-community==0.2.11
langchain-core==0.2.29
langchain-openai==0.1.21
langchain-text-splitters==0.2.2
langsmith==0.1.98
markdown-it-py==3.0.0
marshmallow==3.21.3
mdurl==0.1.2
mmh3==4.1.0
monotonic==1.6
mpmath==1.3.0
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
oauthlib==3.2.2
onnxruntime==1.18.1
openai==1.40.3
opentelemetry-api==1.26.0
opentelemetry-exporter-otlp-proto-common==1.26.0
opentelemetry-exporter-otlp-proto-grpc==1.26.0
opentelemetry-instrumentation==0.47b0
opentelemetry-instrumentation-asgi==0.47b0
opentelemetry-instrumentation-fastapi==0.47b0
opentelemetry-proto==1.26.0
opentelemetry-sdk==1.26.0
opentelemetry-semantic-conventions==0.47b0
opentelemetry-util-http==0.47b0
orjson==3.10.7
overrides==7.7.0
packaging==24.1
posthog==3.5.0
protobuf==4.25.4
pyasn1==0.6.0
pyasn1_modules==0.4.0
pydantic==2.8.2
pydantic_core==2.20.1
Pygments==2.18.0
PyPika==0.48.9
pyproject_hooks==1.1.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
PyYAML==6.0.2
regex==2024.7.24
requests==2.32.3
requests-oauthlib==2.0.0
rich==13.7.1
rsa==4.9
shellingham==1.5.4
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.32
starlette==0.37.2
sympy==1.13.1
tenacity==8.5.0
tiktoken==0.7.0
tokenizers==0.20.0
tomli==2.0.1
tqdm==4.66.5
typer==0.12.3
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
uvicorn==0.30.5
uvloop==0.19.0
watchfiles==0.23.0
websocket-client==1.8.0
websockets==12.0
wrapt==1.16.0
yarl==1.9.4
zipp==3.19.2

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

aiohappyeyeballs==2.3.5

aiohttp==3.10.3

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

asgiref==3.8.1

async-timeout==4.0.3

attrs==24.2.0

backoff==2.2.1

bcrypt==4.2.0

beautifulsoup4==4.12.3

bs4==0.0.2

build==1.2.1

cachetools==5.4.0

certifi==2024.7.4

charset-normalizer==3.3.2

chroma-hnswlib==0.7.6

chromadb==0.5.5

click==8.1.7

coloredlogs==15.0.1

dataclasses-json==0.6.7

Deprecated==1.2.14

distro==1.9.0

exceptiongroup==1.2.2

fastapi==0.112.0

filelock==3.15.4

flatbuffers==24.3.25

frozenlist==1.4.1

fsspec==2024.6.1

google-auth==2.33.0

googleapis-common-protos==1.63.2

greenlet==3.0.3

grpcio==1.65.4

h11==0.14.0

httpcore==1.0.5

httptools==0.6.1

httpx==0.27.0

huggingface-hub==0.24.5

humanfriendly==10.0

idna==3.7

importlib_metadata==8.0.0

importlib_resources==6.4.0

jiter==0.5.0

jsonpatch==1.33

jsonpointer==3.0.0

kubernetes==30.1.0

langchain==0.2.12

langchain-chroma==0.1.2

langchain-community==0.2.11

langchain-core==0.2.29

langchain-openai==0.1.21

langchain-text-splitters==0.2.2

langsmith==0.1.98

markdown-it-py==3.0.0

marshmallow==3.21.3

mdurl==0.1.2

mmh3==4.1.0

monotonic==1.6

mpmath==1.3.0

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

oauthlib==3.2.2

onnxruntime==1.18.1

openai==1.40.3

opentelemetry-api==1.26.0

opentelemetry-exporter-otlp-proto-common==1.26.0

opentelemetry-exporter-otlp-proto-grpc==1.26.0

opentelemetry-instrumentation==0.47b0

opentelemetry-instrumentation-asgi==0.47b0

opentelemetry-instrumentation-fastapi==0.47b0

opentelemetry-proto==1.26.0

opentelemetry-sdk==1.26.0

opentelemetry-semantic-conventions==0.47b0

opentelemetry-util-http==0.47b0

orjson==3.10.7

overrides==7.7.0

packaging==24.1

posthog==3.5.0

protobuf==4.25.4

pyasn1==0.6.0

pyasn1_modules==0.4.0

pydantic==2.8.2

pydantic_core==2.20.1

Pygments==2.18.0

PyPika==0.48.9

pyproject_hooks==1.1.0

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

PyYAML==6.0.2

regex==2024.7.24

requests==2.32.3

requests-oauthlib==2.0.0

rich==13.7.1

rsa==4.9

shellingham==1.5.4

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.32

starlette==0.37.2

sympy==1.13.1

tenacity==8.5.0

tiktoken==0.7.0

tokenizers==0.20.0

tomli==2.0.1

tqdm==4.66.5

typer==0.12.3

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

uvicorn==0.30.5

uvloop==0.19.0

watchfiles==0.23.0

websocket-client==1.8.0

websockets==12.0

wrapt==1.16.0

yarl==1.9.4

zipp==3.19.2

[PYTHON/LANGCHAIN] MultiQueryRetriever 클래스 : 생성자에서 retriever/llm_chain/parser_key 인자를 사용해 MultiQueryRetriever 객체 만들기

■ MultiQueryRetriever 클래스의 생성자에서 retriever/llm_chain/parser_key 인자를 사용해 MultiQueryRetriever 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ 예제

[PYTHON/LANGCHAIN] from_llm 함수 : retriever/llm 인자를 사용해 multiQueryRetriever 객체 만들기

■ from_llm 함수의 retriever/llm 인자를 사용해 multiQueryRetriever 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env 파일에 정의한다. ▶ 예제 코드

[PYTHON/LANGCHAIN] from_documents 함수 : documents/embedding 인자를 사용해 메모리 내 벡터 저장소를 위한 Chroma 객체 만들기

■ from_documents 함수의 documents/embedding 인자를 사용해 메모리 내 벡터 저장소를 위한 Chroma 객체를 만드는 방법을 보여준다. ※ OPENAI_API_KEY 환경 변수 값은 .env

[PYTHON/LANGCHAIN] RecursiveCharacterTextSplitter 클래스 : 생성자에서 chunk_size/chunk_overlap 인자를 사용해 재귀적 문자 텍스트 분할기 만들기

■ RecursiveCharacterTextSplitter 클래스의 생성자에서 chunk_size/chunk_overlap 인자를 사용해 재귀적 문자 텍스트 분할기를 만드는 방법을 보여준다. ▶ 예제 코드 (PY)


from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

from langchain_community.document_loaders import WebBaseLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

webBaseLoader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")

documentList = webBaseLoader.load()

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)

[PYTHON/LANGCHAIN] WebBaseLoader 클래스 : 생성자에서 web_paths/bs_kwargs 인자를 사용해 웹 문서 로더 만들기

■ WebBaseLoader 클래스의 생성자에서 web_paths/bs_kwargs 인자를 사용해 웹 문서 로더를 만드는 방법을 보여준다. ▶ main.py


import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(
    web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),
    bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))
)

import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(

web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),

bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))

)