DOCUMENT LOADER Archives - 3 중 3 번째 페이지

[PYTHON/LANGCHAIN] DirectoryLoader 클래스 : 생성자에서 use_multithreading 인자를 사용해 멀티스레딩 설정하기

■ DirectoryLoader 클래스의 생성자에서 use_multithreading 인자를 사용해 멀티스레딩을 설정하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md", use_multithreading = True)

documentList = directoryLoader.load()

for document in documentList:
    print(document)
    print()

"""
page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}

page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}
"""

from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md", use_multithreading = True)

documentList = directoryLoader.load()

for document in documentList:

print(document)

print()

"""

page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}

page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
backoff==2.2.1
beautifulsoup4==4.12.3
certifi==2024.6.2
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.7
deepdiff==7.0.1
emoji==2.12.1
exceptiongroup==1.2.1
filetype==1.2.0
frozenlist==1.4.1
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpath-python==1.0.6
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langdetect==1.0.9
langsmith==0.1.82
lxml==5.2.2
Markdown==3.6
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
nltk==3.8.1
numpy==1.26.4
ordered-set==4.1.0
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
python-iso639==2024.4.27
python-magic==0.4.27
PyYAML==6.0.1
rapidfuzz==3.9.3
regex==2024.5.15
requests==2.32.3
requests-toolbelt==1.0.0
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.31
tabulate==0.9.0
tenacity==8.4.2
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.2
unstructured==0.14.8
unstructured-client==0.23.7
urllib3==2.2.2
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

backoff==2.2.1

beautifulsoup4==4.12.3

certifi==2024.6.2

chardet==5.2.0

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.7

deepdiff==7.0.1

emoji==2.12.1

exceptiongroup==1.2.1

filetype==1.2.0

frozenlist==1.4.1

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpath-python==1.0.6

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langdetect==1.0.9

langsmith==0.1.82

lxml==5.2.2

Markdown==3.6

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

nltk==3.8.1

numpy==1.26.4

ordered-set==4.1.0

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

python-iso639==2024.4.27

python-magic==0.4.27

PyYAML==6.0.1

rapidfuzz==3.9.3

regex==2024.5.15

requests==2.32.3

requests-toolbelt==1.0.0

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.31

tabulate==0.9.0

tenacity==8.4.2

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.2

unstructured==0.14.8

unstructured-client==0.23.7

urllib3==2.2.2

wrapt==1.16.0

yarl==1.9.4

※ pip install langchain-community unstructured[md]

[PYTHON/LANGCHAIN] DirectoryLoader 클래스 : 생성자에서 show_progress 인자를 사용해 문서 로드 진행 상태 표시하기

■ DirectoryLoader 클래스의 생성자에서 show_progress 인자를 사용해 문서 로드 진행 상태를 표시하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md", show_progress = True)

documentList = directoryLoader.load()

for document in documentList:
    print(document)
    print()

"""
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.92it/s]
page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}

page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}
"""

from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md", show_progress = True)

documentList = directoryLoader.load()

for document in documentList:

print(document)

print()

"""

100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00, 1.92it/s]

page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}

page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
backoff==2.2.1
beautifulsoup4==4.12.3
certifi==2024.6.2
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.7
deepdiff==7.0.1
emoji==2.12.1
exceptiongroup==1.2.1
filetype==1.2.0
frozenlist==1.4.1
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpath-python==1.0.6
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langdetect==1.0.9
langsmith==0.1.82
lxml==5.2.2
Markdown==3.6
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
nltk==3.8.1
numpy==1.26.4
ordered-set==4.1.0
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
python-iso639==2024.4.27
python-magic==0.4.27
PyYAML==6.0.1
rapidfuzz==3.9.3
regex==2024.5.15
requests==2.32.3
requests-toolbelt==1.0.0
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.31
tabulate==0.9.0
tenacity==8.4.2
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.2
unstructured==0.14.8
unstructured-client==0.23.7
urllib3==2.2.2
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

backoff==2.2.1

beautifulsoup4==4.12.3

certifi==2024.6.2

chardet==5.2.0

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.7

deepdiff==7.0.1

emoji==2.12.1

exceptiongroup==1.2.1

filetype==1.2.0

frozenlist==1.4.1

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpath-python==1.0.6

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langdetect==1.0.9

langsmith==0.1.82

lxml==5.2.2

Markdown==3.6

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

nltk==3.8.1

numpy==1.26.4

ordered-set==4.1.0

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

python-iso639==2024.4.27

python-magic==0.4.27

PyYAML==6.0.1

rapidfuzz==3.9.3

regex==2024.5.15

requests==2.32.3

requests-toolbelt==1.0.0

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.31

tabulate==0.9.0

tenacity==8.4.2

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.2

unstructured==0.14.8

unstructured-client==0.23.7

urllib3==2.2.2

wrapt==1.16.0

yarl==1.9.4

※ pip

[PYTHON/LANGCHAIN] DirectoryLoader 클래스 : 생성자에서 glob 인자를 사용해 MD 파일 문서 로드하기

■ DirectoryLoader 클래스의 생성자에서 glob 인자를 사용해 MD 파일 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md")

documentList = directoryLoader.load()

for document in documentList:
    print(document)
    print()

"""
page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}

page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}
"""

from langchain_community.document_loaders import DirectoryLoader

directoryLoader = DirectoryLoader(".", glob = "*.md")

documentList = directoryLoader.load()

for document in documentList:

print(document)

print()

"""

page_content='제목 1단계\n\n제목 2단계\n\n제목 3단계\n\n제목 4단계\n\n제목 5단계\n\n제목 6단계' metadata={'source': 'header.md'}

page_content='첫번째\n\n두번째\n\n세번째\n\n항목1\n\n항목 2\n항목 3\n항목 4' metadata={'source': 'list.md'}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==23.2.0
backoff==2.2.1
beautifulsoup4==4.12.3
certifi==2024.6.2
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
dataclasses-json==0.6.7
deepdiff==7.0.1
emoji==2.12.1
exceptiongroup==1.2.1
filetype==1.2.0
frozenlist==1.4.1
greenlet==3.0.3
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
joblib==1.4.2
jsonpatch==1.33
jsonpath-python==1.0.6
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langdetect==1.0.9
langsmith==0.1.82
lxml==5.2.2
Markdown==3.6
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
nest-asyncio==1.6.0
nltk==3.8.1
numpy==1.26.4
ordered-set==4.1.0
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
pypdf==4.2.0
python-dateutil==2.9.0.post0
python-iso639==2024.4.27
python-magic==0.4.27
PyYAML==6.0.1
rapidfuzz==3.9.3
regex==2024.5.15
requests==2.32.3
requests-toolbelt==1.0.0
six==1.16.0
sniffio==1.3.1
soupsieve==2.5
SQLAlchemy==2.0.31
tabulate==0.9.0
tenacity==8.4.2
tqdm==4.66.4
typing-inspect==0.9.0
typing_extensions==4.12.2
unstructured==0.14.8
unstructured-client==0.23.7
urllib3==2.2.2
wrapt==1.16.0
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

async-timeout==4.0.3

attrs==23.2.0

backoff==2.2.1

beautifulsoup4==4.12.3

certifi==2024.6.2

chardet==5.2.0

charset-normalizer==3.3.2

click==8.1.7

dataclasses-json==0.6.7

deepdiff==7.0.1

emoji==2.12.1

exceptiongroup==1.2.1

filetype==1.2.0

frozenlist==1.4.1

greenlet==3.0.3

h11==0.14.0

httpcore==1.0.5

httpx==0.27.0

idna==3.7

joblib==1.4.2

jsonpatch==1.33

jsonpath-python==1.0.6

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langdetect==1.0.9

langsmith==0.1.82

lxml==5.2.2

Markdown==3.6

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

nest-asyncio==1.6.0

nltk==3.8.1

numpy==1.26.4

ordered-set==4.1.0

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

pypdf==4.2.0

python-dateutil==2.9.0.post0

python-iso639==2024.4.27

python-magic==0.4.27

PyYAML==6.0.1

rapidfuzz==3.9.3

regex==2024.5.15

requests==2.32.3

requests-toolbelt==1.0.0

six==1.16.0

sniffio==1.3.1

soupsieve==2.5

SQLAlchemy==2.0.31

tabulate==0.9.0

tenacity==8.4.2

tqdm==4.66.4

typing-inspect==0.9.0

typing_extensions==4.12.2

unstructured==0.14.8

unstructured-client==0.23.7

urllib3==2.2.2

wrapt==1.16.0

yarl==1.9.4

※ pip install

[PYTHON/LANGCHAIN] CSVLoader 클래스 : CSV 문자열을 문서로 로드하기

■ CSVLoader 클래스에서 CSV 문자열을 문서로 로드하는 방법을 보여준다. ▶ main.py


import tempfile
import os

from langchain_community.document_loaders.csv_loader import CSVLoader

sourceString = """
"Team"     , "Payroll (millions)", "Wins"
"Nationals", 81.34               , 98
"Reds"     , 82.20               , 97
"Yankees"  , 197.96              , 95
"Giants"   , 117.62              , 94
""".strip()

with tempfile.NamedTemporaryFile(delete = False, mode = "w+") as temporaryFileWrapper:
    temporaryFileWrapper.write(sourceString)
    temporaryFileWrapper.flush()

csvLoader = CSVLoader(file_path = temporaryFileWrapper.name)

documentList = csvLoader.load()

for document in documentList:
    print(document)

os.remove(temporaryFileWrapper.file.name)

"""
page_content='Team: Nationals\n"Payroll (millions)": 81.34\n"Wins": 98' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 0}
page_content='Team: Reds\n"Payroll (millions)": 82.20\n"Wins": 97' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 1}
page_content='Team: Yankees\n"Payroll (millions)": 197.96\n"Wins": 95' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 2}
page_content='Team: Giants\n"Payroll (millions)": 117.62\n"Wins": 94' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 3}
"""

import tempfile

import os

from langchain_community.document_loaders.csv_loader import CSVLoader

sourceString = """

"Team" , "Payroll (millions)", "Wins"

"Nationals", 81.34 , 98

"Reds" , 82.20 , 97

"Yankees" , 197.96 , 95

"Giants" , 117.62 , 94

""".strip()

with tempfile.NamedTemporaryFile(delete = False, mode = "w+") as temporaryFileWrapper:

temporaryFileWrapper.write(sourceString)

temporaryFileWrapper.flush()

csvLoader = CSVLoader(file_path = temporaryFileWrapper.name)

documentList = csvLoader.load()

for document in documentList:

print(document)

os.remove(temporaryFileWrapper.file.name)

"""

page_content='Team: Nationals\n"Payroll (millions)": 81.34\n"Wins": 98' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 0}

page_content='Team: Reds\n"Payroll (millions)": 82.20\n"Wins": 97' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 1}

page_content='Team: Yankees\n"Payroll (millions)": 197.96\n"Wins": 95' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 2}

page_content='Team: Giants\n"Payroll (millions)": 117.62\n"Wins": 94' metadata={'source': '/tmp/tmpk46bgw3c', 'row': 3}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install langchain-community 명령을 실행했다.

[PYTHON/LANGCHAIN] CSVLoader 클래스 : 생성자에서 source_column 인자를 사용해 CSV 파일 문서 로드하기

■ CSVLoader 클래스의 생성자에서 source_column 인자를 사용해 CSV 파일 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(file_path = "source.csv", source_column = "성명")

documentList = csvLoader.load()

for document in documentList:
    print(document)

"""
page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': '홍길동', 'row': 0}
page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': '김철수', 'row': 1}
page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': '이영희', 'row': 2}
"""

from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(file_path = "source.csv", source_column = "성명")

documentList = csvLoader.load()

for document in documentList:

print(document)

"""

page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': '홍길동', 'row': 0}

page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': '김철수', 'row': 1}

page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': '이영희', 'row': 2}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install

[PYTHON/LANGCHAIN] CSVLoader 클래스 : 생성자에서 csv_args 인자를 사용해 CSV 파일 문서 로드하기

■ CSVLoader 클래스의 생성자에서 csv_args 인자를 사용해 CSV 파일 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(
    file_path = "source.csv",
    csv_args  = {
        "delimiter"  : ",",
        "quotechar"  : '"',
        "fieldnames" : ["ID", "성명", "점수"] # CSV 파일에 헤더 라인이 없는 경우 설정한다.
    }
)

documentList = csvLoader.load()

for document in documentList:
    print(document)

"""
page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': './source.csv', 'row': 0}
page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': './source.csv', 'row': 1}
page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': './source.csv', 'row': 2}
"""

from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(

file_path = "source.csv",

csv_args = {

"delimiter" : ",",

"quotechar" : '"',

"fieldnames" : ["ID", "성명", "점수"] # CSV 파일에 헤더 라인이 없는 경우 설정한다.

}

)

documentList = csvLoader.load()

for document in documentList:

print(document)

"""

page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': './source.csv', 'row': 0}

page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': './source.csv', 'row': 1}

page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': './source.csv', 'row': 2}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install

[PYTHON/LANGCHAIN] CSVLoader 클래스 : load 메소드를 사용해 CSV 파일 문서 로드하기

■ CSVLoader 클래스의 load 메소드를 사용해 CSV 파일 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(file_path = "source.csv")

documentList = csvLoader.load()

for document in documentList:
    print(document)

"""
page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': './source.csv', 'row': 0}
page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': './source.csv', 'row': 1}
page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': './source.csv', 'row': 2}
"""

from langchain_community.document_loaders.csv_loader import CSVLoader

csvLoader = CSVLoader(file_path = "source.csv")

documentList = csvLoader.load()

for document in documentList:

print(document)

"""

page_content='ID: ID001\n성명: 홍길동\n점수: 90' metadata={'source': './source.csv', 'row': 0}

page_content='ID: ID002\n성명: 김철수\n점수: 100' metadata={'source': './source.csv', 'row': 1}

page_content='ID: ID003\n성명: 이영희\n점수: 80' metadata={'source': './source.csv', 'row': 2}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.6
langchain-community==0.2.6
langchain-core==0.2.10
langchain-text-splitters==0.2.2
langsmith==0.1.82
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.31
tenacity==8.4.2
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.2
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.6

langchain-community==0.2.6

langchain-core==0.2.10

langchain-text-splitters==0.2.2

langsmith==0.1.82

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.31

tenacity==8.4.2

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.2

yarl==1.9.4

※ pip install langchain-community

[PYTHON/LANGCHAIN] OPENAI 모델을 사용해 PDF 문서 로드 및 질문/응답 애플리케이션 만들기

■ OPENAI 모델을 사용해 PDF 문서 로드 및 질의 응답 애플리케이션을 만드는 방법을 보여준다. ▶ main.py


import os

from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters             import RecursiveCharacterTextSplitter
from langchain_openai                     import OpenAIEmbeddings
from langchain_chroma                     import Chroma
from langchain_openai                     import ChatOpenAI
from langchain_core.prompts               import ChatPromptTemplate
from langchain.chains.combine_documents   import create_stuff_documents_chain
from langchain.chains                     import create_retrieval_chain

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

# PDF 문서를 로드한다.
filePath = "./nke-10k-2023.pdf"

pyPDFLoader = PyPDFLoader(filePath)

documentList = pyPDFLoader.load()

# 문서를 분할한다.
recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 1000, chunk_overlap = 200)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

# 벡터 저장소를 설정한다.
chroma = Chroma.from_documents(documents = splitDocumentList, embedding = OpenAIEmbeddings())

# 벡터 저장소 검색기를 설정한다.
vectorStoreRetriever = chroma.as_retriever()

# LLM 모델을 설정한다.
chatOpenAI = ChatOpenAI(model = "gpt-4o")

# 채팅 프롬프트 템플리트를 설정한다.
systemMessage = (
    "You are an assistant for question-answering tasks. "
    "Use the following pieces of retrieved context to answer "
    "the question. If you don't know the answer, say that you "
    "don't know. Use three sentences maximum and keep the "
    "answer concise."
    "\n\n"
    "{context}"
)

chatPromptTemplate = ChatPromptTemplate.from_messages(
    [
        ("system", systemMessage),
        ("human" , "{input}"),
    ]
)

# 질의 응답 체인을 설정한다.
qnaRunnableSequence = create_stuff_documents_chain(chatOpenAI, chatPromptTemplate)

# RAG 체인을 설정한다.
ragRunnableSequence = create_retrieval_chain(vectorStoreRetriever, qnaRunnableSequence)

# 질의 응답을 실행한다.
resultDictionary = ragRunnableSequence.invoke({"input" : "What was Nike's revenue in 2023?"})

print(resultDictionary["context"][0].page_content)

"""
FISCAL 2023 NIKE BRAND REVENUE HIGHLIGHTS
The following tables present NIKE Brand revenues disaggregated by reportable operating segment, distribution channel and
major product line:
FISCAL 2023 COMPARED TO FISCAL 2022
•NIKE, Inc. Revenues were $51.2 billion in fiscal 2023, which increased 10% and 16% compared to fiscal 2022 on a reported
and currency-neutral basis, respectively. The increase was due to higher revenues in North America, Europe, Middle East &
Africa ("EMEA"), APLA and Greater China, which contributed approximately 7, 6, 2 and 1 percentage points to NIKE, Inc.
Revenues, respectively.
•NIKE Brand revenues, which represented over 90% of NIKE, Inc. Revenues,  increased  10% and 16% on a reported and
currency-neutral basis, respectively. This increase was primarily due to higher revenues in Men's, the Jordan Brand,
Women's and Kids' which grew 17%, 35%,11% and 10%, respectively, on a wholesale equivalent basis.
"""

import os

from langchain_community.document_loaders import PyPDFLoader

from langchain_text_splitters import RecursiveCharacterTextSplitter

from langchain_openai import OpenAIEmbeddings

from langchain_chroma import Chroma

from langchain_openai import ChatOpenAI

from langchain_core.prompts import ChatPromptTemplate

from langchain.chains.combine_documents import create_stuff_documents_chain

from langchain.chains import create_retrieval_chain

os.environ["OPENAI_API_KEY"] = "<OPENAI_API_KEY>"

# PDF 문서를 로드한다.

filePath = "./nke-10k-2023.pdf"

pyPDFLoader = PyPDFLoader(filePath)

documentList = pyPDFLoader.load()

# 문서를 분할한다.

recursiveCharacterTextSplitter = RecursiveCharacterTextSplitter(chunk_size = 1000, chunk_overlap = 200)

splitDocumentList = recursiveCharacterTextSplitter.split_documents(documentList)

# 벡터 저장소를 설정한다.

chroma = Chroma.from_documents(documents = splitDocumentList, embedding = OpenAIEmbeddings())

# 벡터 저장소 검색기를 설정한다.

vectorStoreRetriever = chroma.as_retriever()

# LLM 모델을 설정한다.

chatOpenAI = ChatOpenAI(model = "gpt-4o")

# 채팅 프롬프트 템플리트를 설정한다.

systemMessage = (

"You are an assistant for question-answering tasks. "

"Use the following pieces of retrieved context to answer "

"the question. If you don't know the answer, say that you "

"don't know. Use three sentences maximum and keep the "

"answer concise."

"\n\n"

"{context}"

)

chatPromptTemplate = ChatPromptTemplate.from_messages(

[

("system", systemMessage),

("human" , "{input}"),

]

)

# 질의 응답 체인을 설정한다.

qnaRunnableSequence = create_stuff_documents_chain(chatOpenAI, chatPromptTemplate)

# RAG 체인을 설정한다.

ragRunnableSequence = create_retrieval_chain(vectorStoreRetriever, qnaRunnableSequence)

# 질의 응답을 실행한다.

resultDictionary = ragRunnableSequence.invoke({"input" : "What was Nike's revenue in 2023?"})

print(resultDictionary["context"][0].page_content)

"""

FISCAL 2023 NIKE BRAND REVENUE HIGHLIGHTS

The following tables present NIKE Brand revenues disaggregated by reportable operating segment, distribution channel and

major product line:

FISCAL 2023 COMPARED TO FISCAL 2022

•NIKE, Inc. Revenues were $51.2 billion in fiscal 2023, which increased 10% and 16% compared to fiscal 2022 on a reported

and currency-neutral basis, respectively. The increase was due to higher revenues in North America, Europe, Middle East &

Africa ("EMEA"), APLA and Greater China, which contributed approximately 7, 6, 2 and 1 percentage points to NIKE, Inc.

Revenues, respectively.

•NIKE Brand revenues, which represented over 90% of NIKE, Inc. Revenues, increased 10% and 16% on a reported and

currency-neutral basis, respectively. This increase was primarily due to higher revenues in Men's, the Jordan Brand,

Women's and Kids' which grew 17%, 35%,11% and 10%, respectively, on a wholesale equivalent basis.

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
asgiref==3.8.1
async-timeout==4.0.3
attrs==23.2.0
backoff==2.2.1
bcrypt==4.1.3
build==1.2.1
cachetools==5.3.3
certifi==2024.6.2
charset-normalizer==3.3.2
chroma-hnswlib==0.7.3
chromadb==0.5.0
click==8.1.7
coloredlogs==15.0.1
dataclasses-json==0.6.7
Deprecated==1.2.14
distro==1.9.0
dnspython==2.6.1
email_validator==2.1.1
exceptiongroup==1.2.1
fastapi==0.111.0
fastapi-cli==0.0.4
filelock==3.15.1
flatbuffers==24.3.25
frozenlist==1.4.1
fsspec==2024.6.0
google-auth==2.30.0
googleapis-common-protos==1.63.1
greenlet==3.0.3
grpcio==1.64.1
h11==0.14.0
httpcore==1.0.5
httptools==0.6.1
httpx==0.27.0
huggingface-hub==0.23.3
humanfriendly==10.0
idna==3.7
importlib_metadata==7.1.0
importlib_resources==6.4.0
Jinja2==3.1.4
jsonpatch==1.33
jsonpointer==3.0.0
kubernetes==30.1.0
langchain==0.2.4
langchain-chroma==0.1.1
langchain-community==0.2.4
langchain-core==0.2.6
langchain-openai==0.1.8
langchain-text-splitters==0.2.1
langsmith==0.1.77
markdown-it-py==3.0.0
MarkupSafe==2.1.5
marshmallow==3.21.3
mdurl==0.1.2
mmh3==4.1.0
monotonic==1.6
mpmath==1.3.0
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
oauthlib==3.2.2
onnxruntime==1.18.0
openai==1.34.0
opentelemetry-api==1.25.0
opentelemetry-exporter-otlp-proto-common==1.25.0
opentelemetry-exporter-otlp-proto-grpc==1.25.0
opentelemetry-instrumentation==0.46b0
opentelemetry-instrumentation-asgi==0.46b0
opentelemetry-instrumentation-fastapi==0.46b0
opentelemetry-proto==1.25.0
opentelemetry-sdk==1.25.0
opentelemetry-semantic-conventions==0.46b0
opentelemetry-util-http==0.46b0
orjson==3.10.5
overrides==7.7.0
packaging==24.1
posthog==3.5.0
protobuf==4.25.3
pyasn1==0.6.0
pyasn1_modules==0.4.0
pydantic==2.7.4
pydantic_core==2.18.4
Pygments==2.18.0
pypdf==4.2.0
PyPika==0.48.9
pyproject_hooks==1.1.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-multipart==0.0.9
PyYAML==6.0.1
regex==2024.5.15
requests==2.32.3
requests-oauthlib==2.0.0
rich==13.7.1
rsa==4.9
shellingham==1.5.4
six==1.16.0
sniffio==1.3.1
SQLAlchemy==2.0.30
starlette==0.37.2
sympy==1.12.1
tenacity==8.3.0
tiktoken==0.7.0
tokenizers==0.19.1
tomli==2.0.1
tqdm==4.66.4
typer==0.12.3
typing-inspect==0.9.0
typing_extensions==4.12.2
ujson==5.10.0
urllib3==2.2.1
uvicorn==0.30.1
uvloop==0.19.0
watchfiles==0.22.0
websocket-client==1.8.0
websockets==12.0
wrapt==1.16.0
yarl==1.9.4
zipp==3.19.2

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.4.0

asgiref==3.8.1

async-timeout==4.0.3

attrs==23.2.0

backoff==2.2.1

bcrypt==4.1.3

build==1.2.1

cachetools==5.3.3

certifi==2024.6.2

charset-normalizer==3.3.2

chroma-hnswlib==0.7.3

chromadb==0.5.0

click==8.1.7

coloredlogs==15.0.1

dataclasses-json==0.6.7

Deprecated==1.2.14

distro==1.9.0

dnspython==2.6.1

email_validator==2.1.1

exceptiongroup==1.2.1

fastapi==0.111.0

fastapi-cli==0.0.4

filelock==3.15.1

flatbuffers==24.3.25

frozenlist==1.4.1

fsspec==2024.6.0

google-auth==2.30.0

googleapis-common-protos==1.63.1

greenlet==3.0.3

grpcio==1.64.1

h11==0.14.0

httpcore==1.0.5

httptools==0.6.1

httpx==0.27.0

huggingface-hub==0.23.3

humanfriendly==10.0

idna==3.7

importlib_metadata==7.1.0

importlib_resources==6.4.0

Jinja2==3.1.4

jsonpatch==1.33

jsonpointer==3.0.0

kubernetes==30.1.0

langchain==0.2.4

langchain-chroma==0.1.1

langchain-community==0.2.4

langchain-core==0.2.6

langchain-openai==0.1.8

langchain-text-splitters==0.2.1

langsmith==0.1.77

markdown-it-py==3.0.0

MarkupSafe==2.1.5

marshmallow==3.21.3

mdurl==0.1.2

mmh3==4.1.0

monotonic==1.6

mpmath==1.3.0

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

oauthlib==3.2.2

onnxruntime==1.18.0

openai==1.34.0

opentelemetry-api==1.25.0

opentelemetry-exporter-otlp-proto-common==1.25.0

opentelemetry-exporter-otlp-proto-grpc==1.25.0

opentelemetry-instrumentation==0.46b0

opentelemetry-instrumentation-asgi==0.46b0

opentelemetry-instrumentation-fastapi==0.46b0

opentelemetry-proto==1.25.0

opentelemetry-sdk==1.25.0

opentelemetry-semantic-conventions==0.46b0

opentelemetry-util-http==0.46b0

orjson==3.10.5

overrides==7.7.0

packaging==24.1

posthog==3.5.0

protobuf==4.25.3

pyasn1==0.6.0

pyasn1_modules==0.4.0

pydantic==2.7.4

pydantic_core==2.18.4

Pygments==2.18.0

pypdf==4.2.0

PyPika==0.48.9

pyproject_hooks==1.1.0

python-dateutil==2.9.0.post0

python-dotenv==1.0.1

python-multipart==0.0.9

PyYAML==6.0.1

regex==2024.5.15

requests==2.32.3

requests-oauthlib==2.0.0

rich==13.7.1

rsa==4.9

shellingham==1.5.4

six==1.16.0

sniffio==1.3.1

SQLAlchemy==2.0.30

starlette==0.37.2

sympy==1.12.1

tenacity==8.3.0

tiktoken==0.7.0

tokenizers==0.19.1

tomli==2.0.1

tqdm==4.66.4

typer==0.12.3

typing-inspect==0.9.0

typing_extensions==4.12.2

ujson==5.10.0

urllib3==2.2.1

uvicorn==0.30.1

uvloop==0.19.0

watchfiles==0.22.0

websocket-client==1.8.0

websockets==12.0

wrapt==1.16.0

yarl==1.9.4

zipp==3.19.2

※ pip

[PYTHON/LANGCHAIN] PyPDFLoader 클래스 : load 메소드를 사용해 PDF 문서 로드하기

■ PyPDFLoader 클래스의 load 메소드를 사용해 PDF 문서를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import PyPDFLoader

pyPDFLoader = PyPDFLoader("nke-10k-2023.pdf")

documentList = pyPDFLoader.load()

print("문서 길이   :", len(documentList))

document = documentList[0]

print("첫번째 문서 :")
print("    페이지 컨텐츠 :", document.page_content)
print("    메타 데이터   :", document.metadata)

"""
문서 길이   : 106
첫번째 문서 :
    페이지 컨텐츠 : FORM 10-K FORM 10-K
    메타 데이터   : {'source': 'nke-10k-2023.pdf', 'page': 0}
"""

from langchain_community.document_loaders import PyPDFLoader

pyPDFLoader = PyPDFLoader("nke-10k-2023.pdf")

documentList = pyPDFLoader.load()

print("문서 길이 :", len(documentList))

document = documentList[0]

print("첫번째 문서 :")

print(" 페이지 컨텐츠 :", document.page_content)

print(" 메타 데이터 :", document.metadata)

"""

문서 길이 : 106

첫번째 문서 :

페이지 컨텐츠 : FORM 10-K FORM 10-K

메타 데이터 : {'source': 'nke-10k-2023.pdf', 'page': 0}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.4
langchain-community==0.2.4
langchain-core==0.2.6
langchain-text-splitters==0.2.1
langsmith==0.1.77
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.5
packaging==24.1
pydantic==2.7.4
pydantic_core==2.18.4
pypdf==4.2.0
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.30
tenacity==8.3.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.1
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.4

langchain-community==0.2.4

langchain-core==0.2.6

langchain-text-splitters==0.2.1

langsmith==0.1.77

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.5

packaging==24.1

pydantic==2.7.4

pydantic_core==2.18.4

pypdf==4.2.0

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.30

tenacity==8.3.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.1

yarl==1.9.4

※ pip install pypdf langchain_community

[PYTHON/LANGCHAIN] Document 클래스 : metadata 속성을 사용해 유튜브 동영상 정보 구하기

■ Document 클래스의 metadata 속성을 사용해 유튜브 동영상 정보를 구하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

documentList = youtubeLoader.load()

document = documentList[0]

metadataDictionary = document.metadata

print(metadataDictionary)

"""
{
    'source'        : 'o7C9ld6Ln-M',
    'title'         : 'LangServe and LangChain Templates Webinar',
    'description'   : 'Unknown',
    'view_count'    : 5363,
    'thumbnail_url' : 'https://i.ytimg.com/vi/o7C9ld6Ln-M/hqdefault.jpg?sqp=-oaymwEXCJADEOABSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDf7gvV8D3I2UFy0UsA2Wh0qUhA-A',
    'publish_date'  : '2023-11-02 00:00:00',
    'length'        : 2441,
    'author'        : 'LangChain'
}
"""

from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

documentList = youtubeLoader.load()

document = documentList[0]

metadataDictionary = document.metadata

print(metadataDictionary)

"""

{

'source' : 'o7C9ld6Ln-M',

'title' : 'LangServe and LangChain Templates Webinar',

'description' : 'Unknown',

'view_count' : 5363,

'thumbnail_url' : 'https://i.ytimg.com/vi/o7C9ld6Ln-M/hqdefault.jpg?sqp=-oaymwEXCJADEOABSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDf7gvV8D3I2UFy0UsA2Wh0qUhA-A',

'publish_date' : '2023-11-02 00:00:00',

'length' : 2441,

'author' : 'LangChain'

}

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.3
langchain-community==0.2.4
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.77
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.4
packaging==23.2
pydantic==2.7.4
pydantic_core==2.18.4
pytube==15.0.0
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.30
tenacity==8.3.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.1
yarl==1.9.4
youtube-transcript-api==0.6.2

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.3

langchain-community==0.2.4

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.77

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.4

packaging==23.2

pydantic==2.7.4

pydantic_core==2.18.4

pytube==15.0.0

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.30

tenacity==8.3.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.1

yarl==1.9.4

youtube-transcript-api==0.6.2

※ pip install langchain-community

[PYTHON/LANGCHAIN] YoutubeLoader 클래스 : load 메소드를 사용해 유튜브 동영상 정보 로드하기

■ YoutubeLoader 클래스의 load 메소드를 사용해 유튜브 동영상 정보를 로드하는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

documentList = youtubeLoader.load()

"""
1
"""

from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

documentList = youtubeLoader.load()

"""

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.3
langchain-community==0.2.4
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.77
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.4
packaging==23.2
pydantic==2.7.4
pydantic_core==2.18.4
pytube==15.0.0
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.30
tenacity==8.3.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.1
yarl==1.9.4
youtube-transcript-api==0.6.2

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.3

langchain-community==0.2.4

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.77

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.4

packaging==23.2

pydantic==2.7.4

pydantic_core==2.18.4

pytube==15.0.0

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.30

tenacity==8.3.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.1

yarl==1.9.4

youtube-transcript-api==0.6.2

※ pip install langchain-community

[PYTHON/LANGCHAIN] from_youtube_url 함수 : YoutubeLoader 객체 만들기

■ from_youtube_url 함수를 사용해 YoutubeLoader 객체를 만드는 방법을 보여준다. ▶ main.py


from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

from langchain_community.document_loaders import YoutubeLoader

youtubeLoader = YoutubeLoader.from_youtube_url("https://www.youtube.com/watch?v=HAn9vnJy6S4", add_video_info = True)

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.3
langchain-community==0.2.4
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.77
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.4
packaging==23.2
pydantic==2.7.4
pydantic_core==2.18.4
pytube==15.0.0
PyYAML==6.0.1
requests==2.32.3
SQLAlchemy==2.0.30
tenacity==8.3.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.1
yarl==1.9.4
youtube-transcript-api==0.6.2

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.3

langchain-community==0.2.4

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.77

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.4

packaging==23.2

pydantic==2.7.4

pydantic_core==2.18.4

pytube==15.0.0

PyYAML==6.0.1

requests==2.32.3

SQLAlchemy==2.0.30

tenacity==8.3.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.1

yarl==1.9.4

youtube-transcript-api==0.6.2

※ pip install langchain-community pytube youtube-transcript-api 명령을

[PYTHON/LANGCHAIN] WebBaseLoader 클래스 : load 메소드를 사용해 웹 문서 로드하기

■ WebBaseLoader 클래스의 load 메소드를 사용해 웹 문서를 로드하는 방법을 보여준다. ▶ main.py


import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(
    web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),
    bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))
)

documentList = webBaseLoader.load()

import bs4

from langchain_community.document_loaders import WebBaseLoader

webBaseLoader = WebBaseLoader(

web_paths = ("https://lilianweng.github.io/posts/2023-06-23-agent/",),

bs_kwargs = dict(parse_only = bs4.SoupStrainer(class_ = ("post-content", "post-title", "post-header")))

)

documentList = webBaseLoader.load()

▶ requirements.txt


aiohttp==3.9.5
aiosignal==1.3.1
annotated-types==0.7.0
async-timeout==4.0.3
attrs==23.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
certifi==2024.6.2
charset-normalizer==3.3.2
dataclasses-json==0.6.7
frozenlist==1.4.1
greenlet==3.0.3
idna==3.7
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.3
langchain-community==0.2.4
langchain-core==0.2.5
langchain-text-splitters==0.2.1
langsmith==0.1.76
marshmallow==3.21.3
multidict==6.0.5
mypy-extensions==1.0.0
numpy==1.26.4
orjson==3.10.4
packaging==23.2
pydantic==2.7.3
pydantic_core==2.18.4
PyYAML==6.0.1
requests==2.32.3
soupsieve==2.5
SQLAlchemy==2.0.30
tenacity==8.3.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.1
yarl==1.9.4

aiohttp==3.9.5

aiosignal==1.3.1

annotated-types==0.7.0

async-timeout==4.0.3

attrs==23.2.0

beautifulsoup4==4.12.3

bs4==0.0.2

certifi==2024.6.2

charset-normalizer==3.3.2

dataclasses-json==0.6.7

frozenlist==1.4.1

greenlet==3.0.3

idna==3.7

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.2.3

langchain-community==0.2.4

langchain-core==0.2.5

langchain-text-splitters==0.2.1

langsmith==0.1.76

marshmallow==3.21.3

multidict==6.0.5

mypy-extensions==1.0.0

numpy==1.26.4

orjson==3.10.4

packaging==23.2

pydantic==2.7.3

pydantic_core==2.18.4

PyYAML==6.0.1

requests==2.32.3

soupsieve==2.5

SQLAlchemy==2.0.30

tenacity==8.3.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.1

yarl==1.9.4

※ pip install bs4 langchain_community