[PYTHON/LANGCHAIN] BSHTMLLoader 클래스 : 윈도우즈 운영 체제에서 UTF-8 인코딩 파일 사용하기

■ BSHTMLLoader 클래스를 사용해 윈도우즈 운영 체제에서 UTF-8 인코딩 파일을 사용하는 방법을 보여준다.

▶ main.py


import requests

from langchain_community.document_loaders import BSHTMLLoader
from langchain.schema                     import Document
from bs4                                  import BeautifulSoup

response = requests.get("https://en.wikipedia.org/wiki/Car")

with open("car.html", "w", encoding = "utf-8") as textIOWrapper:
    textIOWrapper.write(response.text)

class CustomBSHTMLLoader(BSHTMLLoader):
    def load(self) -> list[Document]:
        try:
            with open(self.file_path, "r", encoding = "utf-8") as textIOWrapper:
                beautifulSoup = BeautifulSoup(textIOWrapper, "html.parser")
                text = beautifulSoup.get_text()
        except UnicodeDecodeError:
            print("UTF-8 디코딩 중 오류가 발생했습니다.")
            with open(self.file_path, "r", encoding = "cp949") as textIOWrapper:
                beautifulSoup = BeautifulSoup(textIOWrapper, "html.parser")
                text = beautifulSoup.get_text()
        metadata = {"source" : self.file_path}
        return [Document(page_content = text, metadata = metadata)]

customBSHTMLLoader = CustomBSHTMLLoader("car.html")

documentList = customBSHTMLLoader.load()

import requests

from langchain_community.document_loaders import BSHTMLLoader

from langchain.schema import Document

from bs4 import BeautifulSoup

response = requests.get("https://en.wikipedia.org/wiki/Car")

with open("car.html", "w", encoding = "utf-8") as textIOWrapper:

textIOWrapper.write(response.text)

class CustomBSHTMLLoader(BSHTMLLoader):

def load(self) -> list[Document]:

try:

with open(self.file_path, "r", encoding = "utf-8") as textIOWrapper:

beautifulSoup = BeautifulSoup(textIOWrapper, "html.parser")

text = beautifulSoup.get_text()

except UnicodeDecodeError:

print("UTF-8 디코딩 중 오류가 발생했습니다.")

with open(self.file_path, "r", encoding = "cp949") as textIOWrapper:

beautifulSoup = BeautifulSoup(textIOWrapper, "html.parser")

text = beautifulSoup.get_text()

metadata = {"source" : self.file_path}

return [Document(page_content = text, metadata = metadata)]

customBSHTMLLoader = CustomBSHTMLLoader("car.html")

documentList = customBSHTMLLoader.load()

▶ requirements.txt


aiohappyeyeballs==2.4.4
aiohttp==3.11.9
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.2.post1
attrs==24.2.0
beautifulsoup4==4.12.3
bs4==0.0.2
certifi==2024.8.30
charset-normalizer==3.4.0
dataclasses-json==0.6.7
frozenlist==1.5.0
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.7
httpx==0.28.0
httpx-sse==0.4.0
idna==3.10
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.3.9
langchain-community==0.3.9
langchain-core==0.3.21
langchain-text-splitters==0.3.2
langsmith==0.1.147
lxml==5.3.0
marshmallow==3.23.1
multidict==6.1.0
mypy-extensions==1.0.0
numpy==2.1.3
orjson==3.10.12
packaging==24.2
propcache==0.2.1
pydantic==2.10.3
pydantic-settings==2.6.1
pydantic_core==2.27.1
python-dotenv==1.0.1
PyYAML==6.0.2
requests==2.32.3
requests-toolbelt==1.0.0
sniffio==1.3.1
soupsieve==2.6
SQLAlchemy==2.0.36
tenacity==9.0.0
typing-inspect==0.9.0
typing_extensions==4.12.2
urllib3==2.2.3
yarl==1.18.3

aiohappyeyeballs==2.4.4

aiohttp==3.11.9

aiosignal==1.3.1

annotated-types==0.7.0

anyio==4.6.2.post1

attrs==24.2.0

beautifulsoup4==4.12.3

bs4==0.0.2

certifi==2024.8.30

charset-normalizer==3.4.0

dataclasses-json==0.6.7

frozenlist==1.5.0

greenlet==3.1.1

h11==0.14.0

httpcore==1.0.7

httpx==0.28.0

httpx-sse==0.4.0

idna==3.10

jsonpatch==1.33

jsonpointer==3.0.0

langchain==0.3.9

langchain-community==0.3.9

langchain-core==0.3.21

langchain-text-splitters==0.3.2

langsmith==0.1.147

lxml==5.3.0

marshmallow==3.23.1

multidict==6.1.0

mypy-extensions==1.0.0

numpy==2.1.3

orjson==3.10.12

packaging==24.2

propcache==0.2.1

pydantic==2.10.3

pydantic-settings==2.6.1

pydantic_core==2.27.1

python-dotenv==1.0.1

PyYAML==6.0.2

requests==2.32.3

requests-toolbelt==1.0.0

sniffio==1.3.1

soupsieve==2.6

SQLAlchemy==2.0.36

tenacity==9.0.0

typing-inspect==0.9.0

typing_extensions==4.12.2

urllib3==2.2.3

yarl==1.18.3

※ pip install langchain-community bs4 lxml 명령을 실행했다.