Add Steel loader for Langchain

libs/community/langchain_community/document_loaders/__init__.py

@@ -19,519 +19,522 @@
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from langchain_community.document_loaders.acreom import (
         AcreomLoader,
     )
     from langchain_community.document_loaders.airbyte import (
         AirbyteCDKLoader,
         AirbyteGongLoader,
         AirbyteHubspotLoader,
         AirbyteSalesforceLoader,
         AirbyteShopifyLoader,
         AirbyteStripeLoader,
         AirbyteTypeformLoader,
         AirbyteZendeskSupportLoader,
     )
     from langchain_community.document_loaders.airbyte_json import (
         AirbyteJSONLoader,
     )
     from langchain_community.document_loaders.airtable import (
         AirtableLoader,
     )
     from langchain_community.document_loaders.apify_dataset import (
         ApifyDatasetLoader,
     )
     from langchain_community.document_loaders.arcgis_loader import (
         ArcGISLoader,
     )
     from langchain_community.document_loaders.arxiv import (
         ArxivLoader,
     )
     from langchain_community.document_loaders.assemblyai import (
         AssemblyAIAudioLoaderById,
         AssemblyAIAudioTranscriptLoader,
     )
     from langchain_community.document_loaders.astradb import (
         AstraDBLoader,
     )
     from langchain_community.document_loaders.async_html import (
         AsyncHtmlLoader,
     )
     from langchain_community.document_loaders.athena import (
         AthenaLoader,
     )
     from langchain_community.document_loaders.azlyrics import (
         AZLyricsLoader,
     )
     from langchain_community.document_loaders.azure_ai_data import (
         AzureAIDataLoader,
     )
     from langchain_community.document_loaders.azure_blob_storage_container import (
         AzureBlobStorageContainerLoader,
     )
     from langchain_community.document_loaders.azure_blob_storage_file import (
         AzureBlobStorageFileLoader,
     )
     from langchain_community.document_loaders.bibtex import (
         BibtexLoader,
     )
     from langchain_community.document_loaders.bigquery import (
         BigQueryLoader,
     )
     from langchain_community.document_loaders.bilibili import (
         BiliBiliLoader,
     )
     from langchain_community.document_loaders.blackboard import (
         BlackboardLoader,
     )
     from langchain_community.document_loaders.blob_loaders import (
         Blob,
         BlobLoader,
         FileSystemBlobLoader,
         YoutubeAudioLoader,
     )
     from langchain_community.document_loaders.blockchain import (
         BlockchainDocumentLoader,
     )
     from langchain_community.document_loaders.brave_search import (
         BraveSearchLoader,
     )
     from langchain_community.document_loaders.browserbase import (
         BrowserbaseLoader,
     )
     from langchain_community.document_loaders.browserless import (
         BrowserlessLoader,
     )
     from langchain_community.document_loaders.cassandra import (
         CassandraLoader,
     )
     from langchain_community.document_loaders.chatgpt import (
         ChatGPTLoader,
     )
     from langchain_community.document_loaders.chm import (
         UnstructuredCHMLoader,
     )
     from langchain_community.document_loaders.chromium import (
         AsyncChromiumLoader,
     )
     from langchain_community.document_loaders.college_confidential import (
         CollegeConfidentialLoader,
     )
     from langchain_community.document_loaders.concurrent import (
         ConcurrentLoader,
     )
     from langchain_community.document_loaders.confluence import (
         ConfluenceLoader,
     )
     from langchain_community.document_loaders.conllu import (
         CoNLLULoader,
     )
     from langchain_community.document_loaders.couchbase import (
         CouchbaseLoader,
     )
     from langchain_community.document_loaders.csv_loader import (
         CSVLoader,
         UnstructuredCSVLoader,
     )
     from langchain_community.document_loaders.cube_semantic import (
         CubeSemanticLoader,
     )
     from langchain_community.document_loaders.datadog_logs import (
         DatadogLogsLoader,
     )
     from langchain_community.document_loaders.dataframe import (
         DataFrameLoader,
     )
     from langchain_community.document_loaders.dedoc import (
         DedocAPIFileLoader,
         DedocFileLoader,
     )
     from langchain_community.document_loaders.diffbot import (
         DiffbotLoader,
     )
     from langchain_community.document_loaders.directory import (
         DirectoryLoader,
     )
     from langchain_community.document_loaders.discord import (
         DiscordChatLoader,
     )
     from langchain_community.document_loaders.doc_intelligence import (
         AzureAIDocumentIntelligenceLoader,
     )
     from langchain_community.document_loaders.docugami import (
         DocugamiLoader,
     )
     from langchain_community.document_loaders.docusaurus import (
         DocusaurusLoader,
     )
     from langchain_community.document_loaders.dropbox import (
         DropboxLoader,
     )
     from langchain_community.document_loaders.duckdb_loader import (
         DuckDBLoader,
     )
     from langchain_community.document_loaders.email import (
         OutlookMessageLoader,
         UnstructuredEmailLoader,
     )
     from langchain_community.document_loaders.epub import (
         UnstructuredEPubLoader,
     )
     from langchain_community.document_loaders.etherscan import (
         EtherscanLoader,
     )
     from langchain_community.document_loaders.evernote import (
         EverNoteLoader,
     )
     from langchain_community.document_loaders.excel import (
         UnstructuredExcelLoader,
     )
     from langchain_community.document_loaders.facebook_chat import (
         FacebookChatLoader,
     )
     from langchain_community.document_loaders.fauna import (
         FaunaLoader,
     )
     from langchain_community.document_loaders.figma import (
         FigmaFileLoader,
     )
     from langchain_community.document_loaders.firecrawl import (
         FireCrawlLoader,
     )
     from langchain_community.document_loaders.gcs_directory import (
         GCSDirectoryLoader,
     )
     from langchain_community.document_loaders.gcs_file import (
         GCSFileLoader,
     )
     from langchain_community.document_loaders.geodataframe import (
         GeoDataFrameLoader,
     )
     from langchain_community.document_loaders.git import (
         GitLoader,
     )
     from langchain_community.document_loaders.gitbook import (
         GitbookLoader,
     )
     from langchain_community.document_loaders.github import (
         GithubFileLoader,
         GitHubIssuesLoader,
     )
     from langchain_community.document_loaders.glue_catalog import (
         GlueCatalogLoader,
     )
     from langchain_community.document_loaders.google_speech_to_text import (
         GoogleSpeechToTextLoader,
     )
     from langchain_community.document_loaders.googledrive import (
         GoogleDriveLoader,
     )
     from langchain_community.document_loaders.gutenberg import (
         GutenbergLoader,
     )
     from langchain_community.document_loaders.hn import (
         HNLoader,
     )
     from langchain_community.document_loaders.html import (
         UnstructuredHTMLLoader,
     )
     from langchain_community.document_loaders.html_bs import (
         BSHTMLLoader,
     )
     from langchain_community.document_loaders.hugging_face_dataset import (
         HuggingFaceDatasetLoader,
     )
     from langchain_community.document_loaders.hugging_face_model import (
         HuggingFaceModelLoader,
     )
     from langchain_community.document_loaders.ifixit import (
         IFixitLoader,
     )
     from langchain_community.document_loaders.image import (
         UnstructuredImageLoader,
     )
     from langchain_community.document_loaders.image_captions import (
         ImageCaptionLoader,
     )
     from langchain_community.document_loaders.imsdb import (
         IMSDbLoader,
     )
     from langchain_community.document_loaders.iugu import (
         IuguLoader,
     )
     from langchain_community.document_loaders.joplin import (
         JoplinLoader,
     )
     from langchain_community.document_loaders.json_loader import (
         JSONLoader,
     )
     from langchain_community.document_loaders.kinetica_loader import KineticaLoader
     from langchain_community.document_loaders.lakefs import (
         LakeFSLoader,
     )
     from langchain_community.document_loaders.larksuite import (
         LarkSuiteDocLoader,
     )
     from langchain_community.document_loaders.llmsherpa import (
         LLMSherpaFileLoader,
     )
     from langchain_community.document_loaders.markdown import (
         UnstructuredMarkdownLoader,
     )
     from langchain_community.document_loaders.mastodon import (
         MastodonTootsLoader,
     )
     from langchain_community.document_loaders.max_compute import (
         MaxComputeLoader,
     )
     from langchain_community.document_loaders.mediawikidump import (
         MWDumpLoader,
     )
     from langchain_community.document_loaders.merge import (
         MergedDataLoader,
     )
     from langchain_community.document_loaders.mhtml import (
         MHTMLLoader,
     )
     from langchain_community.document_loaders.modern_treasury import (
         ModernTreasuryLoader,
     )
     from langchain_community.document_loaders.mongodb import (
         MongodbLoader,
     )
     from langchain_community.document_loaders.needle import (
         NeedleLoader,
     )
     from langchain_community.document_loaders.news import (
         NewsURLLoader,
     )
     from langchain_community.document_loaders.notebook import (
         NotebookLoader,
     )
     from langchain_community.document_loaders.notion import (
         NotionDirectoryLoader,
     )
     from langchain_community.document_loaders.notiondb import (
         NotionDBLoader,
     )
     from langchain_community.document_loaders.obs_directory import (
         OBSDirectoryLoader,
     )
     from langchain_community.document_loaders.obs_file import (
         OBSFileLoader,
     )
     from langchain_community.document_loaders.obsidian import (
         ObsidianLoader,
     )
     from langchain_community.document_loaders.odt import (
         UnstructuredODTLoader,
     )
     from langchain_community.document_loaders.onedrive import (
         OneDriveLoader,
     )
     from langchain_community.document_loaders.onedrive_file import (
         OneDriveFileLoader,
     )
     from langchain_community.document_loaders.open_city_data import (
         OpenCityDataLoader,
     )
     from langchain_community.document_loaders.oracleadb_loader import (
         OracleAutonomousDatabaseLoader,
     )
     from langchain_community.document_loaders.oracleai import (
         OracleDocLoader,
         OracleTextSplitter,
     )
     from langchain_community.document_loaders.org_mode import (
         UnstructuredOrgModeLoader,
     )
     from langchain_community.document_loaders.pdf import (
         AmazonTextractPDFLoader,
         DedocPDFLoader,
         MathpixPDFLoader,
         OnlinePDFLoader,
         PagedPDFSplitter,
         PDFMinerLoader,
         PDFMinerPDFasHTMLLoader,
         PDFPlumberLoader,
         PyMuPDFLoader,
         PyPDFDirectoryLoader,
         PyPDFium2Loader,
         PyPDFLoader,
         UnstructuredPDFLoader,
     )
     from langchain_community.document_loaders.pebblo import (
         PebbloSafeLoader,
         PebbloTextLoader,
     )
     from langchain_community.document_loaders.polars_dataframe import (
         PolarsDataFrameLoader,
     )
     from langchain_community.document_loaders.powerpoint import (
         UnstructuredPowerPointLoader,
     )
     from langchain_community.document_loaders.psychic import (
         PsychicLoader,
     )
     from langchain_community.document_loaders.pubmed import (
         PubMedLoader,
     )
     from langchain_community.document_loaders.pyspark_dataframe import (
         PySparkDataFrameLoader,
     )
     from langchain_community.document_loaders.python import (
         PythonLoader,
     )
     from langchain_community.document_loaders.readthedocs import (
         ReadTheDocsLoader,
     )
     from langchain_community.document_loaders.recursive_url_loader import (
         RecursiveUrlLoader,
     )
     from langchain_community.document_loaders.reddit import (
         RedditPostsLoader,
     )
     from langchain_community.document_loaders.roam import (
         RoamLoader,
     )
     from langchain_community.document_loaders.rocksetdb import (
         RocksetLoader,
     )
     from langchain_community.document_loaders.rss import (
         RSSFeedLoader,
     )
     from langchain_community.document_loaders.rst import (
         UnstructuredRSTLoader,
     )
     from langchain_community.document_loaders.rtf import (
         UnstructuredRTFLoader,
     )
     from langchain_community.document_loaders.s3_directory import (
         S3DirectoryLoader,
     )
     from langchain_community.document_loaders.s3_file import (
         S3FileLoader,
     )
     from langchain_community.document_loaders.scrapfly import (
         ScrapflyLoader,
     )
     from langchain_community.document_loaders.scrapingant import (
         ScrapingAntLoader,
     )
     from langchain_community.document_loaders.sharepoint import (
         SharePointLoader,
     )
     from langchain_community.document_loaders.sitemap import (
         SitemapLoader,
     )
     from langchain_community.document_loaders.slack_directory import (
         SlackDirectoryLoader,
     )
     from langchain_community.document_loaders.snowflake_loader import (
         SnowflakeLoader,
     )
     from langchain_community.document_loaders.spider import (
         SpiderLoader,
     )
     from langchain_community.document_loaders.spreedly import (
         SpreedlyLoader,
     )
     from langchain_community.document_loaders.sql_database import (
         SQLDatabaseLoader,
     )
     from langchain_community.document_loaders.srt import (
         SRTLoader,
     )
     from langchain_community.document_loaders.stripe import (
         StripeLoader,
     )
     from langchain_community.document_loaders.surrealdb import (
         SurrealDBLoader,
     )
     from langchain_community.document_loaders.telegram import (
         TelegramChatApiLoader,
         TelegramChatFileLoader,
         TelegramChatLoader,
     )
     from langchain_community.document_loaders.tencent_cos_directory import (
         TencentCOSDirectoryLoader,
     )
     from langchain_community.document_loaders.tencent_cos_file import (
         TencentCOSFileLoader,
     )
     from langchain_community.document_loaders.tensorflow_datasets import (
         TensorflowDatasetLoader,
     )
     from langchain_community.document_loaders.text import (
         TextLoader,
     )
     from langchain_community.document_loaders.tidb import (
         TiDBLoader,
     )
     from langchain_community.document_loaders.tomarkdown import (
         ToMarkdownLoader,
     )
     from langchain_community.document_loaders.toml import (
         TomlLoader,
     )
     from langchain_community.document_loaders.trello import (
         TrelloLoader,
     )
     from langchain_community.document_loaders.tsv import (
         UnstructuredTSVLoader,
     )
     from langchain_community.document_loaders.twitter import (
         TwitterTweetLoader,
     )
     from langchain_community.document_loaders.unstructured import (
         UnstructuredAPIFileIOLoader,
         UnstructuredAPIFileLoader,
         UnstructuredFileIOLoader,
         UnstructuredFileLoader,
     )
     from langchain_community.document_loaders.url import (
         UnstructuredURLLoader,
     )
     from langchain_community.document_loaders.url_playwright import (
         PlaywrightURLLoader,
     )
     from langchain_community.document_loaders.url_selenium import (
         SeleniumURLLoader,
     )
     from langchain_community.document_loaders.vsdx import (
         VsdxLoader,
     )
     from langchain_community.document_loaders.weather import (
         WeatherDataLoader,
     )
     from langchain_community.document_loaders.web_base import (
         WebBaseLoader,
     )
     from langchain_community.document_loaders.whatsapp_chat import (
         WhatsAppChatLoader,
     )
     from langchain_community.document_loaders.wikipedia import (
         WikipediaLoader,
     )
     from langchain_community.document_loaders.word_document import (
         Docx2txtLoader,
         UnstructuredWordDocumentLoader,
     )
     from langchain_community.document_loaders.xml import (
         UnstructuredXMLLoader,
     )
     from langchain_community.document_loaders.xorbits import (
         XorbitsLoader,
     )
     from langchain_community.document_loaders.youtube import (
         GoogleApiClient,
         GoogleApiYoutubeLoader,
         YoutubeLoader,
     )
     from langchain_community.document_loaders.yuque import (
         YuqueLoader,
     )
+    from langchain_community.document_loaders.steel import (
+        SteelLoader,
+    )
 
 
 _module_lookup = {
     "AZLyricsLoader": "langchain_community.document_loaders.azlyrics",
@@ -730,6 +733,7 @@
     "YoutubeAudioLoader": "langchain_community.document_loaders.blob_loaders",
     "YoutubeLoader": "langchain_community.document_loaders.youtube",
     "YuqueLoader": "langchain_community.document_loaders.yuque",
+    "SteelLoader": "langchain_community.document_loaders.steel",
 }
 
 
@@ -937,4 +941,5 @@
     "YoutubeAudioLoader",
     "YoutubeLoader",
     "YuqueLoader",
+    "SteelLoader",
 ]

libs/community/langchain_community/document_loaders/steel.py

@@ -0,0 +1,70 @@
+import logging
+from typing import Iterator, List, Optional, Union
+
+import requests
+from langchain_core.documents import Document
+
+from langchain_community.document_loaders.base import BaseLoader
+
+logger = logging.getLogger(__name__)
+
+class SteelLoader(BaseLoader):
+    """Load webpages with Steel."""
+
+    def __init__(
+        self, api_token: str, urls: Union[str, List[str]], text_content: bool = True
+    ):
+        """Initialize with API token and the URLs to scrape"""
+        self.api_token = api_token
+        """Steel API token."""
+        self.urls = urls
+        """List of URLs to scrape."""
+        self.text_content = text_content
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Lazy load Documents from URLs."""
+
+        for url in self.urls:
+            try:
+                if self.text_content:
+                    response = requests.post(
+                        "https://api.steel.dev/scrape",
+                        headers={
+                            "Authorization": f"Bearer {self.api_token}",
+                        },
+                        json={
+                            "url": url,
+                            "elements": [
+                                {
+                                    "selector": "body",
+                                }
+                            ],
+                        },
+                    )
+                    response.raise_for_status()
+                    yield Document(
+                        page_content=response.json()["data"][0]["results"][0]["text"],
+                        metadata={
+                            "source": url,
+                        },
+                    )
+                else:
+                    response = requests.post(
+                        "https://api.steel.dev/content",
+                        headers={
+                            "Authorization": f"Bearer {self.api_token}",
+                        },
+                        json={
+                            "url": url,
+                        },
+                    )
+                    response.raise_for_status()
+                    yield Document(
+                        page_content=response.text,
+                        metadata={
+                            "source": url,
+                        },
+                    )
+            except requests.RequestException as e:
+                logger.error(f"Error fetching {url}: {e}")
+                continue

libs/community/langchain_community/document_loaders/tests/test_steel_loader.py

@@ -0,0 +1,62 @@
+import unittest
+from unittest.mock import patch, Mock
+from requests.exceptions import RequestException
+from langchain_core.documents import Document
+from langchain_community.document_loaders.steel import SteelLoader
+
+class TestSteelLoader(unittest.TestCase):
+
+    @patch('requests.post')
+    def test_lazy_load_success_text_content(self, mock_post):
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "data": [{"results": [{"text": "Sample text content"}]}]
+        }
+        mock_response.raise_for_status = Mock()
+        mock_post.return_value = mock_response
+
+        loader = SteelLoader(api_token="fake_token", urls=["http://example.com"], text_content=True)
+        documents = list(loader.lazy_load())
+
+        self.assertEqual(len(documents), 1)
+        self.assertEqual(documents[0].page_content, "Sample text content")
+        self.assertEqual(documents[0].metadata["source"], "http://example.com")
+
+    @patch('requests.post')
+    def test_lazy_load_success_html_content(self, mock_post):
+        mock_response = Mock()
+        mock_response.text = "<html>Sample HTML content</html>"
+        mock_response.raise_for_status = Mock()
+        mock_post.return_value = mock_response
+
+        loader = SteelLoader(api_token="fake_token", urls=["http://example.com"], text_content=False)
+        documents = list(loader.lazy_load())
+
+        self.assertEqual(len(documents), 1)
+        self.assertEqual(documents[0].page_content, "<html>Sample HTML content</html>")
+        self.assertEqual(documents[0].metadata["source"], "http://example.com")
+
+    @patch('requests.post')
+    def test_lazy_load_invalid_api_token(self, mock_post):
+        mock_response = Mock()
+        mock_response.raise_for_status.side_effect = RequestException("Invalid API token")
+        mock_post.return_value = mock_response
+
+        loader = SteelLoader(api_token="invalid_token", urls=["http://example.com"], text_content=True)
+        documents = list(loader.lazy_load())
+
+        self.assertEqual(len(documents), 0)
+
+    @patch('requests.post')
+    def test_lazy_load_unreachable_url(self, mock_post):
+        mock_response = Mock()
+        mock_response.raise_for_status.side_effect = RequestException("URL unreachable")
+        mock_post.return_value = mock_response
+
+        loader = SteelLoader(api_token="fake_token", urls=["http://unreachable-url.com"], text_content=True)
+        documents = list(loader.lazy_load())
+
+        self.assertEqual(len(documents), 0)
+
+if __name__ == '__main__':
+    unittest.main()