feat: add scrapegraph-py PR #84 SDK compatibility

Support both the v2 Client API (PR #82) and the newer ScrapeGraphAI API
(PR #84) which uses Pydantic request models and ApiResult[T] wrappers.

- Add scrapegraph_py_compat helper with runtime API detection
- Route smart_scraper_graph through the compat layer
- Add v3-style examples for extract, search, and scrape

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: VinciGit00

examples/markdownify/markdownify_scrapegraphai_v3.py

@@ -0,0 +1,24 @@
+"""
+Scrape a webpage as markdown using the scrapegraph-py v3 API (PR #84).
+Uses ScrapeGraphAI client + ScrapeRequest model + ApiResult wrapper.
+"""
+
+import json
+import os
+
+from dotenv import load_dotenv
+from scrapegraph_py import ScrapeGraphAI, ScrapeRequest
+
+load_dotenv()
+
+api_key = os.getenv("SGAI_API_KEY") or os.getenv("SCRAPEGRAPH_API_KEY")
+if not api_key:
+    raise ValueError("SGAI_API_KEY not found in environment variables")
+
+with ScrapeGraphAI(api_key=api_key) as sgai:
+    result = sgai.scrape(ScrapeRequest(url="https://example.com"))
+
+    if result.status == "success":
+        print(json.dumps(result.data.model_dump(by_alias=True), indent=2, default=str))
+    else:
+        raise RuntimeError(result.error)

examples/search_graph/scrapegraphai/searchscraper_scrapegraphai_v3.py

@@ -0,0 +1,24 @@
+"""
+Search the web using the scrapegraph-py v3 API (PR #84).
+Uses ScrapeGraphAI client + SearchRequest model + ApiResult wrapper.
+"""
+
+import json
+import os
+
+from dotenv import load_dotenv
+from scrapegraph_py import ScrapeGraphAI, SearchRequest
+
+load_dotenv()
+
+api_key = os.getenv("SGAI_API_KEY") or os.getenv("SCRAPEGRAPH_API_KEY")
+if not api_key:
+    raise ValueError("SGAI_API_KEY not found in environment variables")
+
+with ScrapeGraphAI(api_key=api_key) as sgai:
+    result = sgai.search(SearchRequest(query="Extract webpage information"))
+
+    if result.status == "success":
+        print(json.dumps(result.data.model_dump(by_alias=True), indent=2, default=str))
+    else:
+        raise RuntimeError(result.error)

examples/smart_scraper_graph/scrapegraphai/smartscraper_scrapegraphai_v3.py

@@ -0,0 +1,29 @@
+"""
+Extract structured data using the scrapegraph-py v3 API (PR #84).
+Uses ScrapeGraphAI client + ExtractRequest model + ApiResult wrapper.
+"""
+
+import json
+import os
+
+from dotenv import load_dotenv
+from scrapegraph_py import ExtractRequest, ScrapeGraphAI
+
+load_dotenv()
+
+api_key = os.getenv("SGAI_API_KEY") or os.getenv("SCRAPEGRAPH_API_KEY")
+if not api_key:
+    raise ValueError("SGAI_API_KEY not found in environment variables")
+
+with ScrapeGraphAI(api_key=api_key) as sgai:
+    result = sgai.extract(
+        ExtractRequest(
+            url="https://scrapegraphai.com",
+            prompt="Extract the founders' informations",
+        )
+    )
+
+    if result.status == "success":
+        print(json.dumps(result.data.model_dump(by_alias=True), indent=2, default=str))
+    else:
+        raise RuntimeError(result.error)

scrapegraphai/graphs/smart_scraper_graph.py

@@ -77,21 +77,16 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
         if self.llm_model == "scrapegraphai/smart-scraper":
-            try:
-                from scrapegraph_py import Client
-            except ImportError:
-                raise ImportError(
-                    "scrapegraph_py is not installed. Please install it using 'pip install scrapegraph-py'."
-                )
+            from ..integrations.scrapegraph_py_compat import extract as sgai_extract
 
-            with Client(api_key=self.config.get("api_key")) as sgai_client:
-                # Extract request (v2 API)
-                response = sgai_client.extract(
-                    url=self.source,
-                    prompt=self.prompt,
-                    output_schema=self.schema,
-                )
+            response = sgai_extract(
+                api_key=self.config.get("api_key"),
+                url=self.source,
+                prompt=self.prompt,
+                schema=self.schema,
+            )
 
+            if isinstance(response, dict):
                 if "id" in response:
                     logger.info(f"Request ID: {response['id']}")
                 if "data" in response:

scrapegraphai/integrations/scrapegraph_py_compat.py

@@ -0,0 +1,109 @@
+"""
+Compatibility layer for scrapegraph-py SDK.
+
+Supports both the v2 `Client` API (PR #82) and the newer `ScrapeGraphAI`
+API (PR #84) which uses Pydantic request models and an ApiResult wrapper.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional, Type
+
+from pydantic import BaseModel
+
+
+def _detect_api() -> str:
+    try:
+        from scrapegraph_py import ScrapeGraphAI  # noqa: F401
+
+        return "v3"
+    except ImportError:
+        pass
+    try:
+        from scrapegraph_py import Client  # noqa: F401
+
+        return "v2"
+    except ImportError as e:
+        raise ImportError(
+            "scrapegraph_py is not installed. Install it with 'pip install scrapegraph-py'."
+        ) from e
+
+
+def _schema_to_dict(schema: Optional[Type[BaseModel]]) -> Optional[dict]:
+    if schema is None:
+        return None
+    if isinstance(schema, dict):
+        return schema
+    if isinstance(schema, type) and issubclass(schema, BaseModel):
+        return schema.model_json_schema()
+    return None
+
+
+def _unwrap_result(result: Any) -> dict:
+    if hasattr(result, "status") and hasattr(result, "data"):
+        if result.status != "success":
+            raise RuntimeError(
+                getattr(result, "error", "scrapegraph-py request failed")
+            )
+        data = result.data
+        if hasattr(data, "model_dump"):
+            return data.model_dump(by_alias=True, exclude_none=True)
+        return data if isinstance(data, dict) else {"data": data}
+    return result
+
+
+def extract(
+    api_key: Optional[str],
+    url: str,
+    prompt: str,
+    schema: Optional[Type[BaseModel]] = None,
+) -> dict:
+    """Call the scrapegraph-py extract endpoint across SDK versions."""
+    api = _detect_api()
+
+    if api == "v3":
+        from scrapegraph_py import ExtractRequest, ScrapeGraphAI
+
+        kwargs: dict[str, Any] = {"url": url, "prompt": prompt}
+        schema_dict = _schema_to_dict(schema)
+        if schema_dict is not None:
+            kwargs["schema_"] = schema_dict
+        with ScrapeGraphAI(api_key=api_key) as client:
+            return _unwrap_result(client.extract(ExtractRequest(**kwargs)))
+
+    from scrapegraph_py import Client
+
+    with Client(api_key=api_key) as client:
+        return client.extract(url=url, prompt=prompt, output_schema=schema)
+
+
+def scrape(api_key: Optional[str], url: str) -> dict:
+    """Call the scrapegraph-py scrape endpoint across SDK versions."""
+    api = _detect_api()
+
+    if api == "v3":
+        from scrapegraph_py import ScrapeGraphAI, ScrapeRequest
+
+        with ScrapeGraphAI(api_key=api_key) as client:
+            return _unwrap_result(client.scrape(ScrapeRequest(url=url)))
+
+    from scrapegraph_py import Client
+
+    with Client(api_key=api_key) as client:
+        return client.scrape(url=url)
+
+
+def search(api_key: Optional[str], query: str) -> dict:
+    """Call the scrapegraph-py search endpoint across SDK versions."""
+    api = _detect_api()
+
+    if api == "v3":
+        from scrapegraph_py import ScrapeGraphAI, SearchRequest
+
+        with ScrapeGraphAI(api_key=api_key) as client:
+            return _unwrap_result(client.search(SearchRequest(query=query)))
+
+    from scrapegraph_py import Client
+
+    with Client(api_key=api_key) as client:
+        return client.search(query=query)