Skip to content

Commit 1e9fdc8

Browse files
author
stlc-bot
committed
feat(scrape): add sourcePreference to choose between HTML conversion and published Markdown (#2010)
Stainless-Generated-From: e003576b1e3d7e233e15f53d6eba49195e6f93d2
1 parent 13a00ce commit 1e9fdc8

7 files changed

Lines changed: 146 additions & 20 deletions

File tree

‎src/context/dev/resources/web.py‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,7 @@ def web_crawl_md(
14291429
pdf: web_web_crawl_md_params.Pdf | Omit = omit,
14301430
settle_animations: bool | Omit = omit,
14311431
shorten_base64_images: bool | Omit = omit,
1432+
source_preference: List[Literal["html-conversion", "markdown-direct"]] | Omit = omit,
14321433
stop_after_ms: int | Omit = omit,
14331434
tags: SequenceNotStr[str] | Omit = omit,
14341435
timeout_opts: web_web_crawl_md_params.TimeoutOpts | Omit = omit,
@@ -1465,9 +1466,11 @@ def web_crawl_md(
14651466
include_frames: When true, the contents of iframes are rendered to Markdown for each crawled
14661467
page.
14671468
1468-
include_images: Include image references in the Markdown output
1469+
include_images: Include image references in the Markdown output Only applies to
1470+
`html-conversion` extractions.
14691471
1470-
include_links: Preserve hyperlinks in the Markdown output
1472+
include_links: Preserve hyperlinks in the Markdown output Only applies to `html-conversion`
1473+
extractions.
14711474
14721475
include_selectors: Keep matching HTML subtrees before converting each page to Markdown.
14731476
@@ -1482,7 +1485,18 @@ def web_crawl_md(
14821485
settle_animations: Wait briefly for CSS animations and transitions to settle before reading each
14831486
page.
14841487
1485-
shorten_base64_images: Truncate base64-encoded image data in the Markdown output
1488+
shorten_base64_images: Truncate base64-encoded image data in the Markdown output Only applies to
1489+
`html-conversion` extractions.
1490+
1491+
source_preference: Where the Markdown should come from, in order of preference; the first usable
1492+
source wins. `html-conversion` converts the page HTML, and counts as unusable
1493+
when the conversion has almost no content (under 200 characters).
1494+
`markdown-direct` uses the official Markdown version the page publishes through,
1495+
returned as the site serves it. Defaults to
1496+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
1497+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
1498+
Requests with include or exclude selectors never use `markdown-direct`, so they
1499+
must include `html-conversion`.
14861500
14871501
stop_after_ms: Soft crawl deadline in milliseconds. Returns pages collected before the next
14881502
deadline check.
@@ -1531,6 +1545,7 @@ def web_crawl_md(
15311545
"pdf": pdf,
15321546
"settle_animations": settle_animations,
15331547
"shorten_base64_images": shorten_base64_images,
1548+
"source_preference": source_preference,
15341549
"stop_after_ms": stop_after_ms,
15351550
"tags": tags,
15361551
"timeout_opts": timeout_opts,
@@ -2937,6 +2952,7 @@ async def web_crawl_md(
29372952
pdf: web_web_crawl_md_params.Pdf | Omit = omit,
29382953
settle_animations: bool | Omit = omit,
29392954
shorten_base64_images: bool | Omit = omit,
2955+
source_preference: List[Literal["html-conversion", "markdown-direct"]] | Omit = omit,
29402956
stop_after_ms: int | Omit = omit,
29412957
tags: SequenceNotStr[str] | Omit = omit,
29422958
timeout_opts: web_web_crawl_md_params.TimeoutOpts | Omit = omit,
@@ -2973,9 +2989,11 @@ async def web_crawl_md(
29732989
include_frames: When true, the contents of iframes are rendered to Markdown for each crawled
29742990
page.
29752991
2976-
include_images: Include image references in the Markdown output
2992+
include_images: Include image references in the Markdown output Only applies to
2993+
`html-conversion` extractions.
29772994
2978-
include_links: Preserve hyperlinks in the Markdown output
2995+
include_links: Preserve hyperlinks in the Markdown output Only applies to `html-conversion`
2996+
extractions.
29792997
29802998
include_selectors: Keep matching HTML subtrees before converting each page to Markdown.
29812999
@@ -2990,7 +3008,18 @@ async def web_crawl_md(
29903008
settle_animations: Wait briefly for CSS animations and transitions to settle before reading each
29913009
page.
29923010
2993-
shorten_base64_images: Truncate base64-encoded image data in the Markdown output
3011+
shorten_base64_images: Truncate base64-encoded image data in the Markdown output Only applies to
3012+
`html-conversion` extractions.
3013+
3014+
source_preference: Where the Markdown should come from, in order of preference; the first usable
3015+
source wins. `html-conversion` converts the page HTML, and counts as unusable
3016+
when the conversion has almost no content (under 200 characters).
3017+
`markdown-direct` uses the official Markdown version the page publishes through,
3018+
returned as the site serves it. Defaults to
3019+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
3020+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
3021+
Requests with include or exclude selectors never use `markdown-direct`, so they
3022+
must include `html-conversion`.
29943023
29953024
stop_after_ms: Soft crawl deadline in milliseconds. Returns pages collected before the next
29963025
deadline check.
@@ -3039,6 +3068,7 @@ async def web_crawl_md(
30393068
"pdf": pdf,
30403069
"settle_animations": settle_animations,
30413070
"shorten_base64_images": shorten_base64_images,
3071+
"source_preference": source_preference,
30423072
"stop_after_ms": stop_after_ms,
30433073
"tags": tags,
30443074
"timeout_opts": timeout_opts,

‎src/context/dev/types/batch_submit_params.py‎

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,15 +193,37 @@ class InputScrapeDataMarkdownParams(TypedDict, total=False):
193193
"""Markdown options. Requires `formats.markdown`."""
194194

195195
include_images: Annotated[bool, PropertyInfo(alias="includeImages")]
196-
"""Include images in the Markdown using image syntax with URLs and alt text."""
196+
"""Include images in the Markdown using image syntax with URLs and alt text.
197+
198+
Only applies to `html-conversion` extractions.
199+
"""
197200

198201
include_links: Annotated[bool, PropertyInfo(alias="includeLinks")]
199-
"""Keep link URLs in the Markdown. Set false to return link text without URLs."""
202+
"""Keep link URLs in the Markdown.
203+
204+
Set false to return link text without URLs. Only applies to `html-conversion`
205+
extractions.
206+
"""
200207

201208
inline_images: Annotated[Literal["placeholder", "preserve"], PropertyInfo(alias="inlineImages")]
202209
"""How base64 images appear: `placeholder` (default) or `preserve`.
203210
204-
Requires `includeImages`.
211+
Requires `includeImages`. Only applies to `html-conversion` extractions.
212+
"""
213+
214+
source_preference: Annotated[
215+
List[Literal["html-conversion", "markdown-direct"]], PropertyInfo(alias="sourcePreference")
216+
]
217+
"""
218+
Where the Markdown should come from, in order of preference; the first usable
219+
source wins. `html-conversion` converts the page HTML, and counts as unusable
220+
when the conversion has almost no content (under 200 characters).
221+
`markdown-direct` uses the official Markdown version the page publishes through,
222+
returned as the site serves it. Defaults to
223+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
224+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
225+
Requests with include or exclude selectors never use `markdown-direct`, so they
226+
must include `html-conversion`.
205227
"""
206228

207229

@@ -904,15 +926,37 @@ class InputCrawlDataMarkdownParams(TypedDict, total=False):
904926
"""Markdown options. Requires `formats.markdown`."""
905927

906928
include_images: Annotated[bool, PropertyInfo(alias="includeImages")]
907-
"""Include images in the Markdown using image syntax with URLs and alt text."""
929+
"""Include images in the Markdown using image syntax with URLs and alt text.
930+
931+
Only applies to `html-conversion` extractions.
932+
"""
908933

909934
include_links: Annotated[bool, PropertyInfo(alias="includeLinks")]
910-
"""Keep link URLs in the Markdown. Set false to return link text without URLs."""
935+
"""Keep link URLs in the Markdown.
936+
937+
Set false to return link text without URLs. Only applies to `html-conversion`
938+
extractions.
939+
"""
911940

912941
inline_images: Annotated[Literal["placeholder", "preserve"], PropertyInfo(alias="inlineImages")]
913942
"""How base64 images appear: `placeholder` (default) or `preserve`.
914943
915-
Requires `includeImages`.
944+
Requires `includeImages`. Only applies to `html-conversion` extractions.
945+
"""
946+
947+
source_preference: Annotated[
948+
List[Literal["html-conversion", "markdown-direct"]], PropertyInfo(alias="sourcePreference")
949+
]
950+
"""
951+
Where the Markdown should come from, in order of preference; the first usable
952+
source wins. `html-conversion` converts the page HTML, and counts as unusable
953+
when the conversion has almost no content (under 200 characters).
954+
`markdown-direct` uses the official Markdown version the page publishes through,
955+
returned as the site serves it. Defaults to
956+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
957+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
958+
Requests with include or exclude selectors never use `markdown-direct`, so they
959+
must include `html-conversion`.
916960
"""
917961

918962

‎src/context/dev/types/web_scrape_params.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,15 +163,37 @@ class MarkdownParams(TypedDict, total=False):
163163
"""Markdown options. Requires `formats.markdown`."""
164164

165165
include_images: Annotated[bool, PropertyInfo(alias="includeImages")]
166-
"""Include images in the Markdown using image syntax with URLs and alt text."""
166+
"""Include images in the Markdown using image syntax with URLs and alt text.
167+
168+
Only applies to `html-conversion` extractions.
169+
"""
167170

168171
include_links: Annotated[bool, PropertyInfo(alias="includeLinks")]
169-
"""Keep link URLs in the Markdown. Set false to return link text without URLs."""
172+
"""Keep link URLs in the Markdown.
173+
174+
Set false to return link text without URLs. Only applies to `html-conversion`
175+
extractions.
176+
"""
170177

171178
inline_images: Annotated[Literal["placeholder", "preserve"], PropertyInfo(alias="inlineImages")]
172179
"""How base64 images appear: `placeholder` (default) or `preserve`.
173180
174-
Requires `includeImages`.
181+
Requires `includeImages`. Only applies to `html-conversion` extractions.
182+
"""
183+
184+
source_preference: Annotated[
185+
List[Literal["html-conversion", "markdown-direct"]], PropertyInfo(alias="sourcePreference")
186+
]
187+
"""
188+
Where the Markdown should come from, in order of preference; the first usable
189+
source wins. `html-conversion` converts the page HTML, and counts as unusable
190+
when the conversion has almost no content (under 200 characters).
191+
`markdown-direct` uses the official Markdown version the page publishes through,
192+
returned as the site serves it. Defaults to
193+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
194+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
195+
Requests with include or exclude selectors never use `markdown-direct`, so they
196+
must include `html-conversion`.
175197
"""
176198

177199

‎src/context/dev/types/web_web_crawl_md_params.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from __future__ import annotations
44

5-
from typing import Optional
5+
from typing import List, Optional
66
from typing_extensions import Literal, Required, Annotated, TypedDict
77

88
from .._types import SequenceNotStr
@@ -247,10 +247,16 @@ class WebWebCrawlMdParams(TypedDict, total=False):
247247
"""
248248

249249
include_images: Annotated[bool, PropertyInfo(alias="includeImages")]
250-
"""Include image references in the Markdown output"""
250+
"""
251+
Include image references in the Markdown output Only applies to
252+
`html-conversion` extractions.
253+
"""
251254

252255
include_links: Annotated[bool, PropertyInfo(alias="includeLinks")]
253-
"""Preserve hyperlinks in the Markdown output"""
256+
"""
257+
Preserve hyperlinks in the Markdown output Only applies to `html-conversion`
258+
extractions.
259+
"""
254260

255261
include_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="includeSelectors")]
256262
"""Keep matching HTML subtrees before converting each page to Markdown."""
@@ -274,7 +280,25 @@ class WebWebCrawlMdParams(TypedDict, total=False):
274280
"""
275281

276282
shorten_base64_images: Annotated[bool, PropertyInfo(alias="shortenBase64Images")]
277-
"""Truncate base64-encoded image data in the Markdown output"""
283+
"""
284+
Truncate base64-encoded image data in the Markdown output Only applies to
285+
`html-conversion` extractions.
286+
"""
287+
288+
source_preference: Annotated[
289+
List[Literal["html-conversion", "markdown-direct"]], PropertyInfo(alias="sourcePreference")
290+
]
291+
"""
292+
Where the Markdown should come from, in order of preference; the first usable
293+
source wins. `html-conversion` converts the page HTML, and counts as unusable
294+
when the conversion has almost no content (under 200 characters).
295+
`markdown-direct` uses the official Markdown version the page publishes through,
296+
returned as the site serves it. Defaults to
297+
`["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
298+
that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
299+
Requests with include or exclude selectors never use `markdown-direct`, so they
300+
must include `html-conversion`.
301+
"""
278302

279303
stop_after_ms: Annotated[int, PropertyInfo(alias="stopAfterMs")]
280304
"""Soft crawl deadline in milliseconds.

‎tests/api_resources/test_batch.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ def test_method_submit_with_all_params(self, client: ContextDev) -> None:
311311
"include_images": True,
312312
"include_links": True,
313313
"inline_images": "placeholder",
314+
"source_preference": ["html-conversion"],
314315
},
315316
"max_age_ms": 0,
316317
"parse_params": {
@@ -719,6 +720,7 @@ async def test_method_submit_with_all_params(self, async_client: AsyncContextDev
719720
"include_images": True,
720721
"include_links": True,
721722
"inline_images": "placeholder",
723+
"source_preference": ["html-conversion"],
722724
},
723725
"max_age_ms": 0,
724726
"parse_params": {

‎tests/api_resources/test_web.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
274274
"include_images": True,
275275
"include_links": True,
276276
"inline_images": "placeholder",
277+
"source_preference": ["html-conversion"],
277278
},
278279
max_age_ms=0,
279280
parse_params={
@@ -531,6 +532,7 @@ def test_method_web_crawl_md_with_all_params(self, client: ContextDev) -> None:
531532
},
532533
settle_animations=True,
533534
shorten_base64_images=True,
535+
source_preference=["html-conversion"],
534536
stop_after_ms=10000,
535537
tags=["production", "team-alpha"],
536538
timeout_opts={
@@ -824,6 +826,7 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
824826
"include_images": True,
825827
"include_links": True,
826828
"inline_images": "placeholder",
829+
"source_preference": ["html-conversion"],
827830
},
828831
max_age_ms=0,
829832
parse_params={
@@ -1081,6 +1084,7 @@ async def test_method_web_crawl_md_with_all_params(self, async_client: AsyncCont
10811084
},
10821085
settle_animations=True,
10831086
shorten_base64_images=True,
1087+
source_preference=["html-conversion"],
10841088
stop_after_ms=10000,
10851089
tags=["production", "team-alpha"],
10861090
timeout_opts={

‎uv.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)