feat(scrape): add reddit-post prebuilt parser with full comment trees (#2217)

Stainless-Generated-From: 65b502140edcfd051f63e617a8cc79b61615491e

Author: stlc-bot

lib/context_dev/models/batch_get_results_response.rb

@@ -1075,6 +1075,10 @@ module Type
                 TIKTOK_FOLLOWERS = :"tiktok-followers"
                 TIKTOK_FOLLOWING = :"tiktok-following"
                 TIKTOK_USER_SEARCH = :"tiktok-user-search"
+                REDDIT_POST = :"reddit-post"
+                REDDIT_SEARCH = :"reddit-search"
+                REDDIT_SUBREDDIT = :"reddit-subreddit"
+                REDDIT_USER = :"reddit-user"
                 YELP_SEARCH = :"yelp-search"
                 YELP_REVIEWS = :"yelp-reviews"
                 EMAILS = :emails
@@ -2487,6 +2491,10 @@ module Type
                 TIKTOK_FOLLOWERS = :"tiktok-followers"
                 TIKTOK_FOLLOWING = :"tiktok-following"
                 TIKTOK_USER_SEARCH = :"tiktok-user-search"
+                REDDIT_POST = :"reddit-post"
+                REDDIT_SEARCH = :"reddit-search"
+                REDDIT_SUBREDDIT = :"reddit-subreddit"
+                REDDIT_USER = :"reddit-user"
                 YELP_SEARCH = :"yelp-search"
                 YELP_REVIEWS = :"yelp-reviews"
                 EMAILS = :emails

lib/context_dev/models/batch_submit_params.rb

@@ -576,13 +576,62 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
           #   (/search/user?q={query}) as data.users: id, username, nickname, biography,
           #   avatar_url, sec_uid, follower_count, like_count, and is_verified. The TikTok
           #   parsers accept tiktok.com, www.tiktok.com, and m.tiktok.com URLs but not short
-          #   links (vm.tiktok.com, vt.tiktok.com, tiktok.com/t/…). yelp-search accepts Yelp
-          #   search URLs with find_desc and find_loc, reads start as the result offset, and
-          #   returns up to 20 ranked organic businesses in data.businesses with total and
-          #   pagination. yelp-reviews accepts a Yelp business URL (yelp.com/biz/{alias}),
-          #   uses optional start for the offset, and always sorts newest first; unsupported
-          #   sort and review filters are rejected, while tracking params are ignored. It
-          #   returns up to 20 latest public reviews in data.reviews with business details and
+          #   links (vm.tiktok.com, vt.tiktok.com, tiktok.com/t/…). reddit-post parses a post
+          #   (reddit.com/r/{subreddit}/comments/{id}, reddit.com/comments/{id}, or
+          #   redd.it/{id}) into data.post: title, body as Markdown, type (text, link, image,
+          #   gallery, or video), link_url, subreddit, author, flairs, score, upvote_ratio,
+          #   comment and crosspost counts, NSFW, spoiler, locked, stickied, and archived
+          #   flags, images (every gallery image in order), and video (an MP4 url plus an
+          #   hls_url that carries the audio). data.comments holds the comment tree, each
+          #   comment with id, author, body as Markdown, score, posted_at, depth, parent_id,
+          #   url, is_submitter, is_deleted, and its replies; limit caps the total number of
+          #   comments (default 100), filled shallowest first so whole conversations are kept,
+          #   and data.has_more_comments says whether the thread has more. The sort query of
+          #   the URL (best, top, new, controversial, old, qa) orders the comments.
+          #   reddit-search returns search results for a search URL
+          #   (reddit.com/search?q={query}, or reddit.com/r/{subreddit}/search?q={query} to
+          #   search one subreddit). With type=link or no type it returns posts in the
+          #   reddit-post shape as data.posts, ordered by the sort (relevance, hot, top, new,
+          #   comments) and limited to the t time range (hour, day, week, month, year, all) of
+          #   the URL; type=sr returns communities as data.communities (name, title,
+          #   description, sidebar, subscriber_count, created_at, type, NSFW and quarantine
+          #   flags, language, category, icon and banner URLs, and allowed post types);
+          #   type=user returns accounts as data.users (username, display_name, bio,
+          #   created_at, post and comment karma, avatar and banner URLs, and verified,
+          #   employee, moderator, premium, and NSFW flags). limit caps the number of results
+          #   (default 100), in the order Reddit returns them, and data.has_more says whether
+          #   Reddit has more. reddit-subreddit reads a subreddit URL
+          #   (reddit.com/r/{subreddit}, optionally followed by /hot, /new, /top, /rising, or
+          #   /controversial, with t setting the time range of top and controversial, default
+          #   day) into data.subreddit in the community shape of reddit-search, data.rules
+          #   (name, description, applies_to, violation_reason, created_at), and data.posts in
+          #   the reddit-post shape, ordered like the tab (default hot) and with stickied
+          #   posts flagged; r/all and r/popular return posts with a null subreddit and rules.
+          #   limit caps the posts (default 100) and data.has_more says whether Reddit has
+          #   more. Private and quarantined subreddits return WEBSITE_ACCESS_ERROR.
+          #   reddit-user reads a profile URL (reddit.com/user/{username} or
+          #   reddit.com/u/{username}, optionally followed by /overview, /submitted, or
+          #   /comments, with sort set to new, hot, top, or controversial and t setting the
+          #   time range of top and controversial, default all) into data.user (id, username,
+          #   display_name, bio, created_at, total, post, comment, awardee, and awarder karma,
+          #   avatar and banner URLs, and verified, suspended, employee, moderator, premium,
+          #   and NSFW flags), data.trophies (name, description, icon_url, granted_at, url),
+          #   data.moderated_communities (name, title, url, subscriber_count, icon_url,
+          #   is_nsfw), data.posts in the reddit-post shape, and data.comments, each with id,
+          #   author, body as Markdown, score, posted_at, edited_at, url, subreddit, parent_id
+          #   (null for a top-level comment), is_submitter, is_stickied, distinguished,
+          #   award_count, and the post it was left on (id, title, url, author, is_nsfw). The
+          #   overview tab returns both posts and comments; submitted returns null comments
+          #   and comments returns null posts. limit caps the posts and comments together
+          #   (default 100) and data.has_more says whether Reddit has more. Users who hide
+          #   their activity return empty lists, and suspended accounts return their profile
+          #   with null posts and comments. yelp-search accepts Yelp search URLs with
+          #   find_desc and find_loc, reads start as the result offset, and returns up to 20
+          #   ranked organic businesses in data.businesses with total and pagination.
+          #   yelp-reviews accepts a Yelp business URL (yelp.com/biz/{alias}), uses optional
+          #   start for the offset, and always sorts newest first; unsupported sort and review
+          #   filters are rejected, while tracking params are ignored. It returns up to 20
+          #   latest public reviews in data.reviews with business details and
           #   pagination.next_offset. Each review has rating, full text, updated_at, and
           #   author; owner replies and photos are not included. emails accepts any page and
           #   returns its email addresses as data.emails ([{ email }]), read from mailto
@@ -606,8 +655,10 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
           required :type, const: :prebuilt
 
           # @!attribute limit
-          #   google-maps-reviews only. Maximum number of reviews to return. Defaults to 100;
-          #   maximum 1000.
+          #   google-maps-reviews and reddit parsers only. Maximum number of reviews
+          #   (google-maps-reviews, default 100), comments (reddit-post, default 100), or
+          #   results (reddit-search, reddit-subreddit, and reddit-user, default 100) to
+          #   return; maximum 1000.
           #
           #   @return [Integer, nil]
           optional :limit, Integer
@@ -627,7 +678,7 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
           #
           #   @param parser [Symbol, ContextDev::Models::WebScrapeParams::ParseParams::PrebuiltParser::Parser] google-maps parses Google Maps business data into data.results. Accepts Maps pla
           #
-          #   @param limit [Integer] google-maps-reviews only. Maximum number of reviews to return. Defaults to 100;
+          #   @param limit [Integer] google-maps-reviews and reddit parsers only. Maximum number of reviews (google-m
           #
           #   @param sort [Symbol, ContextDev::Models::WebScrapeParams::ParseParams::PrebuiltParser::Sort] google-maps-reviews only. Order of data.reviews. newest, relevant, highest, and
           #
@@ -691,13 +742,62 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
           # (/search/user?q={query}) as data.users: id, username, nickname, biography,
           # avatar_url, sec_uid, follower_count, like_count, and is_verified. The TikTok
           # parsers accept tiktok.com, www.tiktok.com, and m.tiktok.com URLs but not short
-          # links (vm.tiktok.com, vt.tiktok.com, tiktok.com/t/…). yelp-search accepts Yelp
-          # search URLs with find_desc and find_loc, reads start as the result offset, and
-          # returns up to 20 ranked organic businesses in data.businesses with total and
-          # pagination. yelp-reviews accepts a Yelp business URL (yelp.com/biz/{alias}),
-          # uses optional start for the offset, and always sorts newest first; unsupported
-          # sort and review filters are rejected, while tracking params are ignored. It
-          # returns up to 20 latest public reviews in data.reviews with business details and
+          # links (vm.tiktok.com, vt.tiktok.com, tiktok.com/t/…). reddit-post parses a post
+          # (reddit.com/r/{subreddit}/comments/{id}, reddit.com/comments/{id}, or
+          # redd.it/{id}) into data.post: title, body as Markdown, type (text, link, image,
+          # gallery, or video), link_url, subreddit, author, flairs, score, upvote_ratio,
+          # comment and crosspost counts, NSFW, spoiler, locked, stickied, and archived
+          # flags, images (every gallery image in order), and video (an MP4 url plus an
+          # hls_url that carries the audio). data.comments holds the comment tree, each
+          # comment with id, author, body as Markdown, score, posted_at, depth, parent_id,
+          # url, is_submitter, is_deleted, and its replies; limit caps the total number of
+          # comments (default 100), filled shallowest first so whole conversations are kept,
+          # and data.has_more_comments says whether the thread has more. The sort query of
+          # the URL (best, top, new, controversial, old, qa) orders the comments.
+          # reddit-search returns search results for a search URL
+          # (reddit.com/search?q={query}, or reddit.com/r/{subreddit}/search?q={query} to
+          # search one subreddit). With type=link or no type it returns posts in the
+          # reddit-post shape as data.posts, ordered by the sort (relevance, hot, top, new,
+          # comments) and limited to the t time range (hour, day, week, month, year, all) of
+          # the URL; type=sr returns communities as data.communities (name, title,
+          # description, sidebar, subscriber_count, created_at, type, NSFW and quarantine
+          # flags, language, category, icon and banner URLs, and allowed post types);
+          # type=user returns accounts as data.users (username, display_name, bio,
+          # created_at, post and comment karma, avatar and banner URLs, and verified,
+          # employee, moderator, premium, and NSFW flags). limit caps the number of results
+          # (default 100), in the order Reddit returns them, and data.has_more says whether
+          # Reddit has more. reddit-subreddit reads a subreddit URL
+          # (reddit.com/r/{subreddit}, optionally followed by /hot, /new, /top, /rising, or
+          # /controversial, with t setting the time range of top and controversial, default
+          # day) into data.subreddit in the community shape of reddit-search, data.rules
+          # (name, description, applies_to, violation_reason, created_at), and data.posts in
+          # the reddit-post shape, ordered like the tab (default hot) and with stickied
+          # posts flagged; r/all and r/popular return posts with a null subreddit and rules.
+          # limit caps the posts (default 100) and data.has_more says whether Reddit has
+          # more. Private and quarantined subreddits return WEBSITE_ACCESS_ERROR.
+          # reddit-user reads a profile URL (reddit.com/user/{username} or
+          # reddit.com/u/{username}, optionally followed by /overview, /submitted, or
+          # /comments, with sort set to new, hot, top, or controversial and t setting the
+          # time range of top and controversial, default all) into data.user (id, username,
+          # display_name, bio, created_at, total, post, comment, awardee, and awarder karma,
+          # avatar and banner URLs, and verified, suspended, employee, moderator, premium,
+          # and NSFW flags), data.trophies (name, description, icon_url, granted_at, url),
+          # data.moderated_communities (name, title, url, subscriber_count, icon_url,
+          # is_nsfw), data.posts in the reddit-post shape, and data.comments, each with id,
+          # author, body as Markdown, score, posted_at, edited_at, url, subreddit, parent_id
+          # (null for a top-level comment), is_submitter, is_stickied, distinguished,
+          # award_count, and the post it was left on (id, title, url, author, is_nsfw). The
+          # overview tab returns both posts and comments; submitted returns null comments
+          # and comments returns null posts. limit caps the posts and comments together
+          # (default 100) and data.has_more says whether Reddit has more. Users who hide
+          # their activity return empty lists, and suspended accounts return their profile
+          # with null posts and comments. yelp-search accepts Yelp search URLs with
+          # find_desc and find_loc, reads start as the result offset, and returns up to 20
+          # ranked organic businesses in data.businesses with total and pagination.
+          # yelp-reviews accepts a Yelp business URL (yelp.com/biz/{alias}), uses optional
+          # start for the offset, and always sorts newest first; unsupported sort and review
+          # filters are rejected, while tracking params are ignored. It returns up to 20
+          # latest public reviews in data.reviews with business details and
           # pagination.next_offset. Each review has rating, full text, updated_at, and
           # author; owner replies and photos are not included. emails accepts any page and
           # returns its email addresses as data.emails ([{ email }]), read from mailto
@@ -732,6 +832,10 @@ module Parser
             TIKTOK_FOLLOWERS = :"tiktok-followers"
             TIKTOK_FOLLOWING = :"tiktok-following"
             TIKTOK_USER_SEARCH = :"tiktok-user-search"
+            REDDIT_POST = :"reddit-post"
+            REDDIT_SEARCH = :"reddit-search"
+            REDDIT_SUBREDDIT = :"reddit-subreddit"
+            REDDIT_USER = :"reddit-user"
             YELP_SEARCH = :"yelp-search"
             YELP_REVIEWS = :"yelp-reviews"
             EMAILS = :emails

lib/context_dev/models/web_scrape_params.rb

@@ -928,6 +928,10 @@ module Type
           TIKTOK_FOLLOWERS = :"tiktok-followers"
           TIKTOK_FOLLOWING = :"tiktok-following"
           TIKTOK_USER_SEARCH = :"tiktok-user-search"
+          REDDIT_POST = :"reddit-post"
+          REDDIT_SEARCH = :"reddit-search"
+          REDDIT_SUBREDDIT = :"reddit-subreddit"
+          REDDIT_USER = :"reddit-user"
           YELP_SEARCH = :"yelp-search"
           YELP_REVIEWS = :"yelp-reviews"
           EMAILS = :emails

lib/context_dev/models/web_scrape_response.rb

@@ -2163,6 +2163,26 @@ module ContextDev
                     :"tiktok-user-search",
                     ContextDev::Models::BatchGetResultsResponse::Data::Ok::Result::Parsed::Type::TaggedSymbol
                   )
+                REDDIT_POST =
+                  T.let(
+                    :"reddit-post",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_SEARCH =
+                  T.let(
+                    :"reddit-search",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_SUBREDDIT =
+                  T.let(
+                    :"reddit-subreddit",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_USER =
+                  T.let(
+                    :"reddit-user",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::Result::Parsed::Type::TaggedSymbol
+                  )
                 YELP_SEARCH =
                   T.let(
                     :"yelp-search",
@@ -4889,6 +4909,26 @@ module ContextDev
                     :"tiktok-user-search",
                     ContextDev::Models::BatchGetResultsResponse::Data::Error::Result::Parsed::Type::TaggedSymbol
                   )
+                REDDIT_POST =
+                  T.let(
+                    :"reddit-post",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Error::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_SEARCH =
+                  T.let(
+                    :"reddit-search",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Error::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_SUBREDDIT =
+                  T.let(
+                    :"reddit-subreddit",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Error::Result::Parsed::Type::TaggedSymbol
+                  )
+                REDDIT_USER =
+                  T.let(
+                    :"reddit-user",
+                    ContextDev::Models::BatchGetResultsResponse::Data::Error::Result::Parsed::Type::TaggedSymbol
+                  )
                 YELP_SEARCH =
                   T.let(
                     :"yelp-search",