Update outgoing_links_by_rank -2024.sql (#3891)

Author: henryp25

sql/2024/seo/outgoing_links_by_rank -2024.sql

@@ -1,76 +1,70 @@
 #standardSQL
 # Internal and external link metrics by quantile and rank
-
-CREATE TEMPORARY FUNCTION getOutgoingLinkMetrics(payload STRING)
-RETURNS STRUCT<
-  same_site INT64,
-  same_property INT64,
-  other_property INT64
-> LANGUAGE js AS '''
-var result = {same_site: 0,
-              same_property: 0,
-              other_property: 0};
-
-try {
-    var $ = JSON.parse(payload);
-    var wpt_bodies  = JSON.parse($._wpt_bodies);
-
-    if (!wpt_bodies){
-        return result;
-    }
-
-    var anchors = wpt_bodies.anchors;
-
-    if (anchors){
-      result.same_site = anchors.rendered.same_site;
-      result.same_property = anchors.rendered.same_property;
-      result.other_property = anchors.rendered.other_property;
-    }
-
-} catch (e) {}
-
-return result;
-''';
-
 WITH page_metrics AS (
   SELECT
     client,
     page,
-    getOutgoingLinkMetrics(payload) AS outgoing_link_metrics,
-    JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(payload, '$._wpt_bodies'), '$.is_root_page') AS is_root_page
-  FROM
-    `httparchive.all.pages`
+    is_root_page,
+    IF(rank <= rank_bucket, rank_bucket, NULL) AS rank,
+    ANY_VALUE(custom_metrics.wpt_bodies.anchors) AS anchors
+  FROM httparchive.crawl.pages,
+    UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_bucket
   WHERE
-    DATE = '2024-06-01'
+    date = '2024-06-01'
+  GROUP BY
+    client,
+    page,
+    is_root_page,
+    rank
+  HAVING rank IS NOT NULL
+), metric_details AS (
+  SELECT
+    client,
+    is_root_page,
+    percentile,
+    rank,
+    APPROX_QUANTILES(INT64(anchors.rendered.same_site), 1000)[OFFSET(percentile * 10)] AS outgoing_links_same_site,
+    APPROX_QUANTILES(INT64(anchors.rendered.same_property), 1000)[OFFSET(percentile * 10)] AS outgoing_links_same_property,
+    APPROX_QUANTILES(INT64(anchors.rendered.other_property), 1000)[OFFSET(percentile * 10)] AS outgoing_links_other_property
+  FROM page_metrics,
+    UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+  GROUP BY
+    client,
+    is_root_page,
+    rank,
+    percentile
+  ORDER BY
+    client,
+    is_root_page,
+    rank,
+    percentile
+), page_counts AS (
+  SELECT
+    client,
+    is_root_page,
+    rank,
+    COUNT(DISTINCT page) AS total_pages
+  FROM page_metrics
+  GROUP BY
+    client,
+    is_root_page,
+    rank
 )
 
 SELECT
   client,
-  CASE
-    WHEN is_root_page = 'false' THEN 'Secondary Page'
-    ELSE 'Homepage'
-  END AS page_type,
+  is_root_page,
+  rank,
+  total_pages,
   percentile,
-  rank_grouping,
-  CASE
-    WHEN rank_grouping = 100000000 THEN 'all'
-    ELSE FORMAT("%'d", rank_grouping)
-  END AS ranking,
-  COUNT(DISTINCT page) AS pages,
-  APPROX_QUANTILES(outgoing_link_metrics.same_site, 1000)[OFFSET(percentile * 10)] AS outgoing_links_same_site,
-  APPROX_QUANTILES(outgoing_link_metrics.same_property, 1000)[OFFSET(percentile * 10)] AS outgoing_links_same_property,
-  APPROX_QUANTILES(outgoing_link_metrics.other_property, 1000)[OFFSET(percentile * 10)] AS outgoing_links_other_property
-FROM
-  page_metrics,
-  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile,
-  UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping
-GROUP BY
-  client,
-  page_type,
-  rank_grouping,
-  percentile
+  outgoing_links_same_site,
+  outgoing_links_same_property,
+  outgoing_links_other_property
+FROM metric_details
+LEFT JOIN page_counts
+USING (client, is_root_page, rank)
 ORDER BY
   client,
-  page_type,
-  rank_grouping,
+  is_root_page,
+  rank,
   percentile