Merge branch 'main' of github.com:HTTPArchive/almanac.httparchive.org into production

Author: tunetheweb

sql/2020/security/iframe_attributes_usage.sql

@@ -2,7 +2,7 @@
 # usage of allow and sandbox attribute of iframe elements, per page and over all iframe elements
 SELECT
   client,
-  COUNT(0) AS total_iframes,
+  COUNT(0) AS total_iframes, # Note: These are not the total number of iframes but only the number of iframes with allow/sandbox + 1 for each website without such iframes
   COUNTIF(allow IS NOT NULL) AS freq_allow,
   COUNTIF(allow IS NOT NULL) / COUNT(0) AS pct_allow_frames,
   COUNTIF(sandbox IS NOT NULL) AS freq_sandbox,

sql/2021/security/iframe_attributes_usage.sql

@@ -2,7 +2,7 @@
 # usage of allow and sandbox attribute of iframe elements, per page and over all iframe elements
 SELECT
   client,
-  COUNT(0) AS total_iframes,
+  COUNT(0) AS total_iframes, # Note: These are not the total number of iframes but only the number of iframes with allow/sandbox + 1 for each website without such iframes
   COUNTIF(allow IS NOT NULL) AS freq_allow,
   COUNTIF(allow IS NOT NULL) / COUNT(0) AS pct_allow_frames,
   COUNTIF(sandbox IS NOT NULL) AS freq_sandbox,

sql/2022/security/iframe_attributes_usage.sql

@@ -2,7 +2,7 @@
 # usage of allow and sandbox attribute of iframe elements, per page and over all iframe elements
 SELECT
   client,
-  COUNT(0) AS total_iframes,
+  COUNT(0) AS total_iframes,  # Note: These are not the total number of iframes but only the number of iframes with allow/sandbox + 1 for each website without such iframes
   COUNTIF(allow IS NOT NULL) AS freq_allow,
   COUNTIF(allow IS NOT NULL) / COUNT(0) AS pct_allow_frames,
   COUNTIF(sandbox IS NOT NULL) AS freq_sandbox,

sql/2024/css/README.md

@@ -1,7 +1,7 @@
 #standardSQL
 # Section: Attack Preventions - Preventing attacks using Cross-Origin policies
 # Question: Which are the most common COEP values?
-# Note: Considers headers of main document responses
+# Note: Considers headers of main document responses only
 SELECT
   client,
   coep_header,

sql/2024/security/coep_header_prevalence.sql

@@ -1,7 +1,7 @@
 #standardSQL
 # Section: Attack Preventions - Preventing attacks using Cross-Origin policies
 # Question: Which are the most common COOP values?
-# Note: Considers headers of main document responses
+# Note: Considers headers of main document responses only
 SELECT
   client,
   coop_header,

sql/2024/security/coop_header_prevalence.sql

@@ -1,15 +1,16 @@
 #standardSQL
 # Section: Attack Preventions - Preventing attacks using CSP
 # Question: CSP on home pages: number of unique headers, header length and number of allowed HTTP(S) hosts in all directives
+# Note: for CSP we checked whether the header value is NULL (empty?) (99.65% of CSP headers are not NULL on desktop), we did not do this for other headers?
 CREATE TEMP FUNCTION getNumUniqueHosts(str STRING) AS (
   (SELECT COUNT(DISTINCT x) FROM UNNEST(REGEXP_EXTRACT_ALL(str, r'(?i)(https*://[^\s;]+)[\s;]')) AS x)
 );
 
 SELECT
   client,
   percentile,
-  COUNT(0) AS total_requests,
-  COUNTIF(csp_header IS NOT NULL) AS total_csp_headers,
+  COUNT(0) AS total_csp_headers,
+  COUNTIF(csp_header IS NOT NULL) AS total_non_null_csp_headers,
   COUNTIF(csp_header IS NOT NULL) / COUNT(0) AS pct_csp_headers,
   COUNT(DISTINCT csp_header) AS num_unique_csp_headers,
   APPROX_QUANTILES(LENGTH(csp_header), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS csp_header_length,

sql/2024/security/csp_number_of_allowed_hosts.sql

@@ -3,7 +3,7 @@
 # Question: usage of default/script-src, and within the directive usage of strict-dynamic, nonce values, unsafe-inline and unsafe-eval
 SELECT
   client,
-  total_pages,
+  total_pages_with_csp,
   freq_csp,
   freq_default_script_src,
   SAFE_DIVIDE(freq_default_script_src, freq_csp) AS pct_default_script_src_over_csp,
@@ -22,7 +22,7 @@ SELECT
 FROM (
   SELECT
     client,
-    COUNT(0) AS total_pages,
+    COUNT(0) AS total_pages_with_csp,
     COUNTIF(csp_header IS NOT NULL) AS freq_csp,
     COUNTIF(REGEXP_CONTAINS(csp_header, '(?i)(default|script)-src')) AS freq_default_script_src,
     COUNTIF(REGEXP_CONTAINS(csp_header, '(?i)(default|script)-src[^;]+strict-dynamic')) AS freq_strict_dynamic,

sql/2024/security/csp_script_source_list_keywords.sql

@@ -12,9 +12,9 @@ SELECT
   client,
   policy_type,
   hostname,
-  total_iframes,
+  total_iframes_with_allow_or_sandbox,
   COUNTIF(has_policy) AS freq,
-  COUNTIF(has_policy) / total_iframes AS pct
+  COUNTIF(has_policy) / total_iframes_with_allow_or_sandbox AS pct
 FROM (
   SELECT
     client,
@@ -37,7 +37,7 @@ FROM (
 JOIN (
   SELECT
     client,
-    SUM(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._security'), '$.iframe-allow-sandbox'))) AS total_iframes
+    SUM(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._security'), '$.iframe-allow-sandbox'))) AS total_iframes_with_allow_or_sandbox
   FROM
     `httparchive.all.pages`
   WHERE
@@ -49,7 +49,7 @@ USING
   (client)
 GROUP BY
   client,
-  total_iframes,
+  total_iframes_with_allow_or_sandbox,
   policy_type,
   hostname
 HAVING

sql/2024/security/iframe_attribute_popular_hosts.sql

@@ -1,16 +1,29 @@
 #standardSQL
 # Section: Content Inclusion - Iframe Sandbox/Permissions Policy
-# Question: How often are the allow and sandbox attributes used on iframes? Both per page and over all iframe elements
+# Question: How often are the allow and sandbox attributes used on iframes? Both per page (used in at least one iframe on a page) and over all iframe elements
+WITH total_iframe_count AS (
+  SELECT
+    client,
+    date,
+    SUM(SAFE_CAST(JSON_EXTRACT(custom_metrics, '$.num_iframes') AS INT64)) AS total_iframes
+  FROM
+    `httparchive.all.pages`
+  WHERE
+    (date = '2022-06-01' OR date = '2023-06-01' OR date = '2023-12-01' OR date = '2024-03-01' OR date = '2024-04-01' OR date = '2024-05-01' OR date = '2024-06-01') AND
+    is_root_page
+  GROUP BY client, date
+)
+
 SELECT
   client,
   date,
-  COUNT(0) AS total_iframes,
+  total_iframes,
   COUNTIF(allow IS NOT NULL) AS freq_allow,
-  COUNTIF(allow IS NOT NULL) / COUNT(0) AS pct_allow_frames,
+  COUNTIF(allow IS NOT NULL) / total_iframes AS pct_allow_frames,
   COUNTIF(sandbox IS NOT NULL) AS freq_sandbox,
-  COUNTIF(sandbox IS NOT NULL) / COUNT(0) AS pct_sandbox_frames,
+  COUNTIF(sandbox IS NOT NULL) / total_iframes AS pct_sandbox_frames,
   COUNTIF(allow IS NOT NULL AND sandbox IS NOT NULL) AS freq_both_frames,
-  COUNTIF(allow IS NOT NULL AND sandbox IS NOT NULL) / COUNT(0) AS pct_both_frames,
+  COUNTIF(allow IS NOT NULL AND sandbox IS NOT NULL) / total_iframes AS pct_both_frames,
   COUNT(DISTINCT url) AS total_urls,
   COUNT(DISTINCT IF(allow IS NOT NULL, url, NULL)) AS allow_freq_urls,
   COUNT(DISTINCT IF(allow IS NOT NULL, url, NULL)) / COUNT(DISTINCT url) AS allow_pct_urls,
@@ -36,8 +49,9 @@ FROM (
       is_root_page
   )
   LEFT JOIN UNNEST(iframeAttrs) AS iframeAttr
-  )
+  ) JOIN total_iframe_count USING (client, date)
 GROUP BY
+  total_iframes,
   client,
   date
 ORDER BY