Merge branch 'main' of github.com:HTTPArchive/almanac.httparchive.org into production

Author: tunetheweb

.github/workflows/code-static-analysis.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Set up Python 3.8
         if: ${{ matrix.language == 'python' }}
-        uses: actions/setup-python@v4.2.0
+        uses: actions/setup-python@v4.3.0
         with:
           python-version: '3.8'
       - name: Install dependencies

.github/workflows/linter.yml

@@ -31,7 +31,7 @@ jobs:
         run: |
           echo "VALIDATE_ALL_CODEBASE=false" >> $GITHUB_ENV
       - name: Lint Code Base
-        uses: github/super-linter@v4.9.6
+        uses: github/super-linter@v4.9.7
         #uses: docker://github/super-linter:v4.7.1
         env:
           DEFAULT_BRANCH: main

.github/workflows/lintsql.yml

@@ -19,7 +19,7 @@ jobs:
           # Full git history is needed to get a proper list of changed files within `super-linter`
           fetch-depth: 0
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4.2.0
+        uses: actions/setup-python@v4.3.0
         with:
           python-version: '3.8'
       - name: Lint SQL code

.github/workflows/predeploy.yml

@@ -36,7 +36,7 @@ jobs:
       with:
         node-version: '16'
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       with:
         python-version: '3.8'
     - name: Install Asian Fonts

.github/workflows/test_website.yml

@@ -30,7 +30,7 @@ jobs:
       with:
         node-version: '16'
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       with:
         python-version: '3.8'
     - name: Run the website
@@ -40,7 +40,7 @@ jobs:
     - name: Use more complete checks for generated HTML linting
       run: cp -f .github/linters/.htmlhintrc_morechecks .github/linters/.htmlhintrc
     - name: Lint Generated HTML
-      uses: github/super-linter@v4.9.6
+      uses: github/super-linter@v4.9.7
       env:
         DEFAULT_BRANCH: main
         FILTER_REGEX_INCLUDE: src/static/html/.*

CONTRIBUTING.md

@@ -1,16 +1,11 @@
 # Contributing
 
-## 2022
+## 2023
 
-We haven't begun organizing the 2022 edition yet, but if you'd be interested to contribute, you can sign up to be notified when the project planning kicks off by filling out [this interest form](https://forms.gle/Bb2ZZj6Yw5RnJHbj8). Check off all teams that interest you and give a detailed description of what your areas of expertise are so we can reach out to you with targeted opportunities.
+We haven't begun organizing the 2023 edition yet, but if you'd be interested to contribute, you can sign up to be notified when the project planning kicks off by filling out [this interest form]([https://forms.gle/Bb2ZZj6Yw5RnJHbj8](https://forms.gle/zmk6wXfDrmkkKzXo8)). Check off all teams that interest you and give a detailed description of what your areas of expertise are so we can reach out to you with targeted opportunities.
 
 If you have any questions about contributing, you can start a discussion here on GitHub or visit the [`#web-almanac`](https://join.slack.com/t/httparchive/shared_invite/zt-45sgwmnb-eDEatOhqssqNAKxxOSLAaA) channel on Slack.
 
-
-<!--
-## 2021
-Thanks for your interest in contributing to the Web Almanac! We're currently developing the 2021 edition and actively looking for contributors! If you'd like to contribute to a specific chapter, please see [the list of open chapters](https://github.com/HTTPArchive/almanac.httparchive.org/issues/2167). We're also looking for contributors to join the [translators](https://github.com/HTTPArchive/almanac.httparchive.org/issues/923), [developers](https://github.com/HTTPArchive/almanac.httparchive.org/issues/2172), and [designers](https://github.com/HTTPArchive/almanac.httparchive.org/issues/2173) teams.
--->
 ![star-shaped Almanac character](https://almanac.httparchive.org/static/images/avatars/0.jpg)
 
 There are several ways to contribute to the Web Almanac:

sql/.sqlfluff

@@ -58,7 +58,7 @@ fix_even_unparsable = False
 # This limit skips files over a certain character length
 # and warns the user what has happened.
 # Set this to 0 to disable.
-large_file_skip_char_limit = 40000
+large_file_skip_byte_limit = 40000
 # CPU processes to use while linting.
 # If positive, just implies number of processes.
 # If negative or zero, implies number_of_cpus - specifed_number.

sql/2022/cdn/top_cdns.sql

@@ -1,26 +1,73 @@
 #standardSQL
-# top_cdns.sql: Top CDNs used on the root HTML pages
+# top_cdns.sql: Top CDNs used
 SELECT
+  year,
   client,
   cdn,
   COUNTIF(firstHtml) AS firstHtmlHits,
-  SUM(COUNTIF(firstHtml)) OVER (PARTITION BY client) AS firstHtmlTotalHits,
-  SAFE_DIVIDE(COUNTIF(firstHtml), SUM(COUNTIF(firstHtml)) OVER (PARTITION BY client)) AS firstHtmlHitsPct,
+  SUM(COUNTIF(firstHtml)) OVER (PARTITION BY year, client) AS firstHtmlTotalHits,
+  SAFE_DIVIDE(COUNTIF(firstHtml), SUM(COUNTIF(firstHtml)) OVER (PARTITION BY year, client)) AS firstHtmlHitsPct,
 
   COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain) AS subDomainHits,
-  SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain)) OVER (PARTITION BY client) AS subDomainTotalHits,
-  SAFE_DIVIDE(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain), SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain)) OVER (PARTITION BY client)) AS subDomainHitsPct,
+  SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain)) OVER (PARTITION BY year, client) AS subDomainTotalHits,
+  SAFE_DIVIDE(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain), SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND sameDomain)) OVER (PARTITION BY year, client)) AS subDomainHitsPct,
 
   COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain) AS thirdPartyHits,
-  SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain)) OVER (PARTITION BY client) AS thirdPartyTotalHits,
-  SAFE_DIVIDE(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain), SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain)) OVER (PARTITION BY client)) AS thirdPartyHitsPct,
+  SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain)) OVER (PARTITION BY year, client) AS thirdPartyTotalHits,
+  SAFE_DIVIDE(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain), SUM(COUNTIF(NOT firstHtml AND NOT sameHost AND NOT sameDomain)) OVER (PARTITION BY year, client)) AS thirdPartyHitsPct,
 
   COUNT(0) AS hits,
   SUM(COUNT(0)) OVER (PARTITION BY client) AS totalHits,
   SAFE_DIVIDE(COUNT(0), SUM(COUNT(0)) OVER (PARTITION BY client)) AS hitsPct
 FROM
   (
     SELECT
+      '2019' AS year,
+      client,
+      page,
+      url,
+      firstHtml,
+      respBodySize,
+      IFNULL(NULLIF(REGEXP_EXTRACT(_cdn_provider, r'^([^,]*).*'), ''), 'ORIGIN') AS cdn, # sometimes _cdn provider detection includes multiple entries. we bias for the DNS detected entry which is the first entry
+      NET.HOST(url) = NET.HOST(page) AS sameHost,
+      NET.HOST(url) = NET.HOST(page) OR NET.REG_DOMAIN(url) = NET.REG_DOMAIN(page) AS sameDomain # if toplevel reg_domain will return NULL so we group this as sameDomain
+    FROM
+      `httparchive.almanac.requests`
+    WHERE
+      date = '2019-07-01'
+    UNION ALL
+    SELECT
+      '2020' AS year,
+      client,
+      page,
+      url,
+      firstHtml,
+      respBodySize,
+      IFNULL(NULLIF(REGEXP_EXTRACT(_cdn_provider, r'^([^,]*).*'), ''), 'ORIGIN') AS cdn, # sometimes _cdn provider detection includes multiple entries. we bias for the DNS detected entry which is the first entry
+      NET.HOST(url) = NET.HOST(page) AS sameHost,
+      NET.HOST(url) = NET.HOST(page) OR NET.REG_DOMAIN(url) = NET.REG_DOMAIN(page) AS sameDomain # if toplevel reg_domain will return NULL so we group this as sameDomain
+    FROM
+      `httparchive.almanac.requests`
+    WHERE
+      date = '2020-08-01'
+    UNION ALL
+    SELECT
+      '2021' AS year,
+      client,
+      page,
+      url,
+      firstHtml,
+      respBodySize,
+      IFNULL(NULLIF(REGEXP_EXTRACT(_cdn_provider, r'^([^,]*).*'), ''), 'ORIGIN') AS cdn, # sometimes _cdn provider detection includes multiple entries. we bias for the DNS detected entry which is the first entry
+      NET.HOST(url) = NET.HOST(page) AS sameHost,
+      NET.HOST(url) = NET.HOST(page) OR NET.REG_DOMAIN(url) = NET.REG_DOMAIN(page) AS sameDomain # if toplevel reg_domain will return NULL so we group this as sameDomain
+    FROM
+      `httparchive.almanac.requests`
+    WHERE
+      date = '2021-07-01'
+    UNION ALL
+    SELECT
+      '2022' AS year,
       client,
       page,
       url,
@@ -35,8 +82,10 @@ FROM
       date = '2022-06-01'
   )
 GROUP BY
+  year,
   client,
   cdn
 ORDER BY
+  year DESC,
   client DESC,
   firstHtmlHits DESC

sql/2022/cdn/top_cdns_by_rank.sql

@@ -2,7 +2,11 @@
 # top_cdns_by_rank.sql: Top CDNs used on the root HTML pages by CrUX rank
 SELECT
   client,
-  nested_rank,
+  rank_grouping,
+  CASE
+    WHEN rank_grouping = 10000000 THEN 'all'
+    ELSE FORMAT("%'d", rank_grouping)
+  END AS ranking,
   cdn,
   COUNTIF(firstHtml) AS firstHtmlHits,
   SUM(COUNTIF(firstHtml)) OVER (PARTITION BY client) AS firstHtmlTotalHits,
@@ -34,14 +38,14 @@ FROM (
     `httparchive.almanac.requests`
   WHERE
     date = '2022-06-01'),
-  UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS nested_rank
+  UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping
 WHERE
-  rank <= nested_rank
+  rank <= rank_grouping
 GROUP BY
   client,
-  nested_rank,
+  rank_grouping,
   cdn
 ORDER BY
   client DESC,
-  nested_rank,
+  rank_grouping,
   firstHtmlHits DESC

sql/2022/third-parties/tao_by_third_party.sql

@@ -61,6 +61,7 @@ headers AS (
     requests.origin AS req_origin,
     pages.origin AS page_origin,
     get_tao(LOWER(respOtherHeaders)) AS timing_allow_origin,
+    respOtherHeaders,
     third_party.category AS req_category
   FROM requests
   LEFT JOIN pages
@@ -72,18 +73,30 @@ headers AS (
 base AS (
   SELECT
     client,
+    IF(respOtherHeaders LIKE '%timing-allow-origin = %', 1, 0) AS tao_header_present,
     IF(
       page_origin = req_origin OR
-      timing_allow_origin = '*, ' OR
-      STRPOS(timing_allow_origin, CONCAT(page_origin, ', ')) > 0,
+      timing_allow_origin = '*' OR
+      timing_allow_origin LIKE '*,%' OR
+      timing_allow_origin LIKE '%,*' OR
+      timing_allow_origin LIKE '%,*,%' OR
+      timing_allow_origin LIKE '%, *,%' OR
+      timing_allow_origin = page_origin OR
+      timing_allow_origin LIKE page_origin || ',' OR
+      timing_allow_origin LIKE '%,' || page_origin OR
+      timing_allow_origin LIKE '%, ' || page_origin OR
+      timing_allow_origin LIKE '%,' || page_origin || ',%' OR
+      timing_allow_origin LIKE '%, ' || page_origin || ',%',
       1, 0) AS timing_allowed
   FROM headers
 )
 
 SELECT
   client,
+  SUM(tao_header_present) AS tao_requests,
   SUM(timing_allowed) AS timing_allowed_requests,
   COUNT(0) AS total_requests,
+  SUM(tao_header_present) / COUNT(0) AS pct_tao_requests,
   SUM(timing_allowed) / COUNT(0) AS pct_timing_allowed_requests
 FROM
   base