Merge branch 'main' of github.com:HTTPArchive/almanac.httparchive.org into production

Author: tunetheweb

.github/linters/.ecrc

@@ -8,7 +8,6 @@
   "AllowedContentTypes": [],
   "PassedFiles": [],
   "Disable": {
-    // set these options to true to disable specific checks
     "EndOfLine": false,
     "Indentation": false,
     "IndentSize": false,

.github/workflows/code-static-analysis.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Set up Python 3.8
         if: ${{ matrix.language == 'python' }}
-        uses: actions/setup-python@v4.0.0
+        uses: actions/setup-python@v4.1.0
         with:
           python-version: '3.8'
       - name: Install dependencies

.github/workflows/lintsql.yml

@@ -19,7 +19,7 @@ jobs:
           # Full git history is needed to get a proper list of changed files within `super-linter`
           fetch-depth: 0
       - name: Set up Python 3.8
-        uses: actions/setup-python@v4.0.0
+        uses: actions/setup-python@v4.1.0
         with:
           python-version: '3.8'
       - name: Lint SQL code

.github/workflows/predeploy.yml

@@ -36,7 +36,7 @@ jobs:
       with:
         node-version: '16'
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: '3.8'
     - name: Install Asian Fonts

.github/workflows/test_website.yml

@@ -30,7 +30,7 @@ jobs:
       with:
         node-version: '16'
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4.0.0
+      uses: actions/setup-python@v4.1.0
       with:
         python-version: '3.8'
     - name: Run the website

sql/2021/privacy/most_common_purposes_for_iab_tcf_v2.sql

@@ -7,10 +7,10 @@ CREATE TEMPORARY FUNCTION ExtractKeyValuePairs(input STRING) RETURNS ARRAY < STR
   value STRING > > AS (
   (
     SELECT
-      array(
+      ARRAY(
         SELECT AS STRUCT
-          trim(split(kv, ':') [SAFE_OFFSET(0)]) AS key,
-          trim(split(kv, ':') [SAFE_OFFSET(1)]) AS value
+          TRIM(SPLIT(kv, ':') [SAFE_OFFSET(0)]) AS key,
+          TRIM(SPLIT(kv, ':') [SAFE_OFFSET(1)]) AS value
         FROM
           t.kv
       )

sql/util/functions.sql

@@ -0,0 +1,10 @@
+# Rework CSS parsing
+CREATE OR REPLACE FUNCTION `httparchive.almanac.PARSE_CSS`(stylesheet STRING) RETURNS STRING LANGUAGE js
+OPTIONS (library = ["gs://httparchive/lib/parse-css.js"]) AS R"""
+try {
+  var css = parse(stylesheet)
+  return JSON.stringify(css);
+} catch (e) {
+  return null;
+}
+""";

sql/util/green_web_foundation.sql

@@ -0,0 +1,8 @@
+SELECT -- noqa: L044
+  DATE('2022-06-01') AS date,
+  NET.HOST(LOWER(url)) AS host,
+  NET.REG_DOMAIN(LOWER(url)) AS domain,
+  *
+FROM
+  # This is the raw database dump from GWF.
+  `httparchive.almanac.green_web_foundation_raw`

sql/util/parsed_css.sql

@@ -1,25 +1,10 @@
-CREATE TEMP FUNCTION parseCSS(stylesheet STRING)
-RETURNS STRING
-LANGUAGE js
-OPTIONS (library = "gs://httparchive/lib/parse-css.js")
-AS '''
-  try {
-    var css = parse(stylesheet)
-    return JSON.stringify(css);
-  } catch (e) {
-    '';
-  }
-''';
-
 SELECT
-  date,
-  client,
+  DATE('2022-07-01') AS date,
+  _TABLE_SUFFIX AS client,
   page,
   url,
-  parseCSS(body) AS css
+  css
 FROM
-  `httparchive.almanac.summary_response_bodies`
+  `httparchive.experimental_parsed_css.2022_07_01_*` -- noqa: L062
 WHERE
-  date = '2020-08-01' AND
-  type = 'css' AND
-  LENGTH(body) < 3 * 1024 * 1024 # 3 MB
+  is_root_page