Skip to content

Commit ab3e32c

Browse files
nrllhmgifford
nrllh
and
mgifford
authored
JavaScript Queries for 2024 (#3735)
* initial commit * JavaScript Queries for 2024 * bytes_by_3p.sql fixed * Update breakdown_of_pages_using_async_defer.sql fixing easy linter issue. * Update breakdown_of_scripts_using_async_defer_module_nomodule.sql fixing linter error * Update distribution_of_injected_scripts.sql Fixing linter errors * Update distribution_of_long_tasks_time.sql Fixing linter * Update injected_scripts.sql fixing linter errors * Update percent_of_dns_prefetch.sql Fixing Linter * Update percent_of_preconnect_by_host.sql Fixing linter errors * Update render_blocking_javascript.sql Fixing linter errors * Update render_blocking_javascript_by_rank.sql Fixing Linter errors * Update resource-hints-prefetch-preload-modulepreload-percentage.sql Fixing Linter issues * Update resource-hints-preload-prefetch-modulepreload-distribution.sql Linter errors * Update resource_hints.sql Fixing Linter errors. * Update resource_hints_per_page.sql Fixing Linter errors * Update sourcemaps.sql Fixing Linter Errors * Update usage_of_typescript_and_babel.sql Fixing linter errors * Update web_components_is_attribute.sql Linter error... * Update web_components_pct.sql Fixing linter * Update breakdown_of_pages_using_async_defer.sql This just had to be a typo. Where should just be there once. Linter error. * Update breakdown_of_scripts_using_async_defer_module_nomodule.sql Spacing thanks to linter. * Update resource-hints-prefetch-preload-modulepreload-percentage.sql trailing white space * Update resource-hints-preload-prefetch-modulepreload-distribution.sql trailing spaces * Update resource_hints.sql trailing space * Update breakdown_of_pages_using_async_defer.sql trailing white space * Update bytes_by_3p.sql fixing linter errors * Update distribution_of_injected_scripts.sql linter prefers ' * Update distribution_of_long_tasks_time.sql adjusting for linter * Update distribution_of_number_of_long_tasks.sql Linter * Update frameworks_libraries_by_version.sql Spac ing * Update injected_scripts.sql Trailing spaces * Update module_and_nomodule.sql adjusting for linter * Update percent_of_dns_prefetch.sql Adjusting for linter * Update percent_of_dns_prefetch_by_host.sql Linter * Update percent_of_preconnect_by_host.sql Linter * Update render_blocking_javascript.sql Linter * Update render_blocking_javascript_by_rank.sql Linter * Update resource-hints-prefetch-preload-modulepreload-percentage.sql Linter * Update resource-hints-preload-prefetch-modulepreload-distribution.sql Linter errors * Update resource_hints.sql Linter * Update resource_hints_per_page.sql Linter * Update sourcemaps.sql Linter * Update usage_of_typescript_and_babel.sql Linter * Update web_components_is_attribute.sql Linter * Update web_components_pct.sql Linter * Update web_components_specs.sql Linter * Update module_and_nomodule.sql Adding a space * Update distribution_of_number_of_long_tasks.sql Spacing * Update distribution_of_long_tasks_time.sql Spaces * Update frameworks_libraries_by_version.sql Fixing double spaces. * Update module_and_nomodule.sql Datatypes must be upper case. * Update bytes_by_3p.sql Datatypes must be upper case. --------- Co-authored-by: Mike Gifford <mike.gifford@civicactions.com>
1 parent b20d773 commit ab3e32c

65 files changed

Lines changed: 2508 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎sql/2024/javascript/breakdown_of_pages_using_async_defer.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#standardSQL
2+
# Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts
3+
SELECT
4+
_TABLE_SUFFIX AS client,
5+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) AS async,
6+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) AS defer,
7+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) AS async_and_defer,
8+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) AS module,
9+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) AS nomodule,
10+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) / COUNT(0) AS async_pct,
11+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) / COUNT(0) AS defer_pct,
12+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) / COUNT(0) AS async_and_defer_pct,
13+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) / COUNT(0) AS module_pct,
14+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) / COUNT(0) AS nomodule_pct,
15+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) AS neither,
16+
COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct
17+
FROM
18+
`httparchive.all.pages`
19+
WHERE
20+
date = '2024-06-01'
21+
22+
GROUP BY
23+
client

‎sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#standardSQL
2+
# Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts
3+
CREATE TEMPORARY FUNCTION getScripts(payload STRING) RETURNS STRUCT < total INT64,
4+
inline INT64,
5+
src INT64,
6+
async INT64,
7+
defer INT64,
8+
async_and_defer INT64,
9+
type_module INT64,
10+
nomodule INT64 > LANGUAGE js AS '''
11+
try {
12+
var $ = JSON.parse(payload);
13+
var javascript = JSON.parse($._javascript);
14+
return javascript.script_tags;
15+
} catch (e) {
16+
return {};
17+
}
18+
''';
19+
20+
SELECT
21+
client,
22+
SUM(script.total) AS total_scripts,
23+
SUM(script.inline) AS inline_script,
24+
SUM(script.src) AS external_script,
25+
SUM(script.src) / SUM(script.total) AS pct_external_script,
26+
SUM(script.inline) / SUM(script.total) AS pct_inline_script,
27+
SUM(script.async) AS async,
28+
SUM(script.defer) AS defer,
29+
SUM(script.async_and_defer) AS async_and_defer,
30+
SUM(script.type_module) AS module,
31+
SUM(script.nomodule) AS nomodule,
32+
SUM(script.async) / SUM(script.src) AS pct_external_async,
33+
SUM(script.defer) / SUM(script.src) AS pct_external_defer,
34+
SUM(script.async_and_defer) / SUM(script.src) AS pct_external_async_defer,
35+
SUM(script.type_module) / SUM(script.src) AS pct_external_module,
36+
SUM(script.nomodule) / SUM(script.src) AS pct_external_nomodule
37+
FROM
38+
(
39+
SELECT
40+
client,
41+
getScripts(payload) AS script
42+
FROM
43+
`httparchive.all.pages`
44+
WHERE
45+
date = '2024-06-01'
46+
)
47+
GROUP BY
48+
client

‎sql/2024/javascript/bytes_2019.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2019)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2019_07_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_2020.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2020)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2020_08_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_2021.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2021)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2021_07_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_2022.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2022)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2022_06_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_2023.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2023)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2023_06_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_2024.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#standardSQL
2+
# Sum of JS request bytes per page (2024)
3+
SELECT
4+
percentile,
5+
_TABLE_SUFFIX AS client,
6+
APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
7+
FROM
8+
`httparchive.summary_pages.2024_06_01_*`,
9+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
10+
GROUP BY
11+
percentile,
12+
client
13+
ORDER BY
14+
client,
15+
percentile

‎sql/2024/javascript/bytes_by_3p.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#standardSQL
2+
# Distribution of 1P/3P JS bytes
3+
SELECT
4+
percentile,
5+
client,
6+
host,
7+
APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes
8+
FROM (
9+
SELECT
10+
client,
11+
page,
12+
IF(NET.HOST(url) IN (
13+
SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting'
14+
), 'third party', 'first party') AS host,
15+
SUM(cast(json_value(payload, '$.response.bodySize') AS INT64)) / 1024 AS kbytes
16+
FROM
17+
`httparchive.all.requests`
18+
19+
WHERE
20+
date = '2024-06-01' AND
21+
type = 'script'
22+
GROUP BY
23+
client,
24+
page,
25+
host),
26+
UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
27+
GROUP BY
28+
percentile,
29+
client,
30+
host
31+
ORDER BY
32+
client,
33+
percentile,
34+
host

‎sql/2024/javascript/compression_method.sql‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#standardSQL
2+
CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
3+
RETURNS STRING
4+
DETERMINISTIC
5+
LANGUAGE js AS '''
6+
const parsed_headers = JSON.parse(headers);
7+
const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
8+
if (matching_headers.length > 0) {
9+
return matching_headers[0].value;
10+
}
11+
return null;
12+
''';
13+
14+
SELECT
15+
client,
16+
compression,
17+
COUNT(DISTINCT page) AS pages,
18+
ANY_VALUE(total_pages) AS total_pages,
19+
COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
20+
COUNT(0) AS js_requests,
21+
SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests,
22+
COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests
23+
FROM (
24+
SELECT
25+
client,
26+
page,
27+
getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression
28+
FROM
29+
`httparchive.all.requests`
30+
WHERE
31+
date = '2024-06-01' AND
32+
type = 'script')
33+
JOIN (
34+
SELECT
35+
_TABLE_SUFFIX AS client,
36+
COUNT(0) AS total_pages
37+
FROM
38+
`httparchive.summary_pages.2024_06_01_*`
39+
GROUP BY
40+
client)
41+
USING
42+
(client)
43+
GROUP BY
44+
client,
45+
compression
46+
ORDER BY
47+
pct_js_requests DESC

0 commit comments

Comments
 (0)