Skip to content

Commit 5459e6c

Browse files
nicogbgCBL-Mariner-BotPawelWMS
authored
OOT Modules -> Add kernel version and release nb into release nb (#13645)
Co-authored-by: CBL-Mariner Servicing Account <cblmargh@microsoft.com> Co-authored-by: Pawel Winogrodzki <pawelwi@microsoft.com>
1 parent 17452c2 commit 5459e6c

23 files changed

Lines changed: 287 additions & 233 deletions

File tree

‎.github/workflows/check-package-cgmanifest.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,10 @@ jobs:
4747
echo "Files to validate: '${changed_specs}'"
4848
echo "updated-specs=$(echo ${changed_specs})" >> $GITHUB_ENV
4949
50+
- name: Build the worker chroot
51+
if: ${{ env.updated-specs != '' }}
52+
run: sudo make -C toolkit -j$(nproc) chroot-tools REBUILD_TOOLS=y DAILY_BUILD_ID=lkg
53+
5054
- name: Check each spec
51-
run: |
52-
.github/workflows/overwrite_shell_link.sh
53-
.github/workflows/validate-cg-manifest.sh ${{ env.updated-specs }}
55+
if: ${{ env.updated-specs != '' }}
56+
run: .github/workflows/validate-cg-manifest.sh build/worker/worker_chroot.tar.gz ${{ env.updated-specs }}

‎.github/workflows/overwrite_shell_link.sh‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎.github/workflows/validate-cg-manifest.sh‎

Lines changed: 87 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
# - OR that a #source0 comment is a substring of the cgmanifest URL
1010
# - The URL listed in the cgmanifets is valid (can be downloaded)
1111

12-
# $@ - Paths to spec files to check
12+
# $1 - Path to worker chroot's archive
13+
# $2+ - Paths to spec files to check
1314

14-
# shellcheck source=../../toolkit/scripts/rpmops.sh
15-
source "$(git rev-parse --show-toplevel)"/toolkit/scripts/rpmops.sh
15+
set -euo pipefail
1616

1717
# Specs, which contain multiple source files and are split into many entries inside 'cgmanifest.json'.
1818
ignore_multiple_sources=" \
@@ -56,180 +56,172 @@ ignore_no_source_tarball=" \
5656
web-assets \
5757
"
5858

59-
# Specs where cgmanifest validation has known issues checking URLs.
60-
ignore_known_issues=" \
61-
virglrenderer \
62-
libesmtp"
63-
6459
alt_source_tag="Source9999"
6560

66-
function prepare_lua {
67-
local azl_lua_dir
68-
local azl_srpm_lua_dir
69-
local lua_common_path
70-
local lua_forge_path
71-
local lua_python_path
72-
local rpm_lua_dir
73-
local rpm_macros_dir
74-
75-
rpm_macros_dir="$1"
76-
77-
lua_common_path="common.lua"
78-
lua_forge_path="srpm/forge.lua"
79-
lua_python_path="srpm/python.lua"
80-
rpm_lua_dir="$(rpm --eval "%_rpmluadir")"
81-
azl_lua_dir="$rpm_lua_dir/azl"
82-
azl_srpm_lua_dir="$azl_lua_dir/srpm"
83-
84-
if [[ -z "$rpm_lua_dir" ]]
85-
then
86-
echo "ERROR: no RPM LUA directory set, can't update with Azure Linux's LUA modules!" >&2
87-
exit 1
88-
fi
89-
90-
# We only want to clean-up directories, which were absent from the system.
91-
for dir_path in "$rpm_lua_dir" "$azl_lua_dir" "$azl_srpm_lua_dir"
92-
do
93-
if [[ ! -d "$dir_path" ]]
94-
then
95-
FILES_TO_CLEAN_UP+=("$dir_path")
96-
break
97-
fi
98-
done
99-
sudo mkdir -p "$azl_srpm_lua_dir"
100-
101-
for file_path in "$lua_common_path" "$lua_forge_path" "$lua_python_path"
61+
prepare_chroot_environment() {
62+
local chroot_archive
63+
local chroot_dir_path
64+
local chroot_rpm_macros_dir_path
65+
local dist_name
66+
local dist_number
67+
local dist_tag
68+
local rpm_macros_dir_path
69+
70+
chroot_archive="$1"
71+
chroot_dir_path="$2"
72+
73+
echo "Creating worker chroot under '$chroot_dir_path'."
74+
75+
sudo tar -xf "$chroot_archive" -C "$chroot_dir_path"
76+
sudo chown -R "$(id -u):$(id -g)" "$chroot_dir_path"
77+
78+
rpm_macros_dir_path="$(sudo chroot "$chroot_dir_path" rpm --eval '%{_rpmmacrodir}')"
79+
echo "Creating the RPM macros directory '$rpm_macros_dir_path' in the chroot."
80+
chroot_rpm_macros_dir_path="$chroot_dir_path/$rpm_macros_dir_path"
81+
mkdir -vp "$chroot_rpm_macros_dir_path"
82+
83+
echo "Setting RPM's macros for the RPM queries inside the new chroot:"
84+
dist_tag=$(make -sC toolkit get-dist-tag)
85+
# Dist name is extracted from the dist tag by removing the leading dot and the number suffix.
86+
# Example: ".azl3" -> "azl"
87+
dist_name="$(sed -E 's/^\.(.*)[0-9]+$/\1/' <<<"$dist_tag")"
88+
# Dist number is the number suffix of the dist tag.
89+
# Example: ".azl3" -> "3"
90+
dist_number="$(grep -oP "\d+$" <<<"$dist_tag")"
91+
echo "%dist $dist_tag" | tee "$chroot_rpm_macros_dir_path/macros.dist"
92+
echo "%$dist_name $dist_number" | tee -a "$chroot_rpm_macros_dir_path/macros.dist"
93+
echo "%with_check 1" | tee -a "$chroot_rpm_macros_dir_path/macros.dist"
94+
for macro_file in SPECS/azurelinux-rpm-macros/macros* SPECS/pyproject-rpm-macros/macros.pyproject SPECS/perl/macros.perl
10295
do
103-
system_lua_path="$azl_lua_dir/$file_path"
104-
if [[ ! -f "$system_lua_path" ]]
105-
then
106-
sudo cp "$rpm_macros_dir/$(basename "$file_path")" "$system_lua_path"
107-
FILES_TO_CLEAN_UP+=("$system_lua_path")
108-
fi
96+
sudo cp -v "$macro_file" "$chroot_rpm_macros_dir_path"
10997
done
110-
}
11198

112-
function specs_dir_from_spec_path {
113-
# Assuming we always check specs inside Azure Linux's core GitHub repository.
114-
# If that's the case, the spec paths will always have the following form:
115-
# [repo_directory_path]/[specs_directory]/[package_name]/[package_spec_files]
116-
echo "$(realpath "$(dirname "$1")/../../SPECS")/azurelinux-rpm-macros"
99+
echo
117100
}
118101

119-
rm -f bad_registrations.txt
120-
rm -rf ./cgmanifest_test_dir/
121-
122-
if [[ $# -eq 0 ]]
102+
if [[ $# -lt 2 ]]
123103
then
124104
echo "No specs passed to validate."
125-
exit
105+
exit 1
126106
fi
127107

108+
if [[ ! -f "$1" ]]
109+
then
110+
echo "First argument is not a valid file. Please pass the path to the worker chroot's archive."
111+
exit 1
112+
fi
113+
114+
rm -f bad_registrations.txt
115+
128116
WORK_DIR=$(mktemp -d -t)
129-
FILES_TO_CLEAN_UP=("$WORK_DIR")
130117
function clean_up {
131-
echo "Cleaning up..."
132-
for file_path in "${FILES_TO_CLEAN_UP[@]}"
133-
do
134-
echo " Removing ($file_path)."
135-
sudo rm -rf "$file_path"
136-
done
118+
echo "Removing the temporary directory '$WORK_DIR'."
119+
rm -rf "$WORK_DIR"
137120
}
138121
trap clean_up EXIT SIGINT SIGTERM
139122

123+
prepare_chroot_environment "$1" "$WORK_DIR"
140124

141-
azl_macros_dir="$(specs_dir_from_spec_path "$1")"
142-
prepare_lua "$azl_macros_dir"
143-
125+
shift # Remove the first argument (the chroot archive) from the list of specs to check.
144126
echo "Checking $# specs."
145127

146128
i=0
147129
for original_spec in "$@"
148130
do
149131
i=$((i+1))
150-
echo "[$i/$#] Checking $original_spec"
132+
echo "[$i/$#] Checking $original_spec."
151133

152134
# Using a copy of the spec file, because parsing requires some pre-processing.
153135
original_spec_dir_path="$(dirname "$original_spec")"
154136
cp -r "$original_spec_dir_path" "$WORK_DIR"
155137

156138
original_spec_dir_name="$(basename "$original_spec_dir_path")"
157-
spec="$WORK_DIR/$original_spec_dir_name/$(basename "$original_spec")"
139+
chroot_spec="$original_spec_dir_name/$(basename "$original_spec")"
140+
host_spec="$WORK_DIR/$chroot_spec"
158141

159142
# Skipping specs for signed packages. Their unsigned versions should already be included in the manifest.
160143
if echo "$original_spec" | grep -q "SPECS-SIGNED"
161144
then
162-
echo " $spec is being ignored (reason: signed package), skipping"
145+
echo " $host_spec is being ignored (reason: signed package), skipping."
163146
continue
164147
fi
165148

166149
# Pre-processing alternate sources (commented-out "Source" lines with full URLs), if present. Currently we only care about the first source.
167150
# First, we replace "%%" with "%" in the alternate source's line.
168-
sed -Ei "/^#\s*Source0?:.*%%.*/s/%%/%/g" "$spec"
151+
sed -Ei "/^#\s*Source0?:.*%%.*/s/%%/%/g" "$host_spec"
169152
# Then we uncomment it.
170-
sed -Ei "s/^#\s*Source0?:/$alt_source_tag:/" "$spec"
153+
sed -Ei "s/^#\s*Source0?:/$alt_source_tag:/" "$host_spec"
171154

172155
# Removing trailing comments from "Source" tags.
173-
sed -Ei "s/^(\s*Source[0-9]*:.*)#.*/\1/" "$spec"
156+
sed -Ei "s/^(\s*Source[0-9]*:.*)#.*/\1/" "$host_spec"
174157

175-
name=$(mariner_rpmspec --srpm --qf "%{NAME}" -q "$spec" 2>/dev/null)
158+
name=$(sudo chroot "$WORK_DIR" rpmspec --srpm --qf "%{NAME}" -q "$chroot_spec" 2>/dev/null)
176159
if [[ -z $name ]]
177160
then
178161
echo "Failed to get name from '$original_spec'. Please update the spec or the macros from the 'defines' variable in this script. Error:" >> bad_registrations.txt
179-
mariner_rpmspec --srpm --qf "%{NAME}" -q "$spec" &>> bad_registrations.txt
162+
sudo chroot "$WORK_DIR" rpmspec --srpm --qf "%{NAME}" -q "$chroot_spec" &>> bad_registrations.txt
180163
continue
181164
fi
182165

183166
# Skipping specs from the ignore lists.
184-
if echo "$ignore_multiple_sources $ignore_no_source_tarball $ignore_known_issues" | grep -qP "(^|\s)$name($|\s)"
167+
if echo "$ignore_multiple_sources $ignore_no_source_tarball" | grep -qP "(^|\s)$name($|\s)"
185168
then
186-
echo " $name is being ignored (reason: explicitly ignored package), skipping"
169+
echo " $name is being ignored (reason: explicitly ignored package), skipping."
187170
continue
188171
fi
189172

190-
version=$(mariner_rpmspec --srpm --qf "%{VERSION}" -q "$spec" 2>/dev/null )
173+
version=$(sudo chroot "$WORK_DIR" rpmspec --srpm --qf "%{VERSION}" -q "$chroot_spec" 2>/dev/null )
191174
if [[ -z $version ]]
192175
then
193176
echo "Failed to get version from '$original_spec'. Please update the spec or the macros from the 'defines' variable in this script. Error:" >> bad_registrations.txt
194-
mariner_rpmspec --srpm --qf "%{VERSION}" -q "$spec" &>> bad_registrations.txt
177+
sudo chroot "$WORK_DIR" rpmspec --srpm --qf "%{VERSION}" -q "$chroot_spec" &>> bad_registrations.txt
195178
continue
196179
fi
197180

198-
parsed_spec="$(mariner_rpmspec --parse "$spec" 2>/dev/null)"
181+
parsed_spec="$(sudo chroot "$WORK_DIR" rpmspec --parse "$chroot_spec" 2>/dev/null)"
199182

200183
# Reading the source0 file/URL.
201-
source0=$(echo "$parsed_spec" | grep -P "^\s*Source0?:" | cut -d: -f2- | xargs)
202-
if [[ -z $source0 ]]
184+
if ! echo "$parsed_spec" | grep -qP "^\s*Source0?:"
203185
then
204-
echo " No source file listed for $name:$version, skipping"
186+
echo " No source file listed for $name:$version, skipping."
205187
continue
206188
fi
207189

190+
source0=$(echo "$parsed_spec" | grep -P "^\s*Source0?:" | cut -d: -f2- | xargs)
191+
echo " Source0: $source0."
192+
208193
# Reading the alternate source URL.
209-
source0alt=$(echo "$parsed_spec" | grep -P "^\s*$alt_source_tag:" | cut -d: -f2- | xargs)
194+
source0_alt=""
195+
if echo "$parsed_spec" | grep -qP "^\s*$alt_source_tag:"
196+
then
197+
source0_alt=$(echo "$parsed_spec" | grep -P "^\s*$alt_source_tag:" | cut -d: -f2- | xargs)
198+
echo " Source0Alt: $source0_alt."
199+
fi
210200

211201
# Pull the current registration from the cgmanifest file. Every registration should have a URL, so if we don't find one
212202
# that implies the registration is missing.
213-
manifesturl=$(jq --raw-output ".Registrations[].component.other | select(.name==\"$name\" and .version==\"$version\") | .downloadUrl" cgmanifest.json)
214-
if [[ -z $manifesturl ]]
203+
manifest_url=$(jq --raw-output ".Registrations[].component.other | select(.name==\"$name\" and .version==\"$version\") | .downloadUrl" cgmanifest.json)
204+
if [[ -z $manifest_url ]]
215205
then
216206
echo "Registration for $name:$version is missing" >> bad_registrations.txt
217207
else
218-
if [[ "$manifesturl" != "$source0" && "$manifesturl" != "$source0alt" ]]
208+
echo " Registration URL: $manifest_url."
209+
210+
if [[ "$manifest_url" != "$source0" && "$manifest_url" != "$source0_alt" ]]
219211
then
220212
{
221-
echo "Registration URL for $name:$version ($manifesturl) matches neither the first \"Source\" tag nor the alternate source URL."
213+
echo "Registration URL for $name:$version ($manifest_url) matches neither the first \"Source\" tag nor the alternate source URL."
222214
printf '\tFirst "Source" tag:\t%s\n' "$source0"
223-
printf '\tAlternate source URL:\t%s\n' "$source0alt"
215+
printf '\tAlternate source URL:\t%s\n' "$source0_alt"
224216
} >> bad_registrations.txt
225217
else
226218
# Try a few times to download the source listed in the manifest
227219
# Parsing output instead of using error codes because 'wget' returns code 8 for FTP, even if the file exists.
228220
# Sample HTTP(S) output: Remote file exists.
229221
# Sample FTP output: File ‘time-1.9.tar.gz’ exists.
230-
if ! wget --secure-protocol=TLSv1_2 --spider --timeout=30 --tries=10 "${manifesturl}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
222+
if ! wget --secure-protocol=TLSv1_2 --spider --timeout=30 --tries=10 "${manifest_url}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
231223
then
232-
echo "Registration for $name:$version has invalid URL '$manifesturl' (could not download)" >> bad_registrations.txt
224+
echo "Registration for $name:$version has invalid URL '$manifest_url' (could not download)" >> bad_registrations.txt
233225
fi
234226
fi
235227
fi

‎SPECS-SIGNED/fwctl-signed/fwctl-signed.spec‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,19 @@
3535
%define __os_install_post %{__os_install_post_leave_signatures} %{nil}
3636

3737
%global target_kernel_version_full %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}-%{RPMTAG_RELEASE}' $(/bin/rpm -q --whatprovides kernel-headers))
38-
%global target_azurelinux_build_kernel_version %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}' $(/bin/rpm -q --whatprovides kernel-headers))
38+
%global target_azl_build_kernel_version %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}' $(/bin/rpm -q --whatprovides kernel-headers))
3939
%global target_kernel_release %(/bin/rpm -q --queryformat '%{RPMTAG_RELEASE}' $(/bin/rpm -q --whatprovides kernel-headers) | /bin/cut -d . -f 1)
40+
%global release_suffix _%{target_azl_build_kernel_version}.%{target_kernel_release}
4041

4142
%global KVERSION %{target_kernel_version_full}
4243

4344
%{!?_name: %define _name fwctl}
45+
%{!?_mofed_full_version: %define _mofed_full_version 24.10-17%{release_suffix}%{?dist}}
4446

4547
Summary: %{_name} Driver
4648
Name: %{_name}-signed
4749
Version: 24.10
48-
Release: 19%{?dist}
50+
Release: 20%{release_suffix}%{?dist}
4951
License: GPLv2
5052
Url: http://nvidia.com
5153
Group: System Environment/Base
@@ -70,8 +72,8 @@ fwctl signed kernel modules
7072

7173
%package -n %{_name}
7274
Summary: %{summary}
73-
Requires: mlnx-ofa_kernel = %{version}
74-
Requires: mlnx-ofa_kernel-modules = %{version}
75+
Requires: mlnx-ofa_kernel = %{_mofed_full_version}
76+
Requires: mlnx-ofa_kernel-modules = %{_mofed_full_version}
7577
Requires: kernel = %{target_kernel_version_full}
7678
Requires: kmod
7779

@@ -117,6 +119,9 @@ fi # 1 : closed
117119

118120

119121
%changelog
122+
* Thu May 29 2025 Nicolas Guibourge <nicolasg@microsoft.com> - 24.10-20
123+
- Add kernel version and release nb into release nb
124+
120125
* Fri May 23 2025 CBL-Mariner Servicing Account <cblmargh@microsoft.com> - 24.10-19
121126
- Bump release to rebuild for new kernel release
122127

‎SPECS-SIGNED/iser-signed/iser-signed.spec‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,19 @@
3131
%define __os_install_post %{__os_install_post_leave_signatures} %{nil}
3232

3333
%global target_kernel_version_full %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}-%{RPMTAG_RELEASE}' $(/bin/rpm -q --whatprovides kernel-headers))
34-
%global target_azurelinux_build_kernel_version %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}' $(/bin/rpm -q --whatprovides kernel-headers))
34+
%global target_azl_build_kernel_version %(/bin/rpm -q --queryformat '%{RPMTAG_VERSION}' $(/bin/rpm -q --whatprovides kernel-headers))
3535
%global target_kernel_release %(/bin/rpm -q --queryformat '%{RPMTAG_RELEASE}' $(/bin/rpm -q --whatprovides kernel-headers) | /bin/cut -d . -f 1)
36+
%global release_suffix _%{target_azl_build_kernel_version}.%{target_kernel_release}
3637

3738
%global KVERSION %{target_kernel_version_full}
3839

3940
%{!?_name: %define _name iser}
41+
%{!?_mofed_full_version: %define _mofed_full_version 24.10-17%{release_suffix}%{?dist}}
4042

4143
Summary: %{_name} Driver
4244
Name: %{_name}-signed
4345
Version: 24.10
44-
Release: 19%{?dist}
46+
Release: 20%{release_suffix}%{?dist}
4547
License: GPLv2
4648
Url: http://www.mellanox.com
4749
Group: System Environment/Base
@@ -65,8 +67,8 @@ iser signed kernel modules
6567

6668
%package -n %{_name}
6769
Summary: %{summary}
68-
Requires: mlnx-ofa_kernel = %{version}
69-
Requires: mlnx-ofa_kernel-modules = %{version}
70+
Requires: mlnx-ofa_kernel = %{_mofed_full_version}
71+
Requires: mlnx-ofa_kernel-modules = %{_mofed_full_version}
7072
Requires: kernel = %{target_kernel_version_full}
7173
Requires: kmod
7274

@@ -108,6 +110,9 @@ fi # 1 : closed
108110
%config(noreplace) %{_sysconfdir}/depmod.d/zz02-%{_name}-*.conf
109111

110112
%changelog
113+
* Thu May 29 2025 Nicolas Guibourge <nicolasg@microsoft.com> - 24.10-20
114+
- Add kernel version and release nb into release nb
115+
111116
* Fri May 23 2025 CBL-Mariner Servicing Account <cblmargh@microsoft.com> - 24.10-19
112117
- Bump release to rebuild for new kernel release
113118

0 commit comments

Comments
 (0)