Skip to content

Commit 26dba9e

Browse files
Niall LangleyNJLangley
Niall Langley
authored and
NJLangley
committed
Build improvements for logging and security
1 parent 5057df0 commit 26dba9e

7 files changed

Lines changed: 361 additions & 108 deletions

File tree

‎.github/workflows/build_and_run_unit_tests.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
push:
55
branches:
66
- main
7-
- feature/*
7+
#- feature/*
88
pull_request:
99
branches:
1010
- main

‎.github/workflows/test_azure_devtest_labs_integration.yml‎

Lines changed: 114 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,49 @@ jobs:
2626
RESOURCE_GROUP: AzureDataPipelineTools_CI
2727
LAB_NAME: AzureDataPipelineTools
2828
ARM_TEMPLATE_NAME: sqlcollaborative_AzureDataPipelineTools
29-
29+
FUNCTIONS_PROJECT_NAME: DataPipelineTools.Functions
30+
BUILD_CONFIGURATION: Release
31+
3032

3133
# Steps represent a sequence of tasks that will be executed as part of the job
3234
steps:
3335
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
3436
- uses: actions/checkout@v2
3537

38+
39+
# Build and run the unit tests
40+
- name: Setup .NET
41+
uses: actions/setup-dotnet@v1
42+
with:
43+
dotnet-version: 3.1.x
44+
- name: Restore dependencies
45+
run: dotnet restore
46+
- name: Build
47+
run: dotnet build --no-restore --configuration ${{ env.BUILD_CONFIGURATION }} --output ./output
48+
49+
50+
- name: List Build Files
51+
run: |
52+
ls -ltAR ./${{ env.FUNCTIONS_PROJECT_NAME }}/bin/${{ env.BUILD_CONFIGURATION }}
53+
54+
- name: Run Unit Tests
55+
run: dotnet test --no-build --configuration ${{ env.BUILD_CONFIGURATION }} --output ./output --verbosity normal
56+
57+
# - name: Extract
58+
# run: |
59+
# echo "Retriving service principal client secret for the logged in user..."
60+
# echo $("${{ secrets.AZURE_DEV_TEST_LABS_CREDENTIALS }}" | jq '.clientId')
61+
62+
63+
3664
# Login to Azure
3765
- name: Log into Azure
3866
uses: azure/login@v1
3967
with:
4068
creds: '${{ secrets.AZURE_DEV_TEST_LABS_CREDENTIALS }}'
4169

70+
71+
4272
# --raw-output is requiored on all jq commands that select strings, otherwise they come out with the quotes, which is interpreted as a command, and it likely won't exist!
4373
- name: Create DevTest Labs Environment
4474
id: create-devtest-labs-environment
@@ -52,6 +82,12 @@ jobs:
5282
echo "Connection info:"
5383
az account show | jq '. | {tenantId: .tenantId, subscriptionName: .name, userName: .user.name, userType: .user.type}'
5484
echo
85+
86+
echo "Configuring variables for secrets:"
87+
# We need the secrets in a variable so we can work with them using normal shell script syntax. The '${ { VAR } }' syntax is token replaced before the script runs.
88+
SERVICEPRINCIPALCREDENTIALS=$'${{ secrets.AZURE_DEV_TEST_LABS_CREDENTIALS }}'
89+
echo "========================================================================================================================================================================================================"
90+
echo
5591
5692
ARTIFACT_SOURCE_NAME=$(az lab artifact-source list --resource-group $RESOURCE_GROUP \
5793
--lab-name $LAB_NAME \
@@ -64,26 +100,25 @@ jobs:
64100
BRANCH_NAME=${GITHUB_REF#*refs/heads/}
65101
echo "Branch Name: $BRANCH_NAME"
66102
67-
# This is the object id of the Enterprise Application created from the App Registration
68-
103+
# We need the object id of the Enterprise Application created from the App Registration in order to set permissions in the ARM template. This is **not** the same as the app/client id
69104
echo "Retriving service principal id for the logged in user..."
70105
SERVICEPRINCIPALAPPID=$(az account show | jq --raw-output '.user.name')
71-
echo "Service Principal App Id: $SERVICEPRINCIPALAPPID"
72-
106+
echo "Service Principal App/Client Id: $SERVICEPRINCIPALAPPID"
73107
SERVICEPRINCIPALID=$( az ad sp list --filter "appId eq '$SERVICEPRINCIPALAPPID' and servicePrincipalType eq 'Application'" --query [0].objectId --output tsv)
74-
echo "Service Principal Object Id: $SERVICEPRINCIPALID"
108+
echo "Service Principal Object Id: $SERVICEPRINCIPALID"
75109
76110
echo "Building parameters file for ARM deployment..."
77111
PARAMETERS_FILE="$(pwd)/azuredeploy.parameters.json"
78112
echo $'[ { "name":"'branch'", "value":"'$BRANCH_NAME'" },' \
79113
' { "name":"'commit'", "value":"'$GITHUB_SHA'" },' \
80114
' { "name":"'location'", "value":"UK South" },' \
81-
' { "name":"'devopsServicePrincipalId'", "value":"'$SERVICEPRINCIPALID'" }' \
115+
' { "name":"'devopsServicePrincipalId'", "value":"'$SERVICEPRINCIPALID'" },' \
116+
' { "name":"'devopsServicePrincipalCredentials'", "value":' $SERVICEPRINCIPALCREDENTIALS ' }' \
82117
']' \
83118
| jq '.' > "$PARAMETERS_FILE"
84-
cat $PARAMETERS_FILE
119+
#cat $PARAMETERS_FILE
85120
86-
ENVIRONMENT_INSTANCE_NAME='CI_Build___'"${BRANCH_NAME////__}"'___('"${GITHUB_SHA:0:8}"')'
121+
ENVIRONMENT_INSTANCE_NAME='CI_Build___'"${BRANCH_NAME////__}"'___'"${GITHUB_SHA:0:8}"''
87122
echo "Environment Instance Name: $ENVIRONMENT_INSTANCE_NAME"
88123
89124
echo "::set-output name=ENVIRONMENT_INSTANCE_NAME::$ENVIRONMENT_INSTANCE_NAME"
@@ -115,66 +150,86 @@ jobs:
115150
fi
116151
117152
echo "========================================================================================================================================================================================================"
153+
DEPLOYMENTOUTPUT=$(az deployment group list --resource-group $ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME --query '[0].properties.outputs')
118154
155+
# DEBUG: Use this to get the full deployment output JSON. If the ARM template outputs a full reference to a resource, we can find the bits we need easily.
156+
# echo "::set-output name=DEPLOYMENTOUTPUT::$DEPLOYMENTOUTPUT"
157+
158+
echo "Deployment Outputs"
159+
echo "::set-output name=STORAGE_ACCOUNTCONNECTION_STRING::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageAccountConnectionString.value')"
160+
echo "::set-output name=STORAGE_ACCOUNT_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageAccountName.value')"
161+
echo "::set-output name=FUNCTIONS_APP_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppName.value')"
162+
echo "::set-output name=FUNCTIONS_APP_URI::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppUri.value')"
163+
echo "::set-output name=KEY_VAULT_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.keyVaultName.value')"
164+
echo "::set-output name=FUNCTIONS_APP_KEY::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppKey.value')"
119165
120-
- name: Get DevTest Labs Environment Resource Info
121-
id: get-devtest-labs-environment-resources
122-
uses: azure/CLI@v1
123-
with:
124-
#azcliversion: 2.24.0 # Versions prior to this have a bug creating environments
125-
inlineScript: |
126166
echo "========================================================================================================================================================================================================"
127-
echo "Azure CLI Version: $(az version | jq '."azure-cli"')"
128-
echo
129-
echo "Connection info:"
130-
az account show | jq '. | {tenantId: .tenantId, subscriptionName: .name, userName: .user.name, userType: .user.type}'
131-
echo
167+
168+
169+
170+
# Runs a set of commands using the runners shell
171+
- name: Show create environment outputs
172+
run: |
173+
echo "ENVIRONMENT_INSTANCE_NAME: ${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_NAME }}"
174+
echo "ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME: ${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME }}"
175+
echo "FUNCTIONS_APP_NAME: ${{ steps.create-devtest-labs-environment.outputs.FUNCTIONS_APP_NAME }}"
176+
echo "FUNCTIONS_APP_URI: ${{ steps.create-devtest-labs-environment.outputs.FUNCTIONS_APP_URI }}"
177+
echo "FUNCTIONS_APP_KEY: ${{ steps.create-devtest-labs-environment.outputs.FUNCTIONS_APP_KEY }}"
178+
echo "STORAGE_ACCOUNTCONNECTION_STRING: ${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNTCONNECTION_STRING }}"
179+
echo "STORAGE_ACCOUNT_NAME: ${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNT_NAME }}"
180+
echo "KEY_VAULT_NAME: ${{ steps.create-devtest-labs-environment.outputs.KEY_VAULT_NAME }}"
181+
# echo "ENVIRONMENT_INSTANCE_RESOURCES: ${{ steps.get-devtest-labs-environment-resources.outputs.ENVIRONMENT_INSTANCE_RESOURCES }}"
182+
# az lab arm-template show --resource-group $RESOURCE_GROUP
132183

133-
RESOURCE_GROUP="${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME }}"
134184

135-
ENVIRONMENT_INSTANCE_RESOURCES=$(az resource list --resource-group $RESOURCE_GROUP \
136-
| jq '.[] | { name: .name, id: .id, provisioningState: .provisioningState, kind: .kind, location: .location, tags: .tags }'
137-
)
138185

139-
echo "Output from 'az resource list --resource-group $RESOURCE_GROUP'"
140-
echo
141-
echo $ENVIRONMENT_INSTANCE_RESOURCES
142186

143-
echo "::set-output name=ENVIRONMENT_INSTANCE_RESOURCES::$ENVIRONMENT_INSTANCE_RESOURCES"
144187

145-
echo "========================================================================================================================================================================================================"
146188

189+
# - name: Get DevTest Labs Environment Resource Info
190+
# id: get-devtest-labs-environment-resources
191+
# uses: azure/CLI@v1
192+
# with:
193+
# #azcliversion: 2.24.0 # Versions prior to this have a bug creating environments
194+
# inlineScript: |
195+
# echo "========================================================================================================================================================================================================"
196+
# echo "Azure CLI Version: $(az version | jq '."azure-cli"')"
197+
# echo
198+
# echo "Connection info:"
199+
# az account show | jq '. | {tenantId: .tenantId, subscriptionName: .name, userName: .user.name, userType: .user.type}'
200+
# echo
201+
202+
# RESOURCE_GROUP="${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME }}"
203+
204+
# ENVIRONMENT_INSTANCE_RESOURCES=$(az resource list --resource-group $RESOURCE_GROUP \
205+
# | jq '.[] | { name: .name, id: .id, provisioningState: .provisioningState, kind: .kind, location: .location, tags: .tags }'
206+
# )
207+
208+
# echo "Output from 'az resource list --resource-group $RESOURCE_GROUP'"
209+
# echo
210+
# echo $ENVIRONMENT_INSTANCE_RESOURCES
211+
212+
# echo "::set-output name=ENVIRONMENT_INSTANCE_RESOURCES::$ENVIRONMENT_INSTANCE_RESOURCES"
147213

214+
# echo "========================================================================================================================================================================================================"
148215

216+
217+
# Install AZCopy and copy sample files into the data lake
218+
- name: Install azcopy
219+
uses: kheiakiyama/install-azcopy-action@v1.0.3
220+
with:
221+
version: 'v10'
149222

150-
# Runs a set of commands using the runners shell
151-
- name: Run a multi-line script
223+
- name: Copy files to Azure Data Lake using AZCopy
152224
run: |
153-
echo "ENVIRONMENT_INSTANCE_NAME: ${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_NAME }}"
154-
echo "ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME: ${{ steps.create-devtest-labs-environment.outputs.ENVIRONMENT_INSTANCE_RESOURCE_GROUP_NAME }}"
155-
echo "ENVIRONMENT_INSTANCE_RESOURCES: ${{ steps.get-devtest-labs-environment-resources.outputs.ENVIRONMENT_INSTANCE_RESOURCES }}"
156-
# az lab arm-template show --resource-group $RESOURCE_GROUP \
157-
# --lab-name $LAB_NAME \
158-
# --artifact-source-name $ARTIFACT_SOURCE_NAME \
159-
# --name $ARM_TEMPLATE_NAME
160-
161-
162-
# # Once logged into Azure, run a test command to check it worked
163-
# - name: Azure CLI script
164-
# uses: azure/CLI@v1
165-
# with:
166-
# # This is the version with the fix for creating a DevTest lab
167-
# azcliversion: 2.24.0
168-
# inlineScript: |
169-
# az account show
170-
171-
172-
# # Runs a single command using the runners shell
173-
# - name: Run a one-line script
174-
# run: echo Hello, world!
175-
176-
# # Runs a set of commands using the runners shell
177-
# - name: Run a multi-line script
178-
# run: |
179-
# echo Add other actions to build,
180-
# echo test, and deploy your project.
225+
ls -ltA
226+
#azcopy_v10 --source {SOURCE} --destination {DEST} --dest-key ${{ secrets.STORAGE_KEY }} --recursive --set-content-type
227+
228+
- name: 'Deploy Azure Functions to Lab Environment using RBAC'
229+
uses: Azure/functions-action@v1
230+
with:
231+
app-name: ${{ steps.create-devtest-labs-environment.outputs.FUNCTIONS_APP_NAME }}
232+
package: './${{ env.FUNCTIONS_PROJECT_NAME }}/bin/${{ env.BUILD_CONFIGURATION }}'
233+
234+
235+

0 commit comments

Comments
 (0)