Skip to content

Commit 74a67ed

Browse files
NJLangleyNJLangley
committed
Improved configuration of settings such as tenant id so that the integration tests are able to grab the local dev config to run, when running in the fuctions emulator. Added a base class for data lake integration tests to test the required settings, such as the account, container, auth and path, as this will ensure consistency accross the API and reduce testing code.
1 parent cc0a01c commit 74a67ed

19 files changed

+346
-188
lines changed

‎.github/workflows/deploy.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ RESOURCE_NAME_SUFFIX="_${ACTOR_SHORT}_${RUNTIMESTAMP:8:12}_${GITHUB_SHA:0:7}"
4646
echo "Resource Name Suffix: $RESOURCE_NAME_SUFFIX"
4747

4848
# We need the object id of the Enterprise Application created from the App Registration in order to set permissions in the ARM template. This is **not** the same as the app/client id
49-
echo "Retriving service principal id for the logged in user..."
49+
echo "Retriving service principal info for the logged in user..."
50+
TENANTID=$(az account show | jq --raw-output '.homeTenantId')
5051
SERVICEPRINCIPALAPPID=$(az account show | jq --raw-output '.user.name')
5152
#echo "Service Principal App/Client Id: $SERVICEPRINCIPALAPPID"
5253
SERVICEPRINCIPALID=$( az ad sp list --filter "appId eq '$SERVICEPRINCIPALAPPID' and servicePrincipalType eq 'Application'" --query [0].objectId --output tsv)
@@ -116,6 +117,7 @@ echo "==========================================================================
116117
# DEBUG: Use this to get the full deployment output JSON. If the ARM template outputs a full reference to a resource, we can find the bits we need easily.
117118
#echo "::set-output name=DEPLOYMENTOUTPUT::$DEPLOYMENTOUTPUT"
118119

120+
echo "::set-output name=TENANTID::$TENANTID"
119121
echo "::set-output name=STORAGE_ACCOUNT_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageAccountName.value')"
120122
echo "::set-output name=STORAGE_CONTAINER_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageContainerName.value')"
121123
echo "::set-output name=FUNCTIONS_APP_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppName.value')"

‎.github/workflows/test_azure_devtest_labs_integration.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ jobs:
107107
echo '${{ steps.create-devtest-labs-environment.outputs.RUN_SETTINGS }}' > ${{ env.RUN_SETTINGS_FILENAME }}
108108
109109
110-
# Install AZCopy and copy sample files into the data lake
110+
# Install AZCopy and copy sample files into the data lake
111111
- name: Install azcopy
112112
uses: kheiakiyama/install-azcopy-action@v1.0.3
113113
with:
@@ -138,6 +138,8 @@ jobs:
138138

139139

140140
- name: Run Integration Tests
141+
env:
142+
TENANT_ID: ${{ steps.create-devtest-labs-environment.outputs.TENANTID }}
141143
run: dotnet test --no-build --configuration ${{ env.BUILD_CONFIGURATION }} --filter TestCategory=IntegrationTest --output ./output --results-directory ./test-results --verbosity normal --logger "trx;LogFileName=integration-test-results.trx" --settings ${{ env.RUN_SETTINGS_FILENAME }}
142144

143145

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
## files generated by popular Visual Studio add-ons.
33

44
# Azure Functions localsettings file
5-
local.settings.json
5+
secrets.settings.json
6+
**/Properties/PublishProfiles/*
67

78
# User-specific files
89
*.suo

‎DataPipelineTools.Functions.Tests/DataLake/DataLakeFunctions/Unit/DataLakeGetItemsTests.cs‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎DataPipelineTools.Functions.Tests/Integration/DataLake/DataLakeCheckPathCaseIntegrationTests.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using DataPipelineTools.Tests.Common;
2+
using NUnit.Framework;
3+
4+
namespace DataPipelineTools.Functions.Tests.Integration.DataLake
5+
{
6+
[TestFixture]
7+
[Category(nameof(TestType.IntegrationTest))]
8+
[Parallelizable(ParallelScope.Children)]
9+
public class DataLakeCheckPathCaseIntegrationTests : DataLakeIntegrationTestBase
10+
{
11+
protected override string FunctionUri => $"{FunctionsAppUrl}/api/DataLake/CheckPathCase";
12+
13+
14+
}
15+
}

‎DataPipelineTools.Functions.Tests/Integration/DataLake/DataLakeGetItemsIntegrationTests.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using DataPipelineTools.Tests.Common;
2+
using NUnit.Framework;
3+
4+
namespace DataPipelineTools.Functions.Tests.Integration.DataLake
5+
{
6+
[TestFixture]
7+
[Category(nameof(TestType.IntegrationTest))]
8+
[Parallelizable(ParallelScope.Children)]
9+
public class DataLakeGetItemsIntegrationTests : DataLakeIntegrationTestBase
10+
{
11+
protected override string FunctionUri => $"{FunctionsAppUrl}/api/DataLake/GetItems";
12+
13+
14+
}
15+
}

DataPipelineTools.Functions.Tests/DataLake/DataLakeFunctions/Integration/DataLakeGetItemsIntegrationTests.cs renamed to DataPipelineTools.Functions.Tests/Integration/DataLake/DataLakeIntegrationTestBase.cs

Lines changed: 50 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,20 @@
1-
using DataPipelineTools.Tests.Common;
2-
using Microsoft.Extensions.Logging;
3-
using NUnit.Framework;
4-
using SqlCollaborative.Azure.DataPipelineTools.Functions.DataLake;
5-
using System;
1+
using System;
62
using System.Collections.Generic;
73
using System.Net;
84
using System.Threading.Tasks;
5+
using Microsoft.Extensions.Logging;
6+
using NUnit.Framework;
97
using SqlCollaborative.Azure.DataPipelineTools.DataLake.Model;
8+
using SqlCollaborative.Azure.DataPipelineTools.Functions.DataLake;
109

11-
namespace DataPipelineTools.Functions.Tests.DataLake.DataLakeFunctions.Integration
10+
namespace DataPipelineTools.Functions.Tests.Integration.DataLake
1211
{
13-
[TestFixture]
14-
[Category(nameof(TestType.IntegrationTest))]
15-
[Parallelizable(ParallelScope.Children)]
16-
public class DataLakeGetItemsIntegrationTests : IntegrationTestBase
12+
/// <summary>
13+
/// Base class for integration tests against data lake functions. The test here test the authentication, so are re-usable over
14+
/// all data lake functions.
15+
/// </summary>
16+
public abstract class DataLakeIntegrationTestBase : IntegrationTestBase
1717
{
18-
protected override string FunctionUri => $"{FunctionsAppUrl}/api/DataLake/GetItems";
19-
2018
[SetUp]
2119
public void Setup()
2220
{
@@ -35,7 +33,8 @@ public async Task Test_FunctionReturnsError_With_MissingAccountParam()
3533
{
3634
var parameters = new Dictionary<string, string>
3735
{
38-
{DataLakeConfigFactory.ContainerParam, StorageContainerName}
36+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
37+
{DataLakeConfigFactory.PathParam, "/"}
3938
};
4039
var response = await RunQueryFromParameters(parameters);
4140
LogContent(response);
@@ -52,7 +51,8 @@ public async Task Test_FunctionReturnsError_With_MissingContainerParam()
5251
{
5352
var parameters = new Dictionary<string, string>
5453
{
55-
{DataLakeConfigFactory.AccountParam, StorageAccountName}
54+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
55+
{DataLakeConfigFactory.PathParam, "/"}
5656
};
5757
var response = await RunQueryFromParameters(parameters);
5858
LogContent(response);
@@ -70,7 +70,8 @@ public async Task Test_FunctionIsRunnable_With_FunctionsServicePrincipalAuth()
7070
var parameters = new Dictionary<string, string>
7171
{
7272
{DataLakeConfigFactory.AccountParam, StorageAccountName},
73-
{DataLakeConfigFactory.ContainerParam, StorageContainerName}
73+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
74+
{DataLakeConfigFactory.PathParam, "/"}
7475
};
7576
var response = await RunQueryFromParameters(parameters);
7677
LogContent(response);
@@ -90,6 +91,7 @@ public async Task Test_FunctionIsRunnable_With_UserServicePrincipalAuthAndPlainT
9091
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
9192
{DataLakeConfigFactory.ServicePrincipalClientIdParam, ServicePrincipalName},
9293
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey},
94+
{DataLakeConfigFactory.PathParam, "/"}
9395
};
9496
var response = await RunQueryFromParameters(parameters);
9597
LogContent(response);
@@ -109,7 +111,8 @@ public async Task Test_FunctionIsRunnable_With_UserServicePrincipalAuthAndKeyVau
109111
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
110112
{DataLakeConfigFactory.ServicePrincipalClientIdParam, ServicePrincipalName},
111113
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKeyName},
112-
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName}
114+
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName},
115+
{DataLakeConfigFactory.PathParam, "/"}
113116
};
114117
var response = await RunQueryFromParameters(parameters);
115118
LogContent(response);
@@ -132,7 +135,8 @@ public async Task Test_FunctionReturnsError_With_UserServicePrincipalAuthAndEmpt
132135
{DataLakeConfigFactory.AccountParam, StorageAccountName},
133136
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
134137
{DataLakeConfigFactory.ServicePrincipalClientIdParam, clientId},
135-
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey}
138+
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey},
139+
{DataLakeConfigFactory.PathParam, "/"}
136140
};
137141
var response = await RunQueryFromParameters(parameters);
138142
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -155,7 +159,8 @@ public async Task Test_FunctionReturnsError_With_UserServicePrincipalAuthAndEmpt
155159
{DataLakeConfigFactory.AccountParam, StorageAccountName},
156160
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
157161
{DataLakeConfigFactory.ServicePrincipalClientIdParam, ServicePrincipalName},
158-
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, clientSecret}
162+
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, clientSecret},
163+
{DataLakeConfigFactory.PathParam, "/"}
159164
};
160165
var response = await RunQueryFromParameters(parameters);
161166
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -174,7 +179,8 @@ public async Task Test_FunctionReturnsError_With_UserServicePrincipalAuthAndMiss
174179
{
175180
{DataLakeConfigFactory.AccountParam, StorageAccountName},
176181
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
177-
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey}
182+
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey},
183+
{DataLakeConfigFactory.PathParam, "/"}
178184
};
179185
var response = await RunQueryFromParameters(parameters);
180186
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -187,7 +193,8 @@ public async Task Test_FunctionReturnsError_With_UserServicePrincipalAuthAndMiss
187193
{
188194
{DataLakeConfigFactory.AccountParam, StorageAccountName},
189195
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
190-
{DataLakeConfigFactory.ServicePrincipalClientIdParam, ServicePrincipalName}
196+
{DataLakeConfigFactory.ServicePrincipalClientIdParam, ServicePrincipalName},
197+
{DataLakeConfigFactory.PathParam, "/"}
191198
};
192199
var response = await RunQueryFromParameters(parameters);
193200
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -201,7 +208,8 @@ public async Task Test_FunctionReturnsError_With_InvalidUserServicePrincipalAuth
201208
{DataLakeConfigFactory.AccountParam, StorageAccountName},
202209
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
203210
{DataLakeConfigFactory.ServicePrincipalClientIdParam, Guid.NewGuid().ToString()},
204-
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey}
211+
{DataLakeConfigFactory.ServicePrincipalClientSecretParam, ServicePrincipalSecretKey},
212+
{DataLakeConfigFactory.PathParam, "/"}
205213
};
206214
var response = await RunQueryFromParameters(parameters);
207215
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -223,7 +231,8 @@ public async Task Test_FunctionIsRunnable_With_SasTokenAuthAndPlainTextKey()
223231
{
224232
{DataLakeConfigFactory.AccountParam, StorageAccountName},
225233
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
226-
{DataLakeConfigFactory.SasTokenParam, StorageContainerSasToken}
234+
{DataLakeConfigFactory.SasTokenParam, StorageContainerSasToken},
235+
{DataLakeConfigFactory.PathParam, "/"}
227236
};
228237
var response = await RunQueryFromParameters(parameters);
229238
LogContent(response);
@@ -242,7 +251,8 @@ public async Task Test_FunctionIsRunnable_With_SasTokenAuthAuthAndKeyVaultKeyNam
242251
{DataLakeConfigFactory.AccountParam, StorageAccountName},
243252
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
244253
{DataLakeConfigFactory.SasTokenParam, StorageContainerSasTokenName},
245-
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName}
254+
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName},
255+
{DataLakeConfigFactory.PathParam, "/"}
246256
};
247257
var response = await RunQueryFromParameters(parameters);
248258
LogContent(response);
@@ -263,7 +273,8 @@ public async Task Test_FunctionReturnsError_With_SasTokenAuthAndEmptySasToken(st
263273
{
264274
{DataLakeConfigFactory.AccountParam, StorageAccountName},
265275
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
266-
{DataLakeConfigFactory.SasTokenParam, sasToken}
276+
{DataLakeConfigFactory.SasTokenParam, sasToken},
277+
{DataLakeConfigFactory.PathParam, "/"}
267278
};
268279
var response = await RunQueryFromParameters(parameters);
269280
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -281,7 +292,8 @@ public async Task Test_FunctionReturnsError_With_SasTokenAuthAndInvalidSasToken(
281292
{
282293
{DataLakeConfigFactory.AccountParam, StorageAccountName},
283294
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
284-
{DataLakeConfigFactory.SasTokenParam, RandomString(20)}
295+
{DataLakeConfigFactory.SasTokenParam, RandomString(20)},
296+
{DataLakeConfigFactory.PathParam, "/"}
285297
};
286298
var response = await RunQueryFromParameters(parameters);
287299
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -302,7 +314,8 @@ public async Task Test_FunctionIsRunnable_With_AccountKeyAuthAndPlainTextKey()
302314
{
303315
{DataLakeConfigFactory.AccountParam, StorageAccountName},
304316
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
305-
{DataLakeConfigFactory.AccountKeyParam, StorageAccountAccessKey}
317+
{DataLakeConfigFactory.AccountKeyParam, StorageAccountAccessKey},
318+
{DataLakeConfigFactory.PathParam, "/"}
306319
};
307320
var response = await RunQueryFromParameters(parameters);
308321
LogContent(response);
@@ -321,7 +334,8 @@ public async Task Test_FunctionIsRunnable_With_AccountKeyAuthAuthAndKeyVaultKeyN
321334
{DataLakeConfigFactory.AccountParam, StorageAccountName},
322335
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
323336
{DataLakeConfigFactory.AccountKeyParam, StorageAccountAccessKeyName},
324-
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName}
337+
{DataLakeConfigFactory.KeyVaultParam, KeyVaultName},
338+
{DataLakeConfigFactory.PathParam, "/"}
325339
};
326340
var response = await RunQueryFromParameters(parameters);
327341
LogContent(response);
@@ -342,7 +356,8 @@ public async Task Test_FunctionReturnsError_With_AccountKeyAuthAndEmptyAccountKe
342356
{
343357
{DataLakeConfigFactory.AccountParam, StorageAccountName},
344358
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
345-
{DataLakeConfigFactory.AccountKeyParam, accountKey}
359+
{DataLakeConfigFactory.AccountKeyParam, accountKey},
360+
{DataLakeConfigFactory.PathParam, "/"}
346361
};
347362
var response = await RunQueryFromParameters(parameters);
348363
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -361,7 +376,8 @@ public async Task Test_FunctionReturnsError_With_AccountKeyAuthAndInvalidAccount
361376
{
362377
{DataLakeConfigFactory.AccountParam, StorageAccountName},
363378
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
364-
{DataLakeConfigFactory.AccountKeyParam, RandomString(20)}
379+
{DataLakeConfigFactory.AccountKeyParam, RandomString(20)},
380+
{DataLakeConfigFactory.PathParam, "/"}
365381
};
366382
var response = await RunQueryFromParameters(parameters);
367383
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -399,7 +415,7 @@ public async Task Test_FunctionReturnsError_When_MultipleAuthTypesUsed(bool useU
399415
{
400416
{DataLakeConfigFactory.AccountParam, StorageAccountName},
401417
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
402-
//{DataLakeConfigFactory.AccountKeyParam, RandomString(20)}
418+
{DataLakeConfigFactory.PathParam, "/"}
403419
};
404420
if (useUserServicePrincipal)
405421
{
@@ -450,7 +466,8 @@ public async Task Test_FunctionReturnsError_With_EmptyAccountName(string account
450466
var parameters = new Dictionary<string, string>
451467
{
452468
{DataLakeConfigFactory.AccountParam, accountName},
453-
{DataLakeConfigFactory.ContainerParam, StorageContainerName}
469+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
470+
{DataLakeConfigFactory.PathParam, "/"}
454471
};
455472
var response = await RunQueryFromParameters(parameters);
456473
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
@@ -466,7 +483,8 @@ public async Task Test_FunctionReturnsError_With_EmptyContainerName(string conta
466483
var parameters = new Dictionary<string, string>
467484
{
468485
{DataLakeConfigFactory.AccountParam, StorageAccountName},
469-
{DataLakeConfigFactory.ContainerParam, containerName}
486+
{DataLakeConfigFactory.ContainerParam, containerName},
487+
{DataLakeConfigFactory.PathParam, "/"}
470488
};
471489
var response = await RunQueryFromParameters(parameters);
472490
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);

0 commit comments

Comments
 (0)