Skip to content

Commit 96c04a6

Browse files
NJLangleyNJLangley
committed
Added more integration tests for DataLake/CheckPathCase and DataLake/GetItems functions
1 parent 29688fb commit 96c04a6

File tree

17 files changed

+1636
-102
lines changed

17 files changed

+1636
-102
lines changed

‎.github/workflows/test_azure_devtest_labs_integration.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,17 @@ jobs:
119119
120120
STORAGE_ACCOUNT_NAME="${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNT_NAME }}"
121121
STORAGE_CONTAINER_NAME="${{ steps.create-devtest-labs-environment.outputs.STORAGE_CONTAINER_NAME }}"
122-
# The exclude patter for the .keepDirectory files is not working, so workaround below if to find these files and remove them from ADLS
123-
azcopy copy "./DataPipelineTools.Functions.Tests/TestData" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME" --recursive=true --exclude-pattern=".keepDirectory"
122+
123+
# The exclude pattern for the .keepDirectory files is not working, so workaround below if to find these files and remove them from ADLS
124+
echo "Copying TestData folder into root of ADLS container"
125+
azcopy copy "./DataPipelineTools.Functions.Tests/TestData" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME" --recursive=true #--exclude-pattern=".keepFolder"
126+
127+
# Duplicate one file and one folder with a difference case, so that we can test the function CheckPathCase
128+
echo "Copying TestData/TestFolderCaseDupes/TestDoc3.txt file into ADLS container with different casing on folder structure"
129+
azcopy copy "./DataPipelineTools.Functions.Tests/TestData/TestFolderCaseDupes/TestDoc3.txt" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME/TESTDATA/TestFolderCaseDupes/TestDoc3.txt"
130+
131+
echo "Copying TestData/TestFolderCaseDupes/TestDoc4.txt file into ADLS container with different casing on name"
132+
azcopy copy "./DataPipelineTools.Functions.Tests/TestData/TestFolderCaseDupes/TestDoc4.txt" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME/TestData/TestFolderCaseDupes/TESTDOC4.txt"
124133
125134
# Remove the .keepDirectory files, these are just used to allow us to add empty folders into the sample data in Git
126135
# See here for more info: https://github.com/Azure/azure-storage-azcopy/issues/796

‎DataPipelineTools.Functions.Tests/Integration/DataLake/DataLakeCheckPathCaseIntegrationTests.cs‎

Lines changed: 170 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
using System.Collections.Generic;
2+
using System.Net;
3+
using System.Threading.Tasks;
14
using DataPipelineTools.Tests.Common;
25
using NUnit.Framework;
6+
using SqlCollaborative.Azure.DataPipelineTools.DataLake;
7+
using SqlCollaborative.Azure.DataPipelineTools.Functions.DataLake;
38

49
namespace DataPipelineTools.Functions.Tests.Integration.DataLake
510
{
@@ -8,8 +13,172 @@ namespace DataPipelineTools.Functions.Tests.Integration.DataLake
813
[Parallelizable(ParallelScope.Children)]
914
public class DataLakeCheckPathCaseIntegrationTests : DataLakeIntegrationTestBase
1015
{
11-
protected override string FunctionUri => $"{FunctionsAppUrl}/api/DataLake/CheckPathCase";
16+
protected override string FunctionRelativeUri => "DataLake/CheckPathCase";
1217

1318

19+
[Test]
20+
// Test all combos of leading slashes
21+
[TestCase("TestData/TestFolder1/TestDoc1.txt")]
22+
[TestCase("/TestData/TestFolder1/TestDoc1.txt")]
23+
public async Task Given_ValidFilePath_Should_ReturnSamePath(string testPath)
24+
{
25+
var parameters = new Dictionary<string, string>
26+
{
27+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
28+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
29+
{DataLakeConfigFactory.PathParam, testPath}
30+
};
31+
var response = await RunQueryFromParameters(parameters);
32+
LogContent(response);
33+
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
34+
35+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
36+
dynamic results = GetResultsObject(response);
37+
38+
39+
Assert.IsNull(results.error);
40+
Assert.AreEqual(testPath.Trim('/'), results.validatedPath?.ToString());
41+
}
42+
43+
[Test]
44+
// Test all combos of leading/trailing slashes
45+
[TestCase("TestData/TestFolder1/")]
46+
[TestCase("TestData/TestFolder1")]
47+
[TestCase("/TestData/TestFolder1/")]
48+
[TestCase("/TestData/TestFolder1")]
49+
public async Task Given_ValidFolderPath_Should_ReturnSamePath(string testPath)
50+
{
51+
var parameters = new Dictionary<string, string>
52+
{
53+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
54+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
55+
{DataLakeConfigFactory.PathParam, testPath}
56+
};
57+
var response = await RunQueryFromParameters(parameters);
58+
LogContent(response);
59+
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
60+
61+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
62+
dynamic results = GetResultsObject(response);
63+
64+
65+
Assert.IsNull(results.error);
66+
Assert.AreEqual(testPath.Trim('/'), results.validatedPath?.ToString());
67+
}
68+
69+
[Test]
70+
public async Task Given_NonExistentPath_Should_ReturnBadRequest()
71+
{
72+
var parameters = new Dictionary<string, string>
73+
{
74+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
75+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
76+
{DataLakeConfigFactory.PathParam, "/SomeMadeUpPath"}
77+
};
78+
var response = await RunQueryFromParameters(parameters);
79+
LogContent(response);
80+
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
81+
82+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
83+
dynamic results = GetResultsObject(response);
84+
85+
86+
Assert.AreEqual(DataLakeFunctions.PathNotFoundErrorMessage, results.error?.ToString());
87+
Assert.IsNull(results.validatedPath?.ToString());
88+
}
89+
90+
[Test]
91+
[TestCase("TESTDATA/TestFolder1/")]
92+
[TestCase("TestData/TESTFOLDER1")]
93+
[TestCase("/TESTDATA/TestFolder1/")]
94+
[TestCase("/TestData/TESTFOLDER1")]
95+
public async Task Given_FolderPath_With_WrongCaseAndSingleMatch_Should_ReturnValidPath(string testPath)
96+
{
97+
var parameters = new Dictionary<string, string>
98+
{
99+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
100+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
101+
{DataLakeConfigFactory.PathParam, testPath}
102+
};
103+
var response = await RunQueryFromParameters(parameters);
104+
LogContent(response);
105+
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
106+
107+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
108+
dynamic results = GetResultsObject(response);
109+
110+
Assert.AreEqual("TestData/TestFolder1", results.validatedPath?.ToString());
111+
}
112+
113+
[Test]
114+
[TestCase("TESTDATA/TestFolder1/TestDoc1.txt")]
115+
[TestCase("TestData/TESTFOLDER1/TestDoc1.txt")]
116+
[TestCase("TestData/TestFolder1/TESTDOC1.txt")]
117+
[TestCase("/TESTDATA/TestFolder1/TestDoc1.txt")]
118+
[TestCase("/TestData/TESTFOLDER1/TestDoc1.txt")]
119+
[TestCase("/TestData/TestFolder1/TESTDOC1.txt")]
120+
public async Task Given_FilePath_With_WrongCaseAndSingleMatch_Should_ReturnValidPath(string testPath)
121+
{
122+
var parameters = new Dictionary<string, string>
123+
{
124+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
125+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
126+
{DataLakeConfigFactory.PathParam, testPath}
127+
};
128+
var response = await RunQueryFromParameters(parameters);
129+
LogContent(response);
130+
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
131+
132+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
133+
dynamic results = GetResultsObject(response);
134+
135+
Assert.AreEqual("TestData/TestFolder1/TestDoc1.txt", results.validatedPath?.ToString());
136+
}
137+
138+
[Test]
139+
[TestCase("TestData/TESTFOLDERCASEDUPES/TestDoc3.txt")]
140+
[TestCase("TestData/TESTFOLDERCASEDUPES/TestDoc4.txt")]
141+
public async Task Given_FilePath_With_WrongCaseAndMultipleMatches_Should_ReturnError(string testPath)
142+
{
143+
var parameters = new Dictionary<string, string>
144+
{
145+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
146+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
147+
{DataLakeConfigFactory.PathParam, testPath}
148+
};
149+
var response = await RunQueryFromParameters(parameters);
150+
LogContent(response);
151+
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
152+
153+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
154+
dynamic results = GetResultsObject(response);
155+
156+
Assert.AreEqual(DataLakeService.ErrorMessage.MultipleFileMatchesWithCaseInsensitiveCompare, results.error?.ToString());
157+
}
158+
159+
[Test]
160+
[TestCase("TestData/TESTFOLDERCASEDUPES")]
161+
[TestCase("TestData/TESTFOLDERCASEDUPES/")]
162+
[TestCase("/TestData/TESTFOLDERCASEDUPES")]
163+
[TestCase("testdata/TestFolderCaseDupes")]
164+
[TestCase("testdata/TestFolderCaseDupes/")]
165+
[TestCase("/testdata/TestFolderCaseDupes")]
166+
public async Task Given_FolderPath_With_WrongCaseAndMultipleMatches_Should_ReturnError(string testPath)
167+
{
168+
var parameters = new Dictionary<string, string>
169+
{
170+
{DataLakeConfigFactory.AccountParam, StorageAccountName},
171+
{DataLakeConfigFactory.ContainerParam, StorageContainerName},
172+
{DataLakeConfigFactory.PathParam, testPath}
173+
};
174+
var response = await RunQueryFromParameters(parameters);
175+
LogContent(response);
176+
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
177+
178+
// Check response details. Its important to cast the actual or we test against JToken from the dynamic results
179+
dynamic results = GetResultsObject(response);
180+
181+
Assert.AreEqual(DataLakeService.ErrorMessage.MultipleDirectoryMatchesWithCaseInsensitiveCompare, results.error?.ToString());
182+
}
14183
}
15184
}

0 commit comments

Comments
 (0)