Fixed bug where an incorrectly cased directory in a file path matched two or more directories, and threw an exception even if only a single file within those directories matched. Now all case insensitive matched directories are searched when checking a file path, and an error is only thrown if more than one file matches.

Niall Langley · Niall Langley · commit 828910efc2f2 · 2021-04-20T11:34:18.000+01:00
diff --git a/DataPipelineTools.Tests/Common/FilterFactoryTests/CreateTests.cs b/DataPipelineTools.Tests/Common/FilterFactoryTests/CreateTests.cs
@@ -129,7 +129,8 @@ public void Given_ValueDoesNotCastToNamedColumnType_Should_LogWarningOnce(string
             {nameof(TestPoco.Int64Prop), "42"},
             {nameof(TestPoco.DoubleProp), "42.1"},
             {nameof(TestPoco.DecimalProp), "42.1"},
-            { nameof(TestPoco.DateTimeProp), "2021-01-01T12:00:00"}
+            {nameof(TestPoco.DateTimeProp), "2021-01-01T12:00:00"},
+            {nameof(TestPoco.DateTimeOffsetProp), "2021-01-01T12:00:00"}
         };
         private static readonly string[] SimpleFilterTypes = {"eq", "ne", "lt", "gt", "le", "ge" };
 
diff --git a/DataPipelineTools.Tests/DataLake/DataLakeServiceTests/CheckPathAsyncTests.cs b/DataPipelineTools.Tests/DataLake/DataLakeServiceTests/CheckPathAsyncTests.cs
@@ -36,13 +36,14 @@ public void Given_ValidDirectoryPath_Should_ReturnDirectoryPath()
             Assert.That(resultPath, Is.EqualTo(testPath));
         }
 
-        [Test]
-        public void Given_DirectoryPathWithIncorrectCase_Should_ReturnCorrectedDirectoryPath()
+        [TestCase("raw/database/JAN")]
+        [TestCase("raw/DataBase/jan")]
+        [TestCase("RAW/database/jan")]
+        public void Given_DirectoryPathWithIncorrectCase_Should_ReturnCorrectedDirectoryPath(string testPath)
         {
-            var testPath = "raw/DATABASE";
             var resultPath = Sut.CheckPathAsync(testPath, true).Result;
 
-            Assert.That(resultPath, Is.EqualTo("raw/database"));
+            Assert.That(resultPath, Is.EqualTo("raw/database/jan"));
         }
 
         [Test]
@@ -64,10 +65,11 @@ public void Given_FilePathWithIncorrectFilenameCase_Should_ReturnCorrectedFilePa
         }
 
 
-        [Test]
-        public void Given_FilePathWithIncorrectDirectoryCase_Should_ReturnCorrectedFilePath()
+        [TestCase("raw/database/JAN/extract_1.csv")]
+        [TestCase("raw/DataBase/jan/extract_1.csv")]
+        [TestCase("RAW/database/jan/extract_1.csv")]
+        public void Given_FilePathWithIncorrectDirectoryCase_Should_ReturnCorrectedFilePath(string testPath)
         {
-            var testPath = "raw/database/JAN/extract_1.csv";
             var resultPath = Sut.CheckPathAsync(testPath, false).Result;
 
             Assert.That(resultPath, Is.EqualTo("raw/database/jan/extract_1.csv"));
@@ -109,11 +111,10 @@ public void Given_InvalidFilePath_Should_ReturnNull()
             Assert.That(resultPath, Is.EqualTo(null));
         }
 
-        [Test]
-        public void Given_DirectoryPathWithIncorrectCase_When_MatchesMultiplePaths_Should_Throw()
+        [TestCase("raw/aPi")]
+        [TestCase("RAW/api")]
+        public void Given_DirectoryPathWithIncorrectCase_When_MatchesMultiplePaths_Should_Throw(string testPath)
         {
-            var testPath = "raw/aPi";
-            
             Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, true));
         }
         
@@ -153,29 +154,23 @@ public void Given_ForwardSlashPath_Should_Return_EmptyString()
             Assert.That(resultPath, Is.EqualTo(string.Empty));
         }
 
-        [Test]
-        public void Given_DirectoryPathWithIncorrectCase_Should_ThrowWhenMultipleDirectoriesMatch()
+        [TestCase("RaW/api/jan")]
+        [TestCase("raw/ApI/jan")]
+        [TestCase("raw/api/JaN")]
+        public void Given_DirectoryPathWithIncorrectCase_Should_ThrowWhenMultipleDirectoriesMatch(string testPath)
         {
-            var testPath = "RaW/api/jan";
-
             var exception = Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, true));
             Assert.That(exception, Is.TypeOf(typeof(Exception)));
         }
 
-        [Test]
-        public void Given_FilePathWithIncorrectCase_Should_ThrowWhenMultipleDirectoriesMatch()
+        [TestCase("RaW/api/jan/delta_extract_1.json")]
+        [TestCase("raw/ApI/jan/delta_extract_1.json")]
+        [TestCase("raw/api/JaN/delta_extract_1.json")]
+        [TestCase("raw/api/jan/delta_EXTRACT_1.json")]
+        [TestCase("raw/DataBase/feb/extract_2.csv")]
+        [TestCase("raw/database/feb/Extract_2.csv")]
+        public void Given_PathWithIncorrectCase_Should_ThrowWhenMultipleFilesMatch(string testPath)
         {
-            var testPath = "RaW/api/jan/delta_extract_1.json";
-
-            var exception = Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, false));
-            Assert.That(exception, Is.TypeOf(typeof(Exception)));
-        }
-
-        [Test]
-        public void Given_FilePathWithIncorrectCase_Should_ThrowWhenMultipleFilesMatch()
-        {
-            var testPath = "raw/database/feb/Extract_2.csv";
-
             var exception = Assert.CatchAsync(() => Sut.CheckPathAsync(testPath, false));
             Assert.That(exception, Is.TypeOf(typeof(Exception)));
         }
diff --git a/DataPipelineTools.Tests/DataLake/DataLakeServiceTests/GetItemsAsyncTests.cs b/DataPipelineTools.Tests/DataLake/DataLakeServiceTests/GetItemsAsyncTests.cs
@@ -54,24 +54,11 @@ public void Given_DirectoryPathWithIncorrectCase_Should_ReturnContentsForCorrect
                 Directory = "raw/aPi/feb"
             };
 
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
-            var r = Sut.CheckPathAsync("raw/Api/jan/delta_extract_1.json", false).Result;
-
-            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result;
-
-            Assert.That(result.ContainsKey("fileCount"), Is.True);
-            Assert.That(result.ContainsKey("files"), Is.True);
-
-            Assert.That( (int)result.Property("fileCount").Value, Is.EqualTo(2) );
-
-            var s = result.ToObject<GetItemsResponse>();
-            var responseObject = JsonConvert.DeserializeObject<GetItemsResponse>(result.ToString());
-
-            var files = result.Property("files");
-            var itemsInfo = JsonConvert.DeserializeObject<List<DataLakeItem>>(files.ToString());
-            Assert.That(itemsInfo, Is.EqualTo(2));
-            //Assert.That(itemsInfo.Count(x => x.FullPath == "raw/api/feb"), Is.True);
-            //Assert.That(itemsInfo.Count(x => x.FullPath == "raw/api/feb/delta_extract_3.json"), Is.True);
+            Assert.That(result.fileCount, Is.EqualTo(1));
+            Assert.That(result.files.Count, Is.EqualTo(1));
+            Assert.That(result.files.Count(x => x.FullPath == "raw/api/feb/delta_extract_3.json"), Is.EqualTo(1));
         }
 
         //[Test]
diff --git a/DataPipelineTools.Tests/DataLake/DataLakeTestBase.cs b/DataPipelineTools.Tests/DataLake/DataLakeTestBase.cs
@@ -54,7 +54,7 @@ protected Mock<DataLakeFileSystemClient> BuildMockDataLakeFileSystemClient()
                     var pathLength = 1 + (path?.Length ?? 0);
                     var items = TestData
                         // Include all files starting with the test path, or root paths if the test path is null
-                        .Where(x => x.Name.StartsWith(path ?? string.Empty))// || (path == null && !x.Name.Contains('/')))
+                        .Where(x => x.Name.StartsWith(path ?? string.Empty) && x.Name != path)
                         // Still include them if the recursive flag is set, otherwise check if the relative path after the search path contains 
                         // directory separator to exclude sub dirs
                         .Where(x => recursive || !x.Name.Substring(pathLength > x.Name.Length ? x.Name.Length : pathLength).Contains('/'))
diff --git a/DataPipelineTools.Tests/TestPoco.cs b/DataPipelineTools.Tests/TestPoco.cs
@@ -12,6 +12,7 @@ public class TestPoco
         public decimal DecimalProp { get; set; }
         public double DoubleProp { get; set; }
         public DateTime DateTimeProp { get; set; }
+        public DateTimeOffset DateTimeOffsetProp { get; set; }
         public object ObjectProp { get; set; }
     }
 }
diff --git a/DataPipelineTools/DataLake/DataLakeService.cs b/DataPipelineTools/DataLake/DataLakeService.cs
@@ -103,6 +103,8 @@ private IList<string> MatchPathItemsCaseInsensitive(string basePath, string sear
 
         public async Task<JObject> GetItemsAsync(DataLakeConfig dataLakeConfig, DataLakeGetItemsConfig getItemsConfig)
         {
+            // Check the directory exists. If multiple directories match (ie different casing), it will throw an error, as we don't know
+            // which one we wanted the files from.
             var directory = getItemsConfig.IgnoreDirectoryCase ?
                                 await CheckPathAsync(getItemsConfig.Directory, true) :
                                 getItemsConfig.Directory;

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ public class TestPoco`
`12`	`12`	`public decimal DecimalProp { get; set; }`
`13`	`13`	`public double DoubleProp { get; set; }`
`14`	`14`	`public DateTime DateTimeProp { get; set; }`
	`15`	`+ public DateTimeOffset DateTimeOffsetProp { get; set; }`
`15`	`16`	`public object ObjectProp { get; set; }`
`16`	`17`	`}`
`17`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -103,6 +103,8 @@ private IList<string> MatchPathItemsCaseInsensitive(string basePath, string sear`
`103`	`103`
`104`	`104`	`public async Task<JObject> GetItemsAsync(DataLakeConfig dataLakeConfig, DataLakeGetItemsConfig getItemsConfig)`
`105`	`105`	`{`
	`106`	`+ // Check the directory exists. If multiple directories match (ie different casing), it will throw an error, as we don't know`
	`107`	`+ // which one we wanted the files from.`
`106`	`108`	`var directory = getItemsConfig.IgnoreDirectoryCase ?`
`107`	`109`	`await CheckPathAsync(getItemsConfig.Directory, true) :`
`108`	`110`	`getItemsConfig.Directory;`