Added more tests and refactored some existing ones to remove linq queries on collections to make them more readable

Author: Niall Langley

DataPipelineTools.Tests/DataLake/DataLakeServiceTests/CheckPathAsyncTests.cs

@@ -14,7 +14,7 @@ public class CheckPathAsyncTests: DataLakeTestBase
         public CheckPathAsyncTests()
         {
             // Use the factory to inject the mock logger to get the mock client...
-            var factory = new DataLakeServiceFactory(MockLogger.Object);
+            var factory = new DataLakeServiceFactory(Logger);
             Sut = factory.CreateDataLakeService(MockFileSystemClient.Object);
         }
 

DataPipelineTools.Tests/DataLake/DataLakeServiceTests/GetItemsAsyncTests.cs

@@ -21,7 +21,7 @@ public class GetItemsAsyncTests : DataLakeTestBase
         public GetItemsAsyncTests()
         {
             // Use the factory to inject the mock logger to get the mock client...
-            var factory = new DataLakeServiceFactory(MockLogger.Object);
+            var factory = new DataLakeServiceFactory(Logger);
             Sut = factory.CreateDataLakeService(MockFileSystemClient.Object);
         }
 
@@ -35,7 +35,7 @@ public void Setup()
         }
 
         [Test]
-        public void Given_ValidDirectoryPath_Should_ReturnContents()
+        public void Given_ValidDirectoryPath_Should_ReturnContentsAsJson()
         {
             var itemsConfig = new DataLakeGetItemsConfig
             {
@@ -60,7 +60,7 @@ public void Given_DirectoryPathWithIncorrectCase_Should_ReturnContentsForCorrect
 
             Assert.That(result.FileCount, Is.EqualTo(1));
             Assert.That(result.Files.Count, Is.EqualTo(1));
-            Assert.That(result.Files.Count(x => x.FullPath == "raw/api/feb/delta_extract_3.json"), Is.EqualTo(1));
+            Assert.That(result.Files, Has.All.Property(nameof(DataLakeItem.FullPath)).EqualTo("raw/api/feb/delta_extract_3.json"));
         }
 
 
@@ -101,6 +101,7 @@ public void Given_LimitNRecords_Should_ReturnNRecords(DataLakeGetItemsConfig ite
             var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
             Assert.That(result.FileCount, Is.EqualTo(expectedResultCount));
+            Assert.That(result.Files, Has.Count.EqualTo(expectedResultCount));
         }
 
 
@@ -113,11 +114,13 @@ public void Given_LimitNRecords_Should_ReturnNRecords(DataLakeGetItemsConfig ite
         [TestCaseSource(nameof(OrderByColumn))]
         public void Given_OrderBy_Should_ReturnRecordsOrderedBySpecifiedColumnWithDirectionSpecifiedByOrderByDescendingFlag(DataLakeGetItemsConfig itemsConfig, int[] expectedContentLengths)
         {
+            var expectedFileCount = expectedContentLengths.Length;
+
             var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
-            Assert.That(result.FileCount, Is.EqualTo(expectedContentLengths.Length));
-            for (int i = 0; i < 5; i++)
-                Assert.That(result.Files[i].ContentLength, Is.EqualTo(expectedContentLengths[i]));
+            Assert.That(result.FileCount, Is.EqualTo(expectedFileCount));
+            Assert.That(result.Files, Has.Count.EqualTo(expectedFileCount));
+            Assert.That(result.Files.Select(item => item.ContentLength), Is.EquivalentTo(expectedContentLengths));
         }
 
 
@@ -130,24 +133,24 @@ public void Given_OrderBy_Should_ReturnRecordsOrderedBySpecifiedColumnWithDirect
         [TestCaseSource(nameof(LimitNRecordsAndOrderByColumn))]
         public void Given_LimitNRecordAndOrderBy_Should_ReturnTopNRecordsOrderedBySpecifiedColumnWithDirectionSpecifiedByOrderByDescendingFlag(DataLakeGetItemsConfig itemsConfig, int[] expectedContentLengths)
         {
-            itemsConfig.Limit = expectedContentLengths.Length;
+            var expectedFileCount = expectedContentLengths.Length;
+            itemsConfig.Limit = expectedFileCount;
 
             var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
-            Assert.That(result.FileCount, Is.EqualTo(expectedContentLengths.Length));
-            for (int i = 0; i < expectedContentLengths.Length; i++)
-                Assert.That(result.Files[i].ContentLength, Is.EqualTo(expectedContentLengths[i]));
+            Assert.That(result.FileCount, Is.EqualTo(expectedFileCount));
+            Assert.That(result.Files, Has.Count.EqualTo(expectedFileCount));
+            Assert.That(result.Files.Select(item => item.ContentLength), Is.EquivalentTo(expectedContentLengths));
         }
 
-
-
         [Test]
         public void Given_RecursiveFlagIsTrue_Should_ReturnContentRecursively()
         {
             var itemsConfig = new DataLakeGetItemsConfig {Directory = "raw/database", Recursive = true };
             var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
             Assert.That(result.FileCount, Is.EqualTo(5));
+            Assert.That(result.Files, Has.Count.EqualTo(5));
         }
 
         [Test]
@@ -157,22 +160,154 @@ public void Given_RecursiveFlagIsFalse_Should_ReturnDirectoryContentsOnly()
             var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
             Assert.That(result.FileCount, Is.EqualTo(2));
-            Assert.That(result.Files.All(x => x.Directory == itemsConfig.Directory), Is.True);
+            Assert.That(result.Files, Has.Count.EqualTo(2));
+            Assert.That(result.Files, Has.All.Property(nameof(DataLakeItem.Directory)).EqualTo(itemsConfig.Directory));
+        }
+
+        [Test]
+        public void Given_DirectoryPathWithIncorrectCaseAndIgnoreDirectoryCaseIsFalse_Should_Throw()
+        {
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw/DATABASE", IgnoreDirectoryCase = false };
+            Assert.CatchAsync(() => Sut.GetItemsAsync(DatalakeConfig, itemsConfig));
+        }
+
+        [Test]
+        public void Given_DirectoryPathWithIncorrectCaseAndIgnoreDirectoryCaseTrue_Should_ReturnContents()
+        {
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw/DATABASE", IgnoreDirectoryCase = true };
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
+
+            Assert.That(result.FileCount, Is.GreaterThan(0));
+            Assert.That(result.Files, Has.Count.GreaterThan(0));
+        }
+
+        [Test]
+        public void Given_DirectoryPathWithIncorrectCaseAndIgnoreDirectoryCaseIsTrue_WhenMatchesOneDirectoryPath_Should_ReturnCorrectedPath()
+        {
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw/DATABASE", IgnoreDirectoryCase = true };
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
+
+            Assert.That(result.CorrectedFilePath, Is.EqualTo("raw/database"));
         }
 
+        [Test]
+        public void Given_DirectoryPathWithIncorrectCaseAndIgnoreDirectoryCaseIsTrue_WhenMatcheMultipleDirectoryPaths_Should_Throw()
+        {
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw/ApI", IgnoreDirectoryCase = true };
+            Assert.CatchAsync(() => Sut.GetItemsAsync(DatalakeConfig, itemsConfig));
+        }
 
+        
+
+        private static object[][] Filters = 
+        {
+            new object[] { "raw/api/jan", nameof(DataLakeItem.Name), "eq:delta_extract_4.json", 1},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.Name), "ne:delta_extract_4.json", 4},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.ContentLength), "gt:40", 1},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.ContentLength), "ge:40", 2},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.ContentLength), "lt:20", 1},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.ContentLength), "le:20", 2},
+            new object[] { "raw/api", nameof(DataLakeItem.Directory), "like:*feb*", 1},
+            new object[] { "raw/api", nameof(DataLakeItem.FullPath), "like:delta_extract_[3-5].json", 4},
+            new object[] { "raw/api", nameof(DataLakeItem.Url), "like:.+raw\\/api.+", 8}, // 6 files + 2 directories
+            new object[] { "raw/api", nameof(DataLakeItem.IsDirectory), "eq:true", 2},
+            new object[] { "raw/api", nameof(DataLakeItem.IsDirectory), "eq:false", 6},
+            new object[] { "raw/api", nameof(DataLakeItem.LastModified), "ge:2021-01-04T14:00:00", 2},
+        };
+        [TestCaseSource(nameof(Filters))]
+        public void Given_Filter_Should_ReturnRecordsMatchingFilter(string directory, string filterProperty, string filterExpression, int expectedFileCount)
+        {
+            // Have to build the filter here rather than pass it in using TestCaseSource as the logger is not static
+            var filter = FilterFactory<DataLakeItem>.Create(filterProperty, filterExpression, Logger);
+            var itemsConfig = new DataLakeGetItemsConfig {Directory = directory, Filters = new[] {filter}};
+            
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
+            
+            Assert.That(result.FileCount, Is.EqualTo(expectedFileCount));
+            Assert.That(result.Files, Has.Count.EqualTo(expectedFileCount));
+        }
+
+        private static object[][] InvalidFilters =
+        {
+            new object[] { "raw/api/jan", nameof(DataLakeItem.Name), "neq:delta_extract_4.json"}, // Invalid operator
+            new object[] { "raw/api/jan", nameof(DataLakeItem.ContentLength), "eq:delta_extract_4.json"}, // Invalid value type
+            new object[] { "raw/api/jan", "Some invalid property", "ne:delta_extract_4.json"},
+            new object[] { "raw/api/jan", string.Empty, "ne:delta_extract_4.json"},
+            new object[] { "raw/api/jan", null, "ne:delta_extract_4.json"},
+
+            new object[] { "raw/api/jan", nameof(DataLakeItem.Name), string.Empty},
+            new object[] { "raw/api/jan", nameof(DataLakeItem.Name), null},
+
+        };
+        [TestCaseSource(nameof(InvalidFilters))]
+        public void Given_InvalidFilter_Should_Throw(string directory, string filterProperty, string filterExpression)
+        {
+            // Have to build the filter here rather than pass it in using TestCaseSource as the logger is not static
+            var filter = FilterFactory<DataLakeItem>.Create(filterProperty, filterExpression, Logger);
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = directory, Filters = new[] { filter } };
+
+            Assert.CatchAsync(() => Sut.GetItemsAsync(DatalakeConfig, itemsConfig));
+        }
+
+        [TestCaseSource(nameof(InvalidFilters))]
+        public void Given_MultipleFilters_WhenSomeAreInvalid_Should_Throw(string directory, string filterProperty, string filterExpression)
+        {
+            var filters = new[]
+            {
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.FullPath), "like:*jan*", Logger),
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.IsDirectory), "eq:5", Logger),
+            };
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw", Filters = filters };
+
+            Assert.CatchAsync(() => Sut.GetItemsAsync(DatalakeConfig, itemsConfig));
+        }
+
+        [Test]
+        public void Given_MultipleValidFiltersOfDifferentTypes_Should_ReturnFilteredResults()
+        {
+            var filters = new[]
+            {
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.FullPath), "like:*jan*", Logger),
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.IsDirectory), "eq:true", Logger),
+            };
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw", Filters = filters};
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
+
+            Assert.That(result.FileCount, Is.EqualTo(3));
+            Assert.That(result.Files, Has.Count.EqualTo(3));
+
+            Assert.That(result.Files, Has.All.Property(nameof(DataLakeItem.IsDirectory)).True);
+        }
+
+
+        [Test]
+        public void Given_MultipleValidFiltersOfSameType_Should_ReturnFilteredResults()
+        {
+            var filters = new[]
+            {
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.FullPath), "like:*jan*", Logger),
+                FilterFactory<DataLakeItem>.Create(nameof(DataLakeItem.FullPath), "like:*delta", Logger),
+            };
+            var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw", Filters = filters };
+            var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
+
+            Assert.That(result.FileCount, Is.EqualTo(6));
+            Assert.That(result.Files, Has.Count.EqualTo(6));
+
+            Assert.That(result.Files, Has.All.Property(nameof(DataLakeItem.FullPath)).Contains("jan"));
+            Assert.That(result.Files, Has.All.Property(nameof(DataLakeItem.FullPath)).Contains("delta"));
+        }
 
 
-        //Filters = new[] {FilterFactory<DataLakeItem>.Create("Name", "like:*1*", MockLogger.Object)}
         //[Test]
-        //public void Given_ValidDirectoryPath_Should_ReturnDirectoryPath()
+        //public void Given__Should_Return()
         //{
-        //    var testPath = "raw/database";
-        //    var resultPath = Sut.CheckPathAsync(testPath, true).Result;
+        //    var itemsConfig = new DataLakeGetItemsConfig { Directory = "raw/database", Recursive = false };
+        //    var result = Sut.GetItemsAsync(DatalakeConfig, itemsConfig).Result.ToObject<GetItemsResponse>();
 
-        //    Assert.That(resultPath, Is.EqualTo(testPath));
+        //    Assert.That(result.FileCount, Is.EqualTo(2));
+        //    Assert.That(result.Files.All(x => x.Directory == itemsConfig.Directory), Is.True);
         //}
-
     }
 
     public class GetItemsResponse