Refactoring to add new auth types for data lake

Author: NJLangley

Azure-DevTestLab/Environments/sqlcollaborative_AzureDataPipelineTools/azuredeploy.json

@@ -521,6 +521,10 @@
                         {
                             "name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
                             "value": "[reference(resourceId('microsoft.insights/components', variables('applicationInsightsName')), variables('applicationInsightsApiVersion')).ConnectionString]"
+                        },
+                        {
+                            "name": "TENANT_ID",
+                            "value": "[parameters('devopsServicePrincipalCredentials').tenant_id]"
                         }
                     ]
                 }

DataPipelineTools.Functions.Tests/DataLake/DataLakeFunctions/Integration/DataLakeGetItemsIntegrationTests.cs

@@ -13,13 +13,15 @@ namespace DataPipelineTools.Functions.Tests.DataLake.DataLakeFunctions.Integrati
     [Category(nameof(TestType.IntegrationTest))]
     public class DataLakeGetItemsIntegrationTests: IntegrationTestBase
     {
-        protected string FunctionUri => $"{FunctionsAppUrl}/api/DataLakeGetItems";
+        protected override string FunctionUri => $"{FunctionsAppUrl}/api/DataLakeGetItems";
 
         [SetUp]
         public void Setup()
         {
             Logger.LogInformation($"Running tests in { (IsRunningOnCIServer ? "CI" : "local") } environment using Functions App '{FunctionsAppUrl}'");
             Logger.LogInformation($"TestContext.Parameters.Count: { TestContext.Parameters.Count }");
+
+            Logger.LogInformation($"Key Vault Name: { KeyVaultName }");
         }
 
         [Test]
@@ -28,7 +30,7 @@ public async Task Test_FunctionIsRunnable()
             using (var client = new HttpClient())
             {
                 var queryParams = HttpUtility.ParseQueryString(string.Empty);
-                queryParams["AccountUri"] = this.StorageAccountName;
+                queryParams["account"] = this.StorageAccountName;
                 queryParams["container"] = this.StorageContainerName;
 
                 if (!IsEmulatorRunning)

DataPipelineTools.Functions.Tests/IntegrationTestBase.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -7,7 +8,8 @@
 using Azure.Identity;
 using Azure.Security.KeyVault.Secrets;
 using DataPipelineTools.Tests.Common;
-using Microsoft.VisualBasic.FileIO;
+using Flurl.Util;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using NUnit.Framework;
 using SearchOption = System.IO.SearchOption;
 
@@ -43,6 +45,8 @@ protected bool UseFunctionsEmulator
         protected string ServicePrincipalName => TestContext.Parameters["ServicePrincipalName"];
         protected string ApplicationInsightsName => TestContext.Parameters["ApplicationInsightsName"];
 
+        protected abstract string FunctionUri { get; }
+
 
         // The properties that we get from Azure Key Vault are cached for reuse
         private string _functionsAppKey;
@@ -151,6 +155,39 @@ protected bool IsRunningOnCIServer
         }
 
         protected string GetKeyVaultSecretValue(string secretName)
+        {
+            var client = GetKeyVaultClient();
+
+            try
+            {
+                var result = client.GetSecretAsync(secretName).Result;
+
+                return result?.Value?.Value;
+            }
+            catch (Exception ex)
+            {
+                throw new SettingsException(
+                    $"The key vault {KeyVaultName} is inaccessible or has been deleted. Check your run settings file.\n\nInner Exception Message:\n  {ex.Message.Split('\n').First()}");
+            }
+        }
+
+
+        protected IEnumerable<string> GetKeyVaultSecretNames()
+        {
+            var client = GetKeyVaultClient();
+            try
+            {
+                var results = client.GetPropertiesOfSecrets();
+                return results.Select(x => x.Name);
+            }
+            catch (Exception ex)
+            {
+                throw new SettingsException(
+                    $"The key vault {KeyVaultName} is inaccessible or has been deleted. Check your run settings file.\n\nInner Exception Message:\n  {ex.Message.Split('\n').First()}");
+            }
+        }
+
+        private SecretClient GetKeyVaultClient()
         {
             /* For some reason the DefaultAzureCredential (SharedTokenCacheCredential / VisualStudioCredential) returns a 403 trying to access the key vault, even when access policies are configured correctly
              * We either use one of the following to authenticate:
@@ -159,25 +196,21 @@ protected string GetKeyVaultSecretValue(string secretName)
              *
              * See here for more info: https://docs.microsoft.com/en-us/answers/questions/74848/access-denied-to-first-party-service.html
              */
-            var credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ExcludeSharedTokenCacheCredential = true, ExcludeVisualStudioCredential = true });
+            var credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions
+                {ExcludeSharedTokenCacheCredential = true, ExcludeVisualStudioCredential = true});
 
             if (string.IsNullOrWhiteSpace(KeyVaultName))
                 throw new ArgumentException("The run setting file does not have a value for 'KeyVaultName'");
 
             var keyVaultUri = $"https://{KeyVaultName}.vault.azure.net";
-            var client = new SecretClient(new Uri(keyVaultUri), credential);
-            var result = client.GetSecretAsync(secretName).Result;
+            return new SecretClient(new Uri(keyVaultUri), credential);
 
-            return result?.Value?.Value;
         }
 
 
 
 
 
-
-
-
         #region Azure Functions Local Host
         // We use one time setup and teardown to generate a single instance of the emulator across all classes that implement this base class
 
@@ -186,6 +219,13 @@ protected string GetKeyVaultSecretValue(string secretName)
         [OneTimeSetUp]
         public void StartFunctionsEmulator()
         {
+            // If the run settings is referencing secrets via key vault, make sure we can connect
+            if (TestContext.Parameters.Names.Any(x => x.StartsWith("KeyVaultSecret") && !string.IsNullOrWhiteSpace(TestContext.Parameters[x].ToString())))
+                GetKeyVaultSecretNames();
+            
+
+            // Start the local functions emulator if required. Use a lock so that multiple test classes inheriting from this base class share a
+            // functions emulator instance
             lock (_functionsProcessLock)
             {
                 if (UseFunctionsEmulator)
@@ -201,6 +241,7 @@ public void StartFunctionsEmulator()
         [OneTimeTearDown]
         public void StopFunctionsEmulator()
         {
+            // Once the last instance finishes, stop the local emulator instance if we're using it.
             lock (_functionsProcessLock)
             {
                 if (UseFunctionsEmulator)

DataPipelineTools.Functions/Common/FunctionsBase.cs

@@ -14,16 +14,16 @@ public FunctionsBase(ILogger<FunctionsBase> logger)
             _logger = logger;
         }
 
-        protected JObject GetTemplateResponse(DataLakeConfig dataLakeConfig, object parameters)
+        protected JObject GetTemplateResponse(IDataLakeConnectionConfig dataLakeConnectionConfig, object parameters)
         {
             var assemblyInfo = AssemblyHelpers.GetAssemblyVersionInfoJson();
 
             var responseJson = new JObject();
             if (assemblyInfo.HasValues)
                 responseJson.Add("debugInfo", assemblyInfo);
 
-            if (dataLakeConfig.BaseUrl != null)
-                responseJson.Add("storageContainerUrl", dataLakeConfig.BaseUrl);
+            if (dataLakeConnectionConfig.BaseUrl != null)
+                responseJson.Add("storageContainerUrl", dataLakeConnectionConfig.BaseUrl);
 
             var paramatersJson = JObject.FromObject(parameters);
             responseJson.Add("parameters", paramatersJson);