Support asynchronous stdout/stderr processing

Author: mbg

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -85,6 +85,18 @@ int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory,
             return ret;
         }
 
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
+
+            foreach (var line in stdout)
+            {
+                onOutput(line);
+            }
+
+            return ret;
+        }
+
         public IList<string> DirectoryDeleteIn { get; } = new List<string>();
 
         void IBuildActions.DirectoryDelete(string dir, bool recursive)

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -11,11 +11,26 @@
 
 namespace Semmle.Autobuild.Shared
 {
+    public delegate void BuildOutputHandler(string? data);
+
     /// <summary>
     /// Wrapper around system calls so that the build scripts can be unit-tested.
     /// </summary>
     public interface IBuildActions
     {
+
+        /// <summary>
+        /// Runs a process, captures its output, and provides it asynchronously.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="onOutput">A handler for stdout output.</param>
+        /// <param name="onError">A handler for stderr output.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
         /// <summary>
         /// Runs a process and captures its output.
         /// </summary>
@@ -182,6 +197,26 @@ private static ProcessStartInfo GetProcessStartInfo(string exe, string arguments
             return pi;
         }
 
+        int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
+            using var p = new Process
+            {
+                StartInfo = pi
+            };
+            p.StartInfo.RedirectStandardError = true;
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
+            p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
+
+            p.Start();
+
+            p.BeginErrorReadLine();
+            p.BeginOutputReadLine();
+
+            p.WaitForExit();
+            return p.ExitCode;
+        }
+
         int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
         {
             var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);

csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs

@@ -46,6 +46,30 @@ public abstract class BuildScript
         /// <returns>The exit code from this build script.</returns>
         public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
 
+        /// <summary>
+        /// Runs this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="onOutput">
+        /// A handler for data read from stdout.
+        /// </param>
+        /// <param name="onError">
+        /// A handler for data read from stderr.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
         private class BuildCommand : BuildScript
         {
             private readonly string exe, arguments;
@@ -110,6 +134,24 @@ public override int Run(IBuildActions actions, Action<string, bool> startCallbac
                 return ret;
             }
 
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
         }
 
         private class ReturnBuildCommand : BuildScript
@@ -127,8 +169,13 @@ public override int Run(IBuildActions actions, Action<string, bool> startCallbac
                 stdout = Array.Empty<string>();
                 return func(actions);
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
         }
 
+        /// <summary>
+        /// Allows two build scripts to be composed sequentially.
+        /// </summary>
         private class BindBuildScript : BuildScript
         {
             private readonly BuildScript s1;
@@ -175,6 +222,32 @@ public override int Run(IBuildActions actions, Action<string, bool> startCallbac
                 stdout = @out;
                 return ret2;
             }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    var stdout1 = new List<string>();
+                    var onOutputWrapper = new BuildOutputHandler(data =>
+                    {
+                        if (data is not null)
+                            stdout1.Add(data);
+
+                        onOutput(data);
+                    });
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
         }
 
         /// <summary>