added batch command to run multiple wpt commands

Author: marcelduran

bin/webpagetest

@@ -10,30 +10,28 @@ var program     = require('commander'),
     WebPageTest = require('../lib/webpagetest'),
     mapping     = require('../lib/mapping');
 
-var command, wpt, file, ret,
+var command, file, ret,
     options = {},
     args = [],
+    results = [],
     defaultServer = process.env.WEBPAGETEST_SERVER ||
       WebPageTest.defaultServer;
 
 // replace default values on descriptons
 mapping.commands.listen.info =
   mapping.commands.listen.info.replace('%s', WebPageTest.defaultListenPort);
 
-function output(data, exitStatus, preExit) {
-  if (exitStatus) {
-    process.stderr.write(data);
-  } else {
-    process.stdout.write(data);
-  }
+var output = function(data, exitStatus, preExit) {
+  process.stdout.write(data);
   if (typeof preExit === 'function') {
     preExit();
   }
   process.exit(exitStatus || 0);
-}
+};
 
-var formatData = function (err, data, info) {
-  var encoding = 'utf8';
+var formatData = function(err, data, info) {
+  var dataStr,
+      encoding = 'utf8';
 
   if (err) {
     data = {error: err};
@@ -49,24 +47,26 @@ var formatData = function (err, data, info) {
 
   if (!(data instanceof Buffer)) {
     try {
-      data = JSON.stringify(data, null, 2);
+      dataStr = JSON.stringify(data, null, 2);
     } catch (ex) {
-      data = JSON.stringify({
+      dataStr = JSON.stringify({
         data: data.toString(),
         error: ex.message
       }, null, 2);
     }
   }
 
   if (file) {
-    fs.writeFile(file, data, encoding, function writeFile(err) {
+    fs.writeFile(file, dataStr, encoding, function writeFile(err) {
       if (err) {
         output(err.message, 1);
       }
     });
   } else {
-    output(data, err ? 1 : 0);
+    output(dataStr, err ? 1 : 0);
   }
+
+  return data;
 }
 
 function addOptions(prg, options, nokeyList) {
@@ -135,13 +135,13 @@ Object.keys(mapping.commands).forEach(function eachCommand(name) {
       }
 
       defaultAction(decodeURIComponent(what), opts);
-      if (options.specs) {
+      if (options.specs && !program.batch) {
         options.exitOnResults = true;
       }
     },
     'results': function(id, opts) {
       defaultAction(id, opts);
-      if (opts.specs && !program.dryrun) {
+      if (opts.specs && !program.dryrun && !program.batch) {
         formatData = null;
       }
     },
@@ -158,7 +158,7 @@ Object.keys(mapping.commands).forEach(function eachCommand(name) {
     }
   };
 
-  // command with (optional) parameter 
+  // command with (optional) parameter
   if (cmd.param) {
     if (cmd.optional) {
       openParam = '[';
@@ -179,10 +179,98 @@ Object.keys(mapping.commands).forEach(function eachCommand(name) {
   );
 });
 
-program.parse(process.argv);
+function parseBatch(filename) {
+  var lines;
 
-if (command) {
-  wpt = new WebPageTest(program.server || defaultServer);
+  try {
+    lines = (fs.readFileSync(filename, 'utf8') || '')
+      .split('\n')
+      .filter(function(line) {return line;})
+      .map(function(line) {return line.split(' ');})
+      .map(function(line) {
+        var tmp = [];
+        return line.reduce(function(prev, curr, index) {
+          if (/^[^\\]?['"]/.test(curr)) {
+            tmp.push(curr.slice(1));
+          } else if (tmp.length) {
+            if (/[^\\]['"]$/.test(curr)) {
+              tmp.push(curr.slice(0, curr.length - 1));
+              prev.push(tmp.join(' '));
+              tmp = [];
+            } else {
+              tmp.push(curr);
+            }
+          } else {
+            prev.push(curr);
+          }
+          return prev;
+        }, ['', '']);
+      });
+  } catch (err) {
+    output(err.message + '\n', 1);
+  }
+
+  return lines;
+}
+//return console.log(parseBatch('batch.txt'));
+
+function batch(file) {
+  var accErr,
+      count = 2,
+      originalFormatData = formatData,
+      originalOutput = output;
+
+  function checkDone() {
+    count--;
+    if (!count) {
+      originalOutput(
+        results.length ? JSON.stringify(results, null, 2) : '',
+        accErr ? 1 : 0
+      );
+    }
+  };
+
+  function checkError(err) {
+    accErr = accErr || err;
+    checkDone();
+  }
+
+  function aggregate(index, err, data, info) {
+    results[index] = originalFormatData(err, data, info);
+    checkError(err);
+  }
+
+  program.batch = true;
+  var cmds = parseBatch(file);
+  if (!cmds) {
+    return;
+  }
+  output = function(){};
+
+  cmds.forEach(function(cmd, index) {
+    program.parse(cmd);
+    formatData = options.specs ? checkError : aggregate.bind(null, index);
+    execCommand();
+    cleanupProgram();
+  });
+}
+
+function cleanupProgram() {
+  command = null;
+  options = {};
+  args = [];
+  program.commands.forEach(function(c) {
+    c.options.forEach(function(o) {
+      delete c[o.long.slice(2)];
+    });
+  });
+  delete program.server;
+  delete program.dryrun;
+  delete program.out;
+}
+
+function execCommand() {
+  var wpt = new WebPageTest(program.server || defaultServer);
 
   // options
   if (options) {
@@ -198,7 +286,19 @@ if (command) {
   file = program.out;
 
   wpt[command].apply(wpt, args);
-} else {
+}
+
+program
+  .command('batch <file>')
+  .description(
+    'run commands in batch, i.e. one command per line from <file> in parallel')
+  .action(batch);
+
+program.parse(process.argv);
+
+if (command) {
+  execCommand();
+} else if (!program.batch) {
   output(program.helpInformation(), 1, function defaultHelp() {
     program.emit('--help');
   });

lib/webpagetest.js

@@ -291,7 +291,7 @@ function runTest(what, options, callback) {
 
   function testCallback(cb, err, data) {
     if (err || !(data && data.data && data.data.testId)) {
-      callback(err || data);
+      return callback(err || data);
     }
 
     testId = data.data.testId;

test/command-line-test.js

@@ -283,4 +283,14 @@ describe('WebPageTest Command Line', function() {
       });
   });
 
+  it('gets a batch input returns the batch commands output in order', function(done) {
+    exec(mock('batch ' + path.join(__dirname, 'fixtures/batch.txt')), function(err, data) {
+      if (err) return done(err);
+      data = JSON.parse(data);
+      assert.equal(data[0].url, wptServer + 'testStatus.php?test=120816_V2_2');
+      assert.equal(data[1].url, wptServer + 'xmlResult.php?test=120816_V2_2');
+      done();
+    });
+  });
+
 });

test/fixtures/batch.txt

@@ -0,0 +1,2 @@
+status 120816_V2_2 -s https://www.example.com:1234/foo/bar/ -d
+results 120816_V2_2 -s https://www.example.com:1234/foo/bar/ -d

test/fixtures/command-line/help-test.txt

@@ -47,6 +47,7 @@
     -C, --spdy3                   force SPDY version 3 (Chrome only)
     -J, --swrender                force software rendering, disable GPU acceleration (Chrome only)
     -Q, --noparser                disable threaded HTML parser (Chrome only)
+    -q, --spdynossl               use SPDY without SSL (Chrome only)
         --poll [interval]         poll for results after test is scheduled at every <interval> seconds [5]
         --wait [hostname:port]    wait for test results informed by agent once complete listening on <hostname>:<port> [hostname:first port available above 8000]
         --timeout <seconds>       timeout for polling and waiting results [no timeout]

test/fixtures/command-line/help.txt

@@ -20,6 +20,7 @@
     waterfall [options] <id> get the waterfall PNG image
     screenshot [options] <id> get the fully loaded page screenshot in JPG format (PNG if in full resolution)
     listen [hostname:port] start webpagetest-api server on <hostname>:<port> [hostname:7791]
+    batch <file>           run commands in batch, i.e. one command per line from <file> in parallel
 
   Options: