Merge pull request #9 from rexxars/history-support

Added support for getting history of tests from WPT

Author: marcelduran

lib/helper.js

@@ -4,9 +4,11 @@
  * Released under the MIT License
  */
 
-var xml2js = require('xml2js'),
-    url    = require('url'),
-    os     = require('os');
+var xml2js   = require('xml2js'),
+    url      = require('url'),
+    os       = require('os'),
+    csv      = require('csv'),
+    entities = require('entities');
 
 var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true});
 
@@ -98,6 +100,20 @@ function svToObj(delimiter, headers, sv) {
   return obj;
 }
 
+function csvParser(data, callback) {
+  csv()
+    .from(data.toString(), { columns: true })
+    .transform(function(row, index) {
+      var key, value;
+      for (key in row) {
+        value = row[key].replace(/<b>|<\/b>/g, '');
+        row[key] = entities.decode(value, 2);
+      }
+      return row;
+    })
+    .to.array(callback);
+}
+
 // Net log has a buggy end of file, attempt to fix
 function netLogParser(data) {
   data = (data || '{}').toString();
@@ -223,6 +239,7 @@ module.exports = {
   xmlToObj: xmlToObj,
   csvToObj: svToObj.bind(null, ',', false),
   tsvToObj: svToObj.bind(null, '\t'),
+  csvParser: csvParser,
   netLogParser: netLogParser,
   scriptToString: scriptToString,
   dataURI: dataURI,

lib/mapping.js

@@ -624,6 +624,12 @@ var commands = {
     param: 'id',
     info: 'get test request info/details'
   },
+  'history': {
+    name: 'getHistory',
+    param: 'days',
+    optional: true,
+    info: 'get history of previously run tests'
+  },
   'waterfall': {
     name: 'getWaterfallImage',
     param: 'id',

lib/webpagetest.js

@@ -28,7 +28,8 @@ var paths = {
   har: 'export.php',
   waterfall: 'waterfall.php',
   thumbnail: 'thumbnail.php',
-  cancel: 'cancelTest.php'
+  cancel: 'cancelTest.php',
+  history: 'testlog.php'
 };
 
 var filenames = {
@@ -39,6 +40,7 @@ var filenames = {
   netLog: 'netlog.txt',
   consoleLog: 'console_log.json',
   testInfo: 'testinfo.json',
+  history: 'history.csv',
   waterfall: 'waterfall.png',
   screenshot: 'screen.jpg',
   screenshotStartRender: 'screen_render.jpg',
@@ -509,6 +511,28 @@ function getTestInfo(id, options, callback) {
   }, options);
 }
 
+function getHistory(days, options, callback) {
+  callback = callback || options;
+  options = options === callback ? {} : options;
+  var query = {
+    all: 'on',
+    f: 'csv',
+    days: days ? parseInt(days, 10) : 1
+  };
+
+  return api.call(this, paths.history, function(err, data) {
+    if (err) {
+      return callback(err);
+    } else if (!(data instanceof Buffer)) {
+      return callback(err, data);
+    }
+
+    helper.csvParser(data, function(parsed) {
+      callback(err, parsed);
+    });
+  }, query, options);
+}
+
 function getWaterfallImage(id, options, callback) {
   var query,
       pathname = paths.waterfall;
@@ -618,6 +642,7 @@ WebPageTest.prototype = {
   getNetLogData: getNetLogData,
   getConsoleLogData: getConsoleLogData,
   getTestInfo: getTestInfo,
+  getHistory: getHistory,
   getWaterfallImage: getWaterfallImage,
   getScreenshotImage: getScreenshotImage,
   scriptToString: WebPageTest.scriptToString,
@@ -638,6 +663,7 @@ WebPageTest.prototype = {
   netlog: getNetLogData,
   console: getConsoleLogData,
   testinfo: getTestInfo,
+  history: getHistory,
   waterfall: getWaterfallImage,
   screenshot: getScreenshotImage
 };

package.json

@@ -35,7 +35,9 @@
   "dependencies": {
     "commander": "~2.0.0",
     "mocha": "~1.13.0",
-    "xml2js": "~0.2.8"
+    "xml2js": "~0.2.8",
+    "csv": "~0.3.6",
+    "entities": "~0.3.0"
   },
   "devDependencies": {
     "nock": "~0.22.1"

test/command-line-test.js

@@ -231,6 +231,15 @@ describe('WebPageTest Command Line', function() {
     });
   });
 
+  it('gets a history input returns the API url', function(done) {
+    exec(mock('history 2'), function(err, data) {
+      if (err) return done(err);
+      data = JSON.parse(data);
+      assert.equal(data.url, wptServer + 'testlog.php?all=on&f=csv&days=2');
+      done();
+    });
+  });
+
   it('gets a waterfall image input returns the API url', function(done) {
     exec(mock('waterfall 120816_V2_2'), function(err, data) {
       if (err) return done(err);
@@ -289,7 +298,7 @@ describe('WebPageTest Command Line', function() {
   [
     '', 'status', 'results', 'locations', 'testers', 'test', 'cancel', 'har',
     'pagespeed', 'utilization', 'request', 'timeline', 'netlog', 'console',
-    'testinfo', 'waterfall', 'screenshot', 'listen'
+    'testinfo', 'history', 'waterfall', 'screenshot', 'listen'
   ].forEach(function eachCmd(command) {
       it('gets a ' + command + ' help input and returns the help text', function(done) {
         exec(mock(command + ' --help'), function(err, data) {

test/dryrun-test.js

@@ -235,6 +235,14 @@ describe('Dry Run', function() {
       });
     });
 
+    it('gets history request', function(done) {
+      wpt.getHistory(2, {dryRun: true}, function (err, data) {
+        if (err) return done(err);
+        assert.equal(data.url, wptServer + 'testlog.php?all=on&f=csv&days=2');
+        done();
+      });
+    });
+
     it('gets a waterfall image request', function(done) {
       wpt.getWaterfallImage('120816_V2_2', {dryRun: true}, function (err, data) {
         if (err) return done(err);

test/fixtures/command-line/help-history.txt

@@ -0,0 +1,7 @@
+
+  Usage: history [options] [days]
+
+  Options:
+
+    -h, --help          output usage information
+

test/fixtures/command-line/help.txt

@@ -17,6 +17,7 @@
     netlog [options] <id>  get the Chrome Developer Tools Net log data (if available) from test
     console [options] <id> get the browser console log data (if available) from test
     testinfo <id>          get test request info/details
+    history [days]         get history of previously run tests
     waterfall [options] <id> get the waterfall PNG image
     screenshot [options] <id> get the fully loaded page screenshot in JPG format (PNG if in full resolution)
     listen [hostname:port] start webpagetest-api server on <hostname>:<port> [hostname:7791]

test/fixtures/objects/history.json

@@ -0,0 +1,62 @@
+[
+  {
+    "Date/Time": "11/28/13 16:24:24",
+    "Location": "Dublin, Ireland - Chrome - Cable",
+    "Test ID": "131128_9N_S23",
+    "URL": "http://tech.vg.no/"
+  },
+  {
+    "Date/Time": "11/28/13 16:24:23",
+    "Location": "Dulles, VA - Chrome - Cable",
+    "Test ID": "131128_5W_S28",
+    "URL": "http://www.bdaygreetings.com"
+  },
+  {
+    "Date/Time": "11/28/13 16:24:23",
+    "Location": "Buenos Aires, Argentina - Chrome - Cable",
+    "Test ID": "131128_HA_S27",
+    "URL": "https://buyingflow.mercadolibre.com.ar/?item_id=MLA485711339&quantity=1"
+  },
+  {
+    "Date/Time": "11/28/13 16:24:19",
+    "Location": "Dulles, VA - IE 9 - Cable",
+    "Test ID": "131128_XC_S26",
+    "URL": "http://www.pietzoomers.com"
+  },
+  {
+    "Date/Time": "11/28/13 16:24:19",
+    "Location": "São Paulo, Brasil - Chrome - Cable",
+    "Test ID": "131128_A2_S25",
+    "URL": "http://www.livrariasaraiva.com.br"
+  },
+  {
+    "Date/Time": "11/28/13 16:24:17",
+    "Location": "Buenos Aires, Argentina - IE8 - Cable",
+    "Test ID": "131128_NW_S24",
+    "URL": "http://ti.kwikkiangie.ac.id"
+  },
+  {
+    "Date/Time": "11/28/13 16:23:11",
+    "Location": "Madrid, Spain - IE 8 - Cable",
+    "Test ID": "131128_CM_S15",
+    "URL": "https://www.rastreator.com/seguro-coche/datos-comparativa.aspx"
+  },
+  {
+    "Date/Time": "11/28/13 16:23:04",
+    "Location": "Dublin, Ireland - Chrome - Cable",
+    "Test ID": "131128_H8_S11",
+    "URL": "http://www.bilgi-sor.com"
+  },
+  {
+    "Date/Time": "11/28/13 16:22:40",
+    "Location": "Dulles, VA - IE 9 - Cable",
+    "Test ID": "131128_YA_S0S",
+    "URL": "http://www.debrastefan.com/"
+  },
+  {
+    "Date/Time": "11/28/13 16:22:35",
+    "Location": "Dulles, VA - IE 9 - Cable",
+    "Test ID": "131128_HS_S0M",
+    "URL": "http://recreational-looting.herokuapp.com"
+  }
+]

test/fixtures/responses/history.csv

@@ -0,0 +1,11 @@
+"Date/Time","Location","Test ID","URL"
+"11/28/13 16:24:24","<b>Dublin, Ireland</b> - <b>Chrome</b> - <b>Cable</b>","131128_9N_S23","http://tech.vg.no/"
+"11/28/13 16:24:23","Dulles, VA - <b>Chrome</b> - <b>Cable</b>","131128_5W_S28","http://www.bdaygreetings.com"
+"11/28/13 16:24:23","Buenos Aires, Argentina - <b>Chrome</b> - <b>Cable</b>","131128_HA_S27","https://buyingflow.mercadolibre.com.ar/?item_id=MLA485711339&amp;quantity=1"
+"11/28/13 16:24:19","Dulles, VA - IE 9 - <b>Cable</b>","131128_XC_S26","http://www.pietzoomers.com"
+"11/28/13 16:24:19","S&atilde;o Paulo, Brasil - <b>Chrome</b> - <b>Cable</b>","131128_A2_S25","http://www.livrariasaraiva.com.br"
+"11/28/13 16:24:17","Buenos Aires, Argentina - IE8 - <b>Cable</b>","131128_NW_S24","http://ti.kwikkiangie.ac.id"
+"11/28/13 16:23:11","<b>Madrid, Spain - IE 8</b> - <b>Cable</b>","131128_CM_S15","https://www.rastreator.com/seguro-coche/datos-comparativa.aspx"
+"11/28/13 16:23:04","<b>Dublin, Ireland</b> - <b>Chrome</b> - <b>Cable</b>","131128_H8_S11","http://www.bilgi-sor.com"
+"11/28/13 16:22:40","Dulles, VA - IE 9 - <b>Cable</b>","131128_YA_S0S","http://www.debrastefan.com/"
+"11/28/13 16:22:35","Dulles, VA - IE 9 - <b>Cable</b>","131128_HS_S0M","http://recreational-looting.herokuapp.com"