Merge pull request #111 from evilaliv3/feature/readHook

feat: read hook

Author: AidasK

README.md

@@ -119,8 +119,10 @@ function, it will be passed a FlowFile, a FlowChunk and isTest boolean (Default:
 * `allowDuplicateUploads ` Once a file is uploaded, allow reupload of the same file. By default, if a file is already uploaded, it will be skipped unless the file is removed from the existing Flow object. (Default: `false`)
 * `prioritizeFirstAndLastChunk` Prioritize first and last chunks of all files. This can be handy if you can determine if a file is valid for your service from only the first or last chunk. For example, photo or video meta data is usually located in the first part of a file, making it easy to test support from only the first chunk. (Default: `false`)
 * `testChunks` Make a GET request to the server for each chunks to see if it already exists. If implemented on the server-side, this will allow for upload resumes even after a browser crash or even a computer restart. (Default: `true`)
-* `preprocess` Optional function to process each chunk before testing & sending. Function is passed the chunk as parameter, and should call the `preprocessFinished` method on the chunk when finished. (Default: `null`)
-* `generateUniqueIdentifier` Override the function that generates unique identifiers for each file.  (Default: `null`)
+* `preprocess` Optional function to process each chunk before testing & sending. To the function it will be passed the chunk as parameter, and should call the `preprocessFinished` method on the chunk when finished. (Default: `null`)
+* `initFileFn` Optional function to initialize the fileObject. To the function it will be passed a FlowFile and a FlowChunk arguments.
+* `readFileFn` Optional function wrapping reading operation from the original file. To the function it will be passed the FlowFile, the startByte and endByte, the fileType and the FlowChunk.
+* `generateUniqueIdentifier` Override the function that generates unique identifiers for each file. (Default: `null`)
 * `maxChunkRetries` The maximum number of retries for a chunk before the upload is failed. Valid values are any positive integer and `undefined` for no limit. (Default: `0`)
 * `chunkRetryInterval` The number of milliseconds to wait before retrying a chunk on a non-permanent error.  Valid values are any positive integer and `undefined` for immediate retry. (Default: `undefined`)
 * `progressCallbacksInterval` The time interval in milliseconds between progress reports. Set it

src/flow.js

@@ -29,6 +29,8 @@
    * @param {number} [opts.chunkRetryInterval]
    * @param {Array.<number>} [opts.permanentErrors]
    * @param {Array.<number>} [opts.successStatuses]
+   * @param {Function} [opts.initFileFn]
+   * @param {Function} [opts.readFileFn]
    * @param {Function} [opts.generateUniqueIdentifier]
    * @constructor
    */
@@ -91,9 +93,11 @@
       chunkRetryInterval: null,
       permanentErrors: [404, 415, 500, 501],
       successStatuses: [200, 201, 202],
-      onDropStopPropagation: false
+      onDropStopPropagation: false,
+      initFileFn: null,
+      readFileFn: webAPIFileRead
     };
-
+    
     /**
      * Current options
      * @type {Object}
@@ -144,6 +148,7 @@
      * @type {Object}
      */
     this.opts = Flow.extend({}, this.defaults, opts || {});
+
   }
 
   Flow.prototype = {
@@ -697,6 +702,12 @@
      * @type {Flow}
      */
     this.flowObj = flowObj;
+    
+    /**
+     * Used to store the bytes read
+     * @type {Blob|string}
+     */
+    this.bytes = null;
 
     /**
      * Reference to file
@@ -906,13 +917,17 @@
      * @function
      */
     bootstrap: function () {
+      if (typeof this.flowObj.opts.initFileFn === "function") {
+        this.flowObj.opts.initFileFn(this);
+      }
+
       this.abort(true);
       this.error = false;
       // Rebuild stack of chunks from file
       this._prevProgress = 0;
       var round = this.flowObj.opts.forceChunkSize ? Math.ceil : Math.floor;
       var chunks = Math.max(
-        round(this.file.size / this.flowObj.opts.chunkSize), 1
+        round(this.size / this.flowObj.opts.chunkSize), 1
       );
       for (var offset = 0; offset < chunks; offset++) {
         this.chunks.push(
@@ -971,7 +986,7 @@
       var outstanding = false;
       each(this.chunks, function (chunk) {
         var status = chunk.status();
-        if (status === 'pending' || status === 'uploading' || chunk.preprocessState === 1) {
+        if (status === 'pending' || status === 'uploading' || status === 'reading' || chunk.preprocessState === 1 || chunk.readState === 1) {
           outstanding = true;
           return false;
         }
@@ -1031,11 +1046,24 @@
     }
   };
 
+  /**
+   * Default read function using the webAPI
+   *
+   * @function webAPIFileRead(fileObj, fileType, startByte, endByte, chunk)
+   *
+   */
+  function webAPIFileRead(fileObj, fileType, startByte, endByte, chunk) {
+    var function_name = 'slice';
 
+    if (fileObj.file.slice)
+      function_name =  'slice';
+    else if (fileObj.file.mozSlice)
+      function_name = 'mozSlice';
+    else if (fileObj.file.webkitSlice)
+      function_name = 'webkitSlice';
 
-
-
-
+    chunk.readFinished(fileObj.file[function_name](startByte, endByte, fileType));
+  }
 
 
   /**
@@ -1060,12 +1088,6 @@
      */
     this.fileObj = fileObj;
 
-    /**
-     * File size
-     * @type {number}
-     */
-    this.fileObjSize = fileObj.size;
-
     /**
      * File offset
      * @type {number}
@@ -1096,6 +1118,13 @@
      */
     this.preprocessState = 0;
 
+    /**
+     * Read state
+     * @type {number} 0 = not read, 1 = reading, 2 = finished
+     */
+    this.readState = 0;
+
+
     /**
      * Bytes transferred from total request size
      * @type {number}
@@ -1112,33 +1141,26 @@
      * Size of a chunk
      * @type {number}
      */
-    var chunkSize = this.flowObj.opts.chunkSize;
+    this.chunkSize = this.flowObj.opts.chunkSize;
 
     /**
      * Chunk start byte in a file
      * @type {number}
      */
-    this.startByte = this.offset * chunkSize;
+    this.startByte = this.offset * this.chunkSize;
 
     /**
      * Chunk end byte in a file
      * @type {number}
      */
-    this.endByte = Math.min(this.fileObjSize, (this.offset + 1) * chunkSize);
+    this.endByte = Math.min(this.fileObj.size, (this.offset + 1) * this.chunkSize);
 
     /**
      * XMLHttpRequest
      * @type {XMLHttpRequest}
      */
     this.xhr = null;
 
-    if (this.fileObjSize - this.endByte < chunkSize &&
-        !this.flowObj.opts.forceChunkSize) {
-      // The last chunk will be bigger than the chunk size,
-      // but less than 2*chunkSize
-      this.endByte = this.fileObjSize;
-    }
-
     var $ = this;
 
 
@@ -1192,6 +1214,7 @@
     this.doneHandler = function(event) {
       var status = $.status();
       if (status === 'success' || status === 'error') {
+        delete this.data;
         $.event(status, $.message());
         $.flowObj.uploadNextChunk();
       } else {
@@ -1221,7 +1244,7 @@
         flowChunkNumber: this.offset + 1,
         flowChunkSize: this.flowObj.opts.chunkSize,
         flowCurrentChunkSize: this.endByte - this.startByte,
-        flowTotalSize: this.fileObjSize,
+        flowTotalSize: this.fileObj.size,
         flowIdentifier: this.fileObj.uniqueIdentifier,
         flowFilename: this.fileObj.name,
         flowRelativePath: this.fileObj.relativePath,
@@ -1264,16 +1287,37 @@
      * @function
      */
     preprocessFinished: function () {
+      // Compute the endByte after the preprocess function to allow an
+      // implementer of preprocess to set the fileObj size
+      this.endByte = Math.min(this.fileObj.size, (this.offset + 1) * this.chunkSize);
+      if (this.fileObj.size - this.endByte < this.chunkSize &&
+          !this.flowObj.opts.forceChunkSize) {
+        // The last chunk will be bigger than the chunk size,
+        // but less than 2*this.chunkSize
+        this.endByte = this.fileObj.size;
+      }
       this.preprocessState = 2;
       this.send();
     },
 
+    /**
+     * Finish read state
+     * @function
+     */
+    readFinished: function (bytes) {
+      this.readState = 2;
+      this.bytes = bytes;
+      this.send();
+    },
+
+
     /**
      * Uploads the actual data in a POST call
      * @function
      */
     send: function () {
       var preprocess = this.flowObj.opts.preprocess;
+      var read = this.flowObj.opts.readFileFn;
       if (typeof preprocess === 'function') {
         switch (this.preprocessState) {
           case 0:
@@ -1284,6 +1328,14 @@
             return;
         }
       }
+      switch (this.readState) {
+        case 0:
+          this.readState = 1;
+          read(this.fileObj, this.startByte, this.endByte, this.fileType, this);
+          return;
+        case 1:
+          return;
+      }
       if (this.flowObj.opts.testChunks && !this.tested) {
         this.test();
         return;
@@ -1293,20 +1345,14 @@
       this.total = 0;
       this.pendingRetry = false;
 
-      var func = (this.fileObj.file.slice ? 'slice' :
-        (this.fileObj.file.mozSlice ? 'mozSlice' :
-          (this.fileObj.file.webkitSlice ? 'webkitSlice' :
-            'slice')));
-      var bytes = this.fileObj.file[func](this.startByte, this.endByte, this.fileObj.file.type);
-
       // Set up request and listen for event
       this.xhr = new XMLHttpRequest();
       this.xhr.upload.addEventListener('progress', this.progressHandler, false);
       this.xhr.addEventListener("load", this.doneHandler, false);
       this.xhr.addEventListener("error", this.doneHandler, false);
 
       var uploadMethod = evalOpts(this.flowObj.opts.uploadMethod, this.fileObj, this);
-      var data = this.prepareXhrRequest(uploadMethod, false, this.flowObj.opts.method, bytes);
+      var data = this.prepareXhrRequest(uploadMethod, false, this.flowObj.opts.method, this.bytes);
       this.xhr.send(data);
     },
 
@@ -1329,7 +1375,9 @@
      * @returns {string} 'pending', 'uploading', 'success', 'error'
      */
     status: function (isTest) {
-      if (this.pendingRetry || this.preprocessState === 1) {
+      if (this.readState === 1) {
+        return 'reading';
+      } else if (this.pendingRetry || this.preprocessState === 1) {
         // if pending retry then that's effectively the same as actively uploading,
         // there might just be a slight delay before the retry starts
         return 'uploading';

test/fileSpec.js

@@ -34,4 +34,5 @@ describe('FlowFile functions', function() {
     file.name = '.dwq.dq.wd.qdw.E';
     expect(file.getExtension()).toBe('e');
   });
-});
+
+});