⚠️ Deprecate streams option

For stability and reliability recovering after interrupted commection reasons
veliovgroup · Jan 29, 2021 · ec77bf1 · ec77bf1
1 parent 0599872
commit ec77bf1
Show file tree

Hide file tree

Showing 10 changed files with 42 additions and 85 deletions.
diff --git a/README.md b/README.md
@@ -138,7 +138,6 @@ Template.uploadForm.events({
       // multiple files were selected
       const upload = Images.insert({
         file: e.currentTarget.files[0],
-        streams: 'dynamic',
         chunkSize: 'dynamic'
       }, false);
 
@@ -311,7 +310,7 @@ For more expressive example see [Download demo](https://github.com/VeliovGroup/M
 3. __How to pause/continue upload and get progress/speed/remaining time?__: see *Object* returned from [`insert` method](https://github.com/VeliovGroup/Meteor-Files/blob/master/docs/insert.md)
 4. When using any of `accounts` packages - package `accounts-base` must be explicitly added to `.meteor/packages` above `ostrio:files`
 5. __cURL/POST uploads__ - Take a look on [POST-Example](https://github.com/noris666/Meteor-Files-POST-Example) by [@noris666](https://github.com/noris666)
-6. In __Safari__ (Mobile and Desktop) for `DDP` upload streams are hard-coded to `1` and chunk size is reduced by algorithm, due to error thrown if too many connection is open by the browser or frame is too big. Limit simultaneous uploads to `6` is recommended for Safari. This issue should be fixed in Safari 11. Switching to `http` transport (*which has no such issue*) is recommended for Safari. See [#458](https://github.com/VeliovGroup/Meteor-Files/issues/458)
+6. In __Safari__ (Mobile and Desktop) for `DDP` chunk size is reduced by algorithm, due to error thrown if frame is too big. Limit simultaneous uploads to `6` is recommended for Safari. This issue should be fixed in Safari 11. Switching to `http` transport (*which has no such issue*) is recommended for Safari. See [#458](https://github.com/VeliovGroup/Meteor-Files/issues/458)
 7. Make sure you're using single domain for the Meteor app, and the same domain for hosting Meteor-Files endpoints, see [#737](https://github.com/VeliovGroup/Meteor-Files/issues/737) for details
 8. When proxying requests to Meteor-Files endpoint make sure protocol `http/1.1` is used, see [#742](https://github.com/VeliovGroup/Meteor-Files/issues/742) for details
 

diff --git a/client.js b/client.js
@@ -219,7 +219,6 @@ export class FilesCollection extends FilesCollectionCore {
    *   {String}      fileId         - Optionnal `fileId` used at insert
    *   {Object}      meta           - Additional data as object, use later for search
    *   {Boolean}     allowWebWorkers- Allow/Deny WebWorkers usage
-   *   {Number|dynamic} streams     - Quantity of parallel upload streams, default: 2
    *   {Number|dynamic} chunkSize   - Chunk size for upload
    *   {String}      transport      - Upload transport `http` or `ddp`
    *   {Object}      ddp            - Custom DDP connection. Object returned form `DDP.connect()`

diff --git a/docs/about-transports.md b/docs/about-transports.md
@@ -48,4 +48,3 @@ The cons:
 
 - No mobile browsers support;
 - Chunk size limited to 64KB;
-- Only single stream is supported (*so, it's currently uses synchronous chunk uploads*. If `RTC/DC` will be accepted by community we will implement asynchronous chunks upload).
diff --git a/docs/constructor.md b/docs/constructor.md
@@ -220,7 +220,7 @@
         Isomorphic
       </td>
       <td>
-        Upload &amp; Serve (<em>for 206 responce</em>) chunk size
+        Upload &amp; Serve (<em>for 206 response</em>) chunk size
       </td>
       <td>
         <code>272144</code>

diff --git a/docs/file-subversions.md b/docs/file-subversions.md
@@ -57,8 +57,7 @@ if (Meteor.isClient) {
             } else {
               return "Please upload file in next formats: 'ogg', 'mp4', 'avi', 'webm' with size less than 512 Mb. You have tried to upload file with \"" + this.ext + "\" extension and with \"" + (Math.round((this.size / (1024 * 1024)) * 100) / 100) + "\" Mb";
             }
-          },
-          streams: 8
+          }
         });
       });
     }

diff --git a/docs/insert.md b/docs/insert.md
@@ -204,17 +204,6 @@ Upload file to a Server via DDP or HTTP.
         </ul>
       </td>
     </tr>
-    <tr>
-      <td align="right">
-        `settings.streams` {*Number*|dynamic}
-      </td>
-      <td>
-        Quantity of parallel upload streams
-      </td>
-      <td>
-        `dynamic` is recommended
-      </td>
-    </tr>
     <tr>
       <td align="right">
         `settings.chunkSize` {*Number*|dynamic}
@@ -619,7 +608,6 @@ Template.uploadForm.events({
           }
           template.currentFile.set(false);
         },
-        streams: 'dynamic',
         chunkSize: 'dynamic'
       });
     }
@@ -642,7 +630,6 @@ Template.uploadForm.events({
       // multiple files were selected
       Images.insert({
         file: e.currentTarget.files[0],
-        streams: 'dynamic',
         chunkSize: 'dynamic'
       }, false).on('start', function () {
         template.currentFile.set(this);
@@ -672,7 +659,6 @@ Template.uploadForm.events({
     if (e.currentTarget.files && e.currentTarget.files[0]) {
       const uploader = Images.insert({
         file: e.currentTarget.files[0],
-        streams: 'dynamic',
         chunkSize: 'dynamic'
       }, false);
 
@@ -751,7 +737,6 @@ Template.uploadForm.events({
       // multiple files were selected
       Images.insert({
         file: e.currentTarget.files[0],
-        streams: 'dynamic',
         chunkSize: 'dynamic'
       }, false).pipe(encrypt).pipe(zip).start();
     }

diff --git a/docs/react-example.md b/docs/react-example.md
@@ -66,7 +66,6 @@ class FileUploadComponent extends Component {
             locator: self.props.fileLocator,
             userId: Meteor.userId() // Optional, used to check on server for file tampering
           },
-          streams: 'dynamic',
           chunkSize: 'dynamic',
           allowWebWorkers: true // If you see issues with uploads, change this to false
         }, false)

diff --git a/docs/typescript-definitions.md b/docs/typescript-definitions.md
@@ -145,7 +145,6 @@ declare module "meteor/ostrio:files" {
       onError?: (error: Meteor.Error, fileData: FileData<MetadataType>) => any;
       onProgress?: (progress: number, fileData: FileData<MetadataType>) => any;
       onBeforeUpload?: (fileData: FileData<MetadataType>) => any;
-      streams?: number | 'dynamic';
       chunkSize?: number | 'dynamic';
       allowWebWorkers?: boolean;
       type?: string;

diff --git a/index.d.ts b/index.d.ts
@@ -142,7 +142,6 @@ declare module "meteor/ostrio:files" {
       onError?: (error: Meteor.Error, fileData: FileData<MetadataType>) => any;
       onProgress?: (progress: number, fileData: FileData<MetadataType>) => any;
       onBeforeUpload?: (fileData: FileData<MetadataType>) => any;
-      streams?: number | 'dynamic';
       chunkSize?: number | 'dynamic';
       allowWebWorkers?: boolean;
       type?: string;

diff --git a/upload.js b/upload.js
@@ -105,14 +105,6 @@ export class UploadInstance extends EventEmitter {
       this.config.meta = {};
     }
 
-    if (!this.config.streams) {
-      this.config.streams = 2;
-    }
-
-    if (this.config.streams < 1) {
-      this.config.streams = 2;
-    }
-
     if (!helpers.isString(this.config.transport)) {
       this.config.transport = 'ddp';
     }
@@ -139,7 +131,6 @@ export class UploadInstance extends EventEmitter {
       type: Match.Optional(String),
       onError: Match.Optional(Function),
       onAbort: Match.Optional(Function),
-      streams: Match.OneOf('dynamic', Number),
       onStart: Match.Optional(Function),
       fileName: Match.Optional(String),
       isBase64: Match.Optional(Boolean),
@@ -220,7 +211,6 @@ export class UploadInstance extends EventEmitter {
       this.startTime     = {};
       this.config.debug  = this.collection.debug;
       this.config._debug = this.collection._debug;
-      this.currentChunk  = 0;
       this.transferTime  = 0;
       this.trackerComp   = null;
       this.sentChunks    = 0;
@@ -260,10 +250,9 @@ export class UploadInstance extends EventEmitter {
       this.addListener('prepare', this.prepare);
       this.addListener('sendChunk', this.sendChunk);
       this.addListener('proceedChunk', this.proceedChunk);
-      this.addListener('createStreams', this.createStreams);
 
       this.addListener('calculateStats', helpers.throttle(() => {
-        const _t = (this.transferTime / this.sentChunks) / this.config.streams;
+        const _t = (this.transferTime / (this.sentChunks || 1));
         this.result.estimateTime.set((_t * (this.fileLength - this.sentChunks)));
         this.result.estimateSpeed.set((this.config.chunkSize / (_t / 1000)));
 
@@ -362,16 +351,15 @@ export class UploadInstance extends EventEmitter {
     if (opts.binData) {
       if (this.config.transport === 'ddp') {
         this.config.ddp.call(this.collection._methodNames._Write, opts, (error) => {
-          this.transferTime += (+new Date) - this.startTime[opts.chunkId];
+          this.transferTime += Date.now() - this.startTime[opts.chunkId];
           if (error) {
             if (this.result.state.get() !== 'aborted') {
               this.emit('end', error);
             }
           } else {
-            ++this.sentChunks;
-            if (this.sentChunks >= this.fileLength) {
+            if (++this.sentChunks >= this.fileLength) {
               this.emit('sendEOF');
-            } else if (this.currentChunk < this.fileLength) {
+            } else {
               this.emit('upload');
             }
             this.emit('calculateStats');
@@ -390,23 +378,22 @@ export class UploadInstance extends EventEmitter {
             'x-chunkid': opts.chunkId,
             'content-type': 'text/plain'
           }
-        }).then((responce) => {
-          if (responce.status === 204) {
+        }).then((response) => {
+          if (response.status === 204) {
             this.collection._debug('[FilesCollection] [sendChunk] [fetch()] [then] chunk successfully sent');
-            this.transferTime += +new Date() - this.startTime[opts.chunkId];
-            ++this.sentChunks;
-            if (this.sentChunks >= this.fileLength) {
+            this.transferTime += Date.now() - this.startTime[opts.chunkId];
+            if (++this.sentChunks >= this.fileLength) {
               this.emit('sendEOF');
-            } else if (this.currentChunk < this.fileLength) {
+            } else {
               this.emit('upload');
             }
             this.emit('calculateStats');
           } else {
-            this.emit('end', new Meteor.Error(responce.status, 'Can\'t continue upload, session expired. Please, start upload again.'));
+            this.emit('end', new Meteor.Error(response.status, 'Can\'t continue upload, session expired. Please, start upload again.'));
           }
         }).catch((error) => {
           this.collection._debug('[FilesCollection] [sendChunk] [fetch()] [error] EXCEPTION while sending chunk', error);
-          this.transferTime += +new Date() - this.startTime[opts.chunkId];
+          this.transferTime += Date.now() - this.startTime[opts.chunkId];
           Meteor.setTimeout(() => {
             if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') {
               this.result.pause();
@@ -452,8 +439,14 @@ export class UploadInstance extends EventEmitter {
 
           this.emit('end', void 0, result);
         }).catch((error) => {
-          console.warn('Something went wrong! [sendEOF] method doesn\'t returned JSON! Looks like you\'re on Cordova app or behind proxy, switching to DDP transport is recommended.');
-          this.emit('end', error, {});
+          Meteor.setTimeout(() => {
+            if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') {
+              this.result.pause();
+            } else if (this.result.state.get() !== 'aborted') {
+              console.warn('Something went wrong! [sendEOF] method doesn\'t returned JSON! Looks like you\'re on Cordova app or behind proxy, switching to DDP transport is recommended.');
+              this.emit('end', error);
+            }
+          }, 512);
         });
       }
     }
@@ -512,34 +505,24 @@ export class UploadInstance extends EventEmitter {
       return this;
     }
 
-    if (this.currentChunk <= this.fileLength) {
-      ++this.currentChunk;
+    if (this.sentChunks + 1 <= this.fileLength) {
       if (this.worker) {
         this.worker.postMessage({
           f: this.config.file,
-          cc: this.currentChunk,
+          cc: this.sentChunks + 1,
           cs: this.config.chunkSize,
           ib: this.config.isBase64
         });
       } else {
-        this.emit('proceedChunk', this.currentChunk);
+        this.emit('proceedChunk', this.sentChunks + 1);
       }
     } else {
       this.emit('sendEOF');
     }
-    this.startTime[this.currentChunk] = +new Date();
+    this.startTime[this.sentChunks + 1] = Date.now();
     return this;
   }
 
-  createStreams() {
-    this.collection._debug('[FilesCollection] [UploadInstance] [createStreams]');
-    let i = 1;
-    while (i <= this.config.streams) {
-      this.emit('upload');
-      i++;
-    }
-  }
-
   prepare() {
     let _len;
 
@@ -569,21 +552,7 @@ export class UploadInstance extends EventEmitter {
       _len = Math.ceil(this.fileData.size / this.config.chunkSize);
     }
 
-    if (this.config.streams === 'dynamic') {
-      this.config.streams = helpers.clone(_len);
-      if (this.config.streams > 24) { this.config.streams = 24; }
-
-      if (this.config.transport === 'http') {
-        this.config.streams = Math.round(this.config.streams / 2);
-      } else if (isSafari) {
-        this.config.streams = 1;
-      }
-    }
-
     this.fileLength = _len <= 0 ? 1 : _len;
-    if (this.config.streams > this.fileLength) {
-      this.config.streams = this.fileLength;
-    }
     this.result.config.fileLength = this.fileLength;
 
     const opts = {
@@ -599,14 +568,20 @@ export class UploadInstance extends EventEmitter {
 
     const handleStart = (error) => {
       if (error) {
-        this.collection._debug('[FilesCollection] [_Start] Error:', error);
-        this.emit('end', error);
+        Meteor.setTimeout(() => {
+          if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') {
+            this.result.pause();
+          } else if (this.result.state.get() !== 'aborted') {
+            this.collection._debug('[FilesCollection] [_Start] Error:', error);
+            this.emit('end', error);
+          }
+        }, 512);
       } else {
         this.result.continueFunc = () => {
           this.collection._debug('[FilesCollection] [insert] [continueFunc]');
-          this.emit('createStreams');
+          this.emit('upload');
         };
-        this.emit('createStreams');
+        this.emit('upload');
       }
     };
 
@@ -627,8 +602,12 @@ export class UploadInstance extends EventEmitter {
           'x-start': '1',
           'x-mtok': (helpers.isObject(Meteor.connection) ? Meteor.connection._lastSessionId : void 0) || null
         }
-      }).then(() => {
-        handleStart();
+      }).then((response) => {
+        if (response.status === 204) {
+          handleStart();
+        } else {
+          this.emit('end', new Meteor.Error(response.status, 'Can\'t start upload, make sure you\'re connected to the Internet. Reload the page or try again later.'));
+        }
       }).catch(handleStart);
     }
   }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,4 +48,3 @@ The cons:

		- No mobile browsers support;
		- Chunk size limited to 64KB;
		- Only single stream is supported (so, it's currently uses synchronous chunk uploads. If `RTC/DC` will be accepted by community we will implement asynchronous chunks upload).
-Original file line number
+Diff line change
@@ Expand Up / @@ -57,8 +57,7 @@ if (Meteor.isClient) { @@
                 } else {
                   return "Please upload file in next formats: 'ogg', 'mp4', 'avi', 'webm' with size less than 512 Mb. You have tried to upload file with \"" + this.ext + "\" extension and with \"" + (Math.round((this.size / (1024 * 1024)) * 100) / 100) + "\" Mb";
                 }
-              },
-              streams: 8
+              }
             });
           });
         }
@@ Expand Down @@