From ec77bf13c99b7c5571b7bfe1bef69a825fb64c93 Mon Sep 17 00:00:00 2001 From: "dr.dimitru" Date: Sat, 30 Jan 2021 01:09:05 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20Deprecate=20`streams`=20op?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For stability and reliability recovering after interrupted commection reasons --- README.md | 3 +- client.js | 1 - docs/about-transports.md | 1 - docs/constructor.md | 2 +- docs/file-subversions.md | 3 +- docs/insert.md | 15 ------ docs/react-example.md | 1 - docs/typescript-definitions.md | 1 - index.d.ts | 1 - upload.js | 99 ++++++++++++++-------------------- 10 files changed, 42 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 468b28b9..29780c1d 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,6 @@ Template.uploadForm.events({ // multiple files were selected const upload = Images.insert({ file: e.currentTarget.files[0], - streams: 'dynamic', chunkSize: 'dynamic' }, false); @@ -311,7 +310,7 @@ For more expressive example see [Download demo](https://github.com/VeliovGroup/M 3. __How to pause/continue upload and get progress/speed/remaining time?__: see *Object* returned from [`insert` method](https://github.com/VeliovGroup/Meteor-Files/blob/master/docs/insert.md) 4. When using any of `accounts` packages - package `accounts-base` must be explicitly added to `.meteor/packages` above `ostrio:files` 5. __cURL/POST uploads__ - Take a look on [POST-Example](https://github.com/noris666/Meteor-Files-POST-Example) by [@noris666](https://github.com/noris666) -6. In __Safari__ (Mobile and Desktop) for `DDP` upload streams are hard-coded to `1` and chunk size is reduced by algorithm, due to error thrown if too many connection is open by the browser or frame is too big. Limit simultaneous uploads to `6` is recommended for Safari. This issue should be fixed in Safari 11. Switching to `http` transport (*which has no such issue*) is recommended for Safari. See [#458](https://github.com/VeliovGroup/Meteor-Files/issues/458) +6. In __Safari__ (Mobile and Desktop) for `DDP` chunk size is reduced by algorithm, due to error thrown if frame is too big. Limit simultaneous uploads to `6` is recommended for Safari. This issue should be fixed in Safari 11. Switching to `http` transport (*which has no such issue*) is recommended for Safari. See [#458](https://github.com/VeliovGroup/Meteor-Files/issues/458) 7. Make sure you're using single domain for the Meteor app, and the same domain for hosting Meteor-Files endpoints, see [#737](https://github.com/VeliovGroup/Meteor-Files/issues/737) for details 8. When proxying requests to Meteor-Files endpoint make sure protocol `http/1.1` is used, see [#742](https://github.com/VeliovGroup/Meteor-Files/issues/742) for details diff --git a/client.js b/client.js index fd0ecbe5..4010f45d 100644 --- a/client.js +++ b/client.js @@ -219,7 +219,6 @@ export class FilesCollection extends FilesCollectionCore { * {String} fileId - Optionnal `fileId` used at insert * {Object} meta - Additional data as object, use later for search * {Boolean} allowWebWorkers- Allow/Deny WebWorkers usage - * {Number|dynamic} streams - Quantity of parallel upload streams, default: 2 * {Number|dynamic} chunkSize - Chunk size for upload * {String} transport - Upload transport `http` or `ddp` * {Object} ddp - Custom DDP connection. Object returned form `DDP.connect()` diff --git a/docs/about-transports.md b/docs/about-transports.md index 0560903f..a12d0203 100644 --- a/docs/about-transports.md +++ b/docs/about-transports.md @@ -48,4 +48,3 @@ The cons: - No mobile browsers support; - Chunk size limited to 64KB; -- Only single stream is supported (*so, it's currently uses synchronous chunk uploads*. If `RTC/DC` will be accepted by community we will implement asynchronous chunks upload). diff --git a/docs/constructor.md b/docs/constructor.md index f2e62c4e..4054cebd 100644 --- a/docs/constructor.md +++ b/docs/constructor.md @@ -220,7 +220,7 @@ Isomorphic - Upload & Serve (for 206 responce) chunk size + Upload & Serve (for 206 response) chunk size 272144 diff --git a/docs/file-subversions.md b/docs/file-subversions.md index a366476c..8bb08371 100644 --- a/docs/file-subversions.md +++ b/docs/file-subversions.md @@ -57,8 +57,7 @@ if (Meteor.isClient) { } else { return "Please upload file in next formats: 'ogg', 'mp4', 'avi', 'webm' with size less than 512 Mb. You have tried to upload file with \"" + this.ext + "\" extension and with \"" + (Math.round((this.size / (1024 * 1024)) * 100) / 100) + "\" Mb"; } - }, - streams: 8 + } }); }); } diff --git a/docs/insert.md b/docs/insert.md index 96394035..5ed901cb 100644 --- a/docs/insert.md +++ b/docs/insert.md @@ -204,17 +204,6 @@ Upload file to a Server via DDP or HTTP. - - - `settings.streams` {*Number*|dynamic} - - - Quantity of parallel upload streams - - - `dynamic` is recommended - - `settings.chunkSize` {*Number*|dynamic} @@ -619,7 +608,6 @@ Template.uploadForm.events({ } template.currentFile.set(false); }, - streams: 'dynamic', chunkSize: 'dynamic' }); } @@ -642,7 +630,6 @@ Template.uploadForm.events({ // multiple files were selected Images.insert({ file: e.currentTarget.files[0], - streams: 'dynamic', chunkSize: 'dynamic' }, false).on('start', function () { template.currentFile.set(this); @@ -672,7 +659,6 @@ Template.uploadForm.events({ if (e.currentTarget.files && e.currentTarget.files[0]) { const uploader = Images.insert({ file: e.currentTarget.files[0], - streams: 'dynamic', chunkSize: 'dynamic' }, false); @@ -751,7 +737,6 @@ Template.uploadForm.events({ // multiple files were selected Images.insert({ file: e.currentTarget.files[0], - streams: 'dynamic', chunkSize: 'dynamic' }, false).pipe(encrypt).pipe(zip).start(); } diff --git a/docs/react-example.md b/docs/react-example.md index f36e9e02..74c3a814 100644 --- a/docs/react-example.md +++ b/docs/react-example.md @@ -66,7 +66,6 @@ class FileUploadComponent extends Component { locator: self.props.fileLocator, userId: Meteor.userId() // Optional, used to check on server for file tampering }, - streams: 'dynamic', chunkSize: 'dynamic', allowWebWorkers: true // If you see issues with uploads, change this to false }, false) diff --git a/docs/typescript-definitions.md b/docs/typescript-definitions.md index 57bae7ea..e1d1f11d 100644 --- a/docs/typescript-definitions.md +++ b/docs/typescript-definitions.md @@ -145,7 +145,6 @@ declare module "meteor/ostrio:files" { onError?: (error: Meteor.Error, fileData: FileData) => any; onProgress?: (progress: number, fileData: FileData) => any; onBeforeUpload?: (fileData: FileData) => any; - streams?: number | 'dynamic'; chunkSize?: number | 'dynamic'; allowWebWorkers?: boolean; type?: string; diff --git a/index.d.ts b/index.d.ts index fdd185ed..6c3024df 100644 --- a/index.d.ts +++ b/index.d.ts @@ -142,7 +142,6 @@ declare module "meteor/ostrio:files" { onError?: (error: Meteor.Error, fileData: FileData) => any; onProgress?: (progress: number, fileData: FileData) => any; onBeforeUpload?: (fileData: FileData) => any; - streams?: number | 'dynamic'; chunkSize?: number | 'dynamic'; allowWebWorkers?: boolean; type?: string; diff --git a/upload.js b/upload.js index 5bb7e5e9..293fc850 100644 --- a/upload.js +++ b/upload.js @@ -105,14 +105,6 @@ export class UploadInstance extends EventEmitter { this.config.meta = {}; } - if (!this.config.streams) { - this.config.streams = 2; - } - - if (this.config.streams < 1) { - this.config.streams = 2; - } - if (!helpers.isString(this.config.transport)) { this.config.transport = 'ddp'; } @@ -139,7 +131,6 @@ export class UploadInstance extends EventEmitter { type: Match.Optional(String), onError: Match.Optional(Function), onAbort: Match.Optional(Function), - streams: Match.OneOf('dynamic', Number), onStart: Match.Optional(Function), fileName: Match.Optional(String), isBase64: Match.Optional(Boolean), @@ -220,7 +211,6 @@ export class UploadInstance extends EventEmitter { this.startTime = {}; this.config.debug = this.collection.debug; this.config._debug = this.collection._debug; - this.currentChunk = 0; this.transferTime = 0; this.trackerComp = null; this.sentChunks = 0; @@ -260,10 +250,9 @@ export class UploadInstance extends EventEmitter { this.addListener('prepare', this.prepare); this.addListener('sendChunk', this.sendChunk); this.addListener('proceedChunk', this.proceedChunk); - this.addListener('createStreams', this.createStreams); this.addListener('calculateStats', helpers.throttle(() => { - const _t = (this.transferTime / this.sentChunks) / this.config.streams; + const _t = (this.transferTime / (this.sentChunks || 1)); this.result.estimateTime.set((_t * (this.fileLength - this.sentChunks))); this.result.estimateSpeed.set((this.config.chunkSize / (_t / 1000))); @@ -362,16 +351,15 @@ export class UploadInstance extends EventEmitter { if (opts.binData) { if (this.config.transport === 'ddp') { this.config.ddp.call(this.collection._methodNames._Write, opts, (error) => { - this.transferTime += (+new Date) - this.startTime[opts.chunkId]; + this.transferTime += Date.now() - this.startTime[opts.chunkId]; if (error) { if (this.result.state.get() !== 'aborted') { this.emit('end', error); } } else { - ++this.sentChunks; - if (this.sentChunks >= this.fileLength) { + if (++this.sentChunks >= this.fileLength) { this.emit('sendEOF'); - } else if (this.currentChunk < this.fileLength) { + } else { this.emit('upload'); } this.emit('calculateStats'); @@ -390,23 +378,22 @@ export class UploadInstance extends EventEmitter { 'x-chunkid': opts.chunkId, 'content-type': 'text/plain' } - }).then((responce) => { - if (responce.status === 204) { + }).then((response) => { + if (response.status === 204) { this.collection._debug('[FilesCollection] [sendChunk] [fetch()] [then] chunk successfully sent'); - this.transferTime += +new Date() - this.startTime[opts.chunkId]; - ++this.sentChunks; - if (this.sentChunks >= this.fileLength) { + this.transferTime += Date.now() - this.startTime[opts.chunkId]; + if (++this.sentChunks >= this.fileLength) { this.emit('sendEOF'); - } else if (this.currentChunk < this.fileLength) { + } else { this.emit('upload'); } this.emit('calculateStats'); } else { - this.emit('end', new Meteor.Error(responce.status, 'Can\'t continue upload, session expired. Please, start upload again.')); + this.emit('end', new Meteor.Error(response.status, 'Can\'t continue upload, session expired. Please, start upload again.')); } }).catch((error) => { this.collection._debug('[FilesCollection] [sendChunk] [fetch()] [error] EXCEPTION while sending chunk', error); - this.transferTime += +new Date() - this.startTime[opts.chunkId]; + this.transferTime += Date.now() - this.startTime[opts.chunkId]; Meteor.setTimeout(() => { if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') { this.result.pause(); @@ -452,8 +439,14 @@ export class UploadInstance extends EventEmitter { this.emit('end', void 0, result); }).catch((error) => { - console.warn('Something went wrong! [sendEOF] method doesn\'t returned JSON! Looks like you\'re on Cordova app or behind proxy, switching to DDP transport is recommended.'); - this.emit('end', error, {}); + Meteor.setTimeout(() => { + if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') { + this.result.pause(); + } else if (this.result.state.get() !== 'aborted') { + console.warn('Something went wrong! [sendEOF] method doesn\'t returned JSON! Looks like you\'re on Cordova app or behind proxy, switching to DDP transport is recommended.'); + this.emit('end', error); + } + }, 512); }); } } @@ -512,34 +505,24 @@ export class UploadInstance extends EventEmitter { return this; } - if (this.currentChunk <= this.fileLength) { - ++this.currentChunk; + if (this.sentChunks + 1 <= this.fileLength) { if (this.worker) { this.worker.postMessage({ f: this.config.file, - cc: this.currentChunk, + cc: this.sentChunks + 1, cs: this.config.chunkSize, ib: this.config.isBase64 }); } else { - this.emit('proceedChunk', this.currentChunk); + this.emit('proceedChunk', this.sentChunks + 1); } } else { this.emit('sendEOF'); } - this.startTime[this.currentChunk] = +new Date(); + this.startTime[this.sentChunks + 1] = Date.now(); return this; } - createStreams() { - this.collection._debug('[FilesCollection] [UploadInstance] [createStreams]'); - let i = 1; - while (i <= this.config.streams) { - this.emit('upload'); - i++; - } - } - prepare() { let _len; @@ -569,21 +552,7 @@ export class UploadInstance extends EventEmitter { _len = Math.ceil(this.fileData.size / this.config.chunkSize); } - if (this.config.streams === 'dynamic') { - this.config.streams = helpers.clone(_len); - if (this.config.streams > 24) { this.config.streams = 24; } - - if (this.config.transport === 'http') { - this.config.streams = Math.round(this.config.streams / 2); - } else if (isSafari) { - this.config.streams = 1; - } - } - this.fileLength = _len <= 0 ? 1 : _len; - if (this.config.streams > this.fileLength) { - this.config.streams = this.fileLength; - } this.result.config.fileLength = this.fileLength; const opts = { @@ -599,14 +568,20 @@ export class UploadInstance extends EventEmitter { const handleStart = (error) => { if (error) { - this.collection._debug('[FilesCollection] [_Start] Error:', error); - this.emit('end', error); + Meteor.setTimeout(() => { + if (!Meteor.status().connected || `${error}` === 'Error: network' || `${error}` === 'Error: Connection lost') { + this.result.pause(); + } else if (this.result.state.get() !== 'aborted') { + this.collection._debug('[FilesCollection] [_Start] Error:', error); + this.emit('end', error); + } + }, 512); } else { this.result.continueFunc = () => { this.collection._debug('[FilesCollection] [insert] [continueFunc]'); - this.emit('createStreams'); + this.emit('upload'); }; - this.emit('createStreams'); + this.emit('upload'); } }; @@ -627,8 +602,12 @@ export class UploadInstance extends EventEmitter { 'x-start': '1', 'x-mtok': (helpers.isObject(Meteor.connection) ? Meteor.connection._lastSessionId : void 0) || null } - }).then(() => { - handleStart(); + }).then((response) => { + if (response.status === 204) { + handleStart(); + } else { + this.emit('end', new Meteor.Error(response.status, 'Can\'t start upload, make sure you\'re connected to the Internet. Reload the page or try again later.')); + } }).catch(handleStart); } }