diff --git a/client/components/cards/attachments.jade b/client/components/cards/attachments.jade index c01195ea1..975d21e84 100644 --- a/client/components/cards/attachments.jade +++ b/client/components/cards/attachments.jade @@ -72,3 +72,16 @@ template(name="attachmentActionsPopup") a.js-confirm-delete i.fa.fa-close | {{_ 'delete'}} + p.attachment-storage + | {{versions.original.storage}} + + if $neq versions.original.storage "fs" + a.js-move-storage-fs + i.fa.fa-arrow-right + | {{_ 'attachment-move-storage-fs'}} + + if $neq versions.original.storage "gridfs" + if versions.original.storage + a.js-move-storage-gridfs + i.fa.fa-arrow-right + | {{_ 'attachment-move-storage-gridfs'}} diff --git a/client/components/cards/attachments.js b/client/components/cards/attachments.js index fb39dce79..91ff81ae3 100644 --- a/client/components/cards/attachments.js +++ b/client/components/cards/attachments.js @@ -138,6 +138,14 @@ BlazeComponent.extendComponent({ Cards.findOne(this.data().meta.cardId).unsetCover(); Popup.back(); }, + 'click .js-move-storage-fs'() { + Meteor.call('moveToStorage', this.data()._id, "fs"); + Popup.back(); + }, + 'click .js-move-storage-gridfs'() { + Meteor.call('moveToStorage', this.data()._id, "gridfs"); + Popup.back(); + }, } ] } diff --git a/i18n/en.i18n.json b/i18n/en.i18n.json index 6ba37690e..cfdf89a5a 100644 --- a/i18n/en.i18n.json +++ b/i18n/en.i18n.json @@ -1165,5 +1165,7 @@ "copyChecklistPopup-title": "Copy Checklist", "card-show-lists": "Card Show Lists", "subtaskActionsPopup-title": "Subtask Actions", - "attachmentActionsPopup-title": "Attachment Actions" + "attachmentActionsPopup-title": "Attachment Actions", + "attachment-move-storage-fs": "Move attachment to filesystem", + "attachment-move-storage-gridfs": "Move attachment to GridFS" } diff --git a/models/attachments.js b/models/attachments.js index 6d2e95f0e..38c3566dd 100644 --- a/models/attachments.js +++ b/models/attachments.js @@ -2,30 +2,7 @@ import { Meteor } from 'meteor/meteor'; import { FilesCollection } from 'meteor/ostrio:files'; import fs from 'fs'; import path from 'path'; -import { createBucket } from './lib/grid/createBucket'; -import { createOnAfterUpload } from './lib/fsHooks/createOnAfterUpload'; -import { createInterceptDownload } from './lib/fsHooks/createInterceptDownload'; -import { createOnAfterRemove } from './lib/fsHooks/createOnAfterRemove'; - -let attachmentBucket; -if (Meteor.isServer) { - attachmentBucket = createBucket('attachments'); -} - -const insertActivity = (fileObj, activityType) => - Activities.insert({ - userId: fileObj.userId, - type: 'card', - activityType, - attachmentId: fileObj._id, - // this preserves the name so that notifications can be meaningful after - // this file is removed - attachmentName: fileObj.name, - boardId: fileObj.meta.boardId, - cardId: fileObj.meta.cardId, - listId: fileObj.meta.listId, - swimlaneId: fileObj.meta.swimlaneId, - }); +import AttachmentStoreStrategy from '/models/lib/attachmentStoreStrategy'; // XXX Enforce a schema for the Attachments FilesCollection // see: https://github.com/VeliovGroup/Meteor-Files/wiki/Schema @@ -40,20 +17,20 @@ Attachments = new FilesCollection({ } return path.normalize(`assets/app/uploads/${this.collectionName}`); }, - onAfterUpload: function onAfterUpload(fileRef) { - createOnAfterUpload(attachmentBucket).call(this, fileRef); - // If the attachment doesn't have a source field - // or its source is different than import - if (!fileRef.meta.source || fileRef.meta.source !== 'import') { - // Add activity about adding the attachment - insertActivity(fileRef, 'addAttachment'); - } + onAfterUpload(fileObj) { + Object.keys(fileObj.versions).forEach(versionName => { + AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).onAfterUpload(); + }) }, - interceptDownload: createInterceptDownload(attachmentBucket), - onAfterRemove: function onAfterRemove(files) { - createOnAfterRemove(attachmentBucket).call(this, files); + interceptDownload(http, fileObj, versionName) { + const ret = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).interceptDownload(http); + return ret; + }, + onAfterRemove(files) { files.forEach(fileObj => { - insertActivity(fileObj, 'deleteAttachment'); + Object.keys(fileObj.versions).forEach(versionName => { + AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).onAfterRemove(); + }); }); }, // We authorize the attachment download either: @@ -82,6 +59,45 @@ if (Meteor.isServer) { fetch: ['meta'], }); + Meteor.methods({ + moveToStorage(fileObjId, storageDestination) { + check(fileObjId, String); + check(storageDestination, String); + + const fileObj = Attachments.findOne({_id: fileObjId}); + + Object.keys(fileObj.versions).forEach(versionName => { + const strategyRead = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName); + const strategyWrite = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName, storageDestination); + + if (strategyRead.constructor.name != strategyWrite.constructor.name) { + const readStream = strategyRead.getReadStream(); + const writeStream = strategyWrite.getWriteStream(); + + writeStream.on('error', error => { + console.error('[writeStream error]: ', error, fileObjId); + }); + + readStream.on('error', error => { + console.error('[readStream error]: ', error, fileObjId); + }); + + writeStream.on('finish', Meteor.bindEnvironment((finishedData) => { + strategyWrite.writeStreamFinished(finishedData); + })); + + // https://forums.meteor.com/t/meteor-code-must-always-run-within-a-fiber-try-wrapping-callbacks-that-you-pass-to-non-meteor-libraries-with-meteor-bindenvironmen/40099/8 + readStream.on('end', Meteor.bindEnvironment(() => { + Attachments.update({ _id: fileObj._id }, { $set: { [`versions.${versionName}.storage`]: strategyWrite.getStorageName() } }); + strategyRead.unlink(); + })); + + readStream.pipe(writeStream); + } + }); + }, + }); + Meteor.startup(() => { Attachments.collection._ensureIndex({ 'meta.cardId': 1 }); const storagePath = Attachments.storagePath(); diff --git a/models/avatars.js b/models/avatars.js index ff47aa150..7e9fe0e05 100644 --- a/models/avatars.js +++ b/models/avatars.js @@ -28,9 +28,22 @@ Avatars = new FilesCollection({ } return 'avatar-too-big'; }, - onAfterUpload: createOnAfterUpload(avatarsBucket), - interceptDownload: createInterceptDownload(avatarsBucket), - onAfterRemove: createOnAfterRemove(avatarsBucket), + onAfterUpload(fileObj) { + Object.keys(fileObj.versions).forEach(versionName => { + createOnAfterUpload(this, avatarsBucket, fileObj, versionName); + }); + }, + interceptDownload(http, fileObj, versionName) { + const ret = createInterceptDownload(this, avatarsBucket, fileObj, http, versionName); + return ret; + }, + onAfterRemove(files) { + files.forEach(fileObj => { + Object.keys(fileObj.versions).forEach(versionName => { + createOnAfterRemove(this, avatarsBucket, fileObj, versionName); + }); + }); + }, }); function isOwner(userId, doc) { diff --git a/models/lib/attachmentStoreStrategy.js b/models/lib/attachmentStoreStrategy.js new file mode 100644 index 000000000..c86aa9e1e --- /dev/null +++ b/models/lib/attachmentStoreStrategy.js @@ -0,0 +1,246 @@ +import fs from 'fs'; +import { createBucket } from './grid/createBucket'; +import { createObjectId } from './grid/createObjectId'; +import { createOnAfterUpload } from './fsHooks/createOnAfterUpload'; +import { createInterceptDownload } from './fsHooks/createInterceptDownload'; +import { createOnAfterRemove } from './fsHooks/createOnAfterRemove'; + +const insertActivity = (fileObj, activityType) => + Activities.insert({ + userId: fileObj.userId, + type: 'card', + activityType, + attachmentId: fileObj._id, + attachmentName: fileObj.name, + boardId: fileObj.meta.boardId, + cardId: fileObj.meta.cardId, + listId: fileObj.meta.listId, + swimlaneId: fileObj.meta.swimlaneId, + }); + +let attachmentBucket; +if (Meteor.isServer) { + attachmentBucket = createBucket('attachments'); +} + +/** Strategy to store attachments */ +class AttachmentStoreStrategy { + + /** constructor + * @param filesCollection the current FilesCollection instance + * @param fileObj the current file object + * @param versionName the current version + */ + constructor(filesCollection, fileObj, versionName) { + this.filesCollection = filesCollection; + this.fileObj = fileObj; + this.versionName = versionName; + } + + /** after successfull upload */ + onAfterUpload() { + // If the attachment doesn't have a source field or its source is different than import + if (!this.fileObj.meta.source || this.fileObj.meta.source !== 'import') { + // Add activity about adding the attachment + insertActivity(this.fileObj, 'addAttachment'); + } + } + + /** download the file + * @param http the current http request + */ + interceptDownload(http) { + } + + /** after file remove */ + onAfterRemove() { + insertActivity(this.fileObj, 'deleteAttachment'); + } + + /** returns a read stream + * @return the read stream + */ + getReadStream() { + } + + /** returns a write stream + * @return the write stream + */ + getWriteStream() { + } + + /** writing finished + * @param finishedData the data of the write stream finish event + */ + writeStreamFinished(finishedData) { + } + + /** remove the file */ + unlink() { + } + + /** return the storage name + * @return the storage name + */ + getStorageName() { + } + + static getFileStrategy(filesCollection, fileObj, versionName, storage) { + if (!storage) { + storage = fileObj.versions[versionName].storage || "gridfs"; + } + let ret; + if (["fs", "gridfs"].includes(storage)) { + if (storage == "fs") { + ret = new AttachmentStoreStrategyFilesystem(filesCollection, fileObj, versionName); + } else if (storage == "gridfs") { + ret = new AttachmentStoreStrategyGridFs(filesCollection, fileObj, versionName); + } + } + console.log("getFileStrategy: ", ret.constructor.name); + return ret; + } +} + +/** Strategy to store attachments at GridFS (MongoDB) */ +class AttachmentStoreStrategyGridFs extends AttachmentStoreStrategy { + + /** constructor + * @param filesCollection the current FilesCollection instance + * @param fileObj the current file object + * @param versionName the current version + */ + constructor(filesCollection, fileObj, versionName) { + super(filesCollection, fileObj, versionName); + } + + /** after successfull upload */ + onAfterUpload() { + createOnAfterUpload(this.filesCollection, attachmentBucket, this.fileObj, this.versionName); + super.onAfterUpload(); + } + + /** download the file + * @param http the current http request + */ + interceptDownload(http) { + const ret = createInterceptDownload(this.filesCollection, attachmentBucket, this.fileObj, http, this.versionName); + return ret; + } + + /** after file remove */ + onAfterRemove() { + this.unlink(); + super.onAfterRemove(); + } + + /** returns a read stream + * @return the read stream + */ + getReadStream() { + const gridFsFileId = (this.fileObj.versions[this.versionName].meta || {}) + .gridFsFileId; + let ret; + if (gridFsFileId) { + const gfsId = createObjectId({ gridFsFileId }); + ret = attachmentBucket.openDownloadStream(gfsId); + } + return ret; + } + + /** returns a write stream + * @return the write stream + */ + getWriteStream() { + const fileObj = this.fileObj; + const versionName = this.versionName; + const metadata = { ...fileObj.meta, versionName, fileId: fileObj._id }; + const ret = attachmentBucket.openUploadStream(this.fileObj.name, { + contentType: fileObj.type || 'binary/octet-stream', + metadata, + }); + return ret; + } + + /** writing finished + * @param finishedData the data of the write stream finish event + */ + writeStreamFinished(finishedData) { + const gridFsFileIdName = this.getGridFsFileIdName(); + Attachments.update({ _id: this.fileObj._id }, { $set: { [gridFsFileIdName]: finishedData._id.toHexString(), } }); + } + + /** remove the file */ + unlink() { + createOnAfterRemove(this.filesCollection, attachmentBucket, this.fileObj, this.versionName); + const gridFsFileIdName = this.getGridFsFileIdName(); + Attachments.update({ _id: this.fileObj._id }, { $unset: { [gridFsFileIdName]: 1 } }); + } + + /** return the storage name + * @return the storage name + */ + getStorageName() { + return "gridfs"; + } + + /** returns the property name of gridFsFileId + * @return the property name of gridFsFileId + */ + getGridFsFileIdName() { + const ret = `versions.${this.versionName}.meta.gridFsFileId`; + return ret; + } +} + +/** Strategy to store attachments at filesystem */ +class AttachmentStoreStrategyFilesystem extends AttachmentStoreStrategy { + + /** constructor + * @param filesCollection the current FilesCollection instance + * @param fileObj the current file object + * @param versionName the current version + */ + constructor(filesCollection, fileObj, versionName) { + super(filesCollection, fileObj, versionName); + } + + /** returns a read stream + * @return the read stream + */ + getReadStream() { + const ret = fs.createReadStream(this.fileObj.versions[this.versionName].path) + return ret; + } + + /** returns a write stream + * @return the write stream + */ + getWriteStream() { + const newFileName = this.fileObj.name; + const filePath = this.fileObj.versions[this.versionName].path; + const ret = fs.createWriteStream(filePath); + return ret; + } + + /** writing finished + * @param finishedData the data of the write stream finish event + */ + writeStreamFinished(finishedData) { + } + + /** remove the file */ + unlink() { + const filePath = this.fileObj.versions[this.versionName].path; + fs.unlink(filePath, () => {}); + } + + /** return the storage name + * @return the storage name + */ + getStorageName() { + return "fs"; + } +} + +export default AttachmentStoreStrategy; diff --git a/models/lib/fsHooks/createInterceptDownload.js b/models/lib/fsHooks/createInterceptDownload.js index c375d03f8..1d094c3f9 100644 --- a/models/lib/fsHooks/createInterceptDownload.js +++ b/models/lib/fsHooks/createInterceptDownload.js @@ -1,7 +1,7 @@ import { createObjectId } from '../grid/createObjectId'; -export const createInterceptDownload = bucket => - function interceptDownload(http, file, versionName) { +export const createInterceptDownload = + function interceptDownload(filesCollection, bucket, file, http, versionName) { const { gridFsFileId } = file.versions[versionName].meta || {}; if (gridFsFileId) { // opens the download stream using a given gfs id @@ -24,7 +24,7 @@ export const createInterceptDownload = bucket => http.response.end('not found'); }); - http.response.setHeader('Cache-Control', this.cacheControl); + http.response.setHeader('Cache-Control', filesCollection.cacheControl); http.response.setHeader( 'Content-Disposition', getContentDisposition(file.name, http?.params?.query?.download), diff --git a/models/lib/fsHooks/createOnAfterRemove.js b/models/lib/fsHooks/createOnAfterRemove.js index f2e0a4ba7..446cf1f0b 100644 --- a/models/lib/fsHooks/createOnAfterRemove.js +++ b/models/lib/fsHooks/createOnAfterRemove.js @@ -1,17 +1,12 @@ import { createObjectId } from '../grid/createObjectId'; -export const createOnAfterRemove = bucket => - function onAfterRemove(files) { - files.forEach(file => { - Object.keys(file.versions).forEach(versionName => { - const gridFsFileId = (file.versions[versionName].meta || {}) - .gridFsFileId; - if (gridFsFileId) { - const gfsId = createObjectId({ gridFsFileId }); - bucket.delete(gfsId, err => { - // if (err) console.error(err); - }); - } +export const createOnAfterRemove = + function onAfterRemove(filesCollection, bucket, file, versionName) { + const gridFsFileId = (file.versions[versionName].meta || {}) + .gridFsFileId; + if (gridFsFileId) { + const gfsId = createObjectId({ gridFsFileId }); + bucket.delete(gfsId, err => { }); - }); + } }; diff --git a/models/lib/fsHooks/createOnAfterUpload.js b/models/lib/fsHooks/createOnAfterUpload.js index 3c6c54036..173d02f3c 100644 --- a/models/lib/fsHooks/createOnAfterUpload.js +++ b/models/lib/fsHooks/createOnAfterUpload.js @@ -1,51 +1,42 @@ import { Meteor } from 'meteor/meteor'; import fs from 'fs'; -export const createOnAfterUpload = bucket => - function onAfterUpload(file) { - const self = this; +export const createOnAfterUpload = function onAfterUpload(filesCollection, bucket, file, versionName) { + const self = filesCollection; + const metadata = { ...file.meta, versionName, fileId: file._id }; + fs.createReadStream(file.versions[versionName].path) - // here you could manipulate your file - // and create a new version, for example a scaled 'thumbnail' - // ... + // this is where we upload the binary to the bucket using bucket.openUploadStream + // see http://mongodb.github.io/node-mongodb-native/3.2/api/GridFSBucket.html#openUploadStream + .pipe( + bucket.openUploadStream(file.name, { + contentType: file.type || 'binary/octet-stream', + metadata, + }), + ) - // then we read all versions we have got so far - Object.keys(file.versions).forEach(versionName => { - const metadata = { ...file.meta, versionName, fileId: file._id }; - fs.createReadStream(file.versions[versionName].path) + // and we unlink the file from the fs on any error + // that occurred during the upload to prevent zombie files + .on('error', err => { + console.error("[createOnAfterUpload error]", err); + self.unlink(self.collection.findOne(file._id), versionName); // Unlink files from FS + }) - // this is where we upload the binary to the bucket using bucket.openUploadStream - // see http://mongodb.github.io/node-mongodb-native/3.2/api/GridFSBucket.html#openUploadStream - .pipe( - bucket.openUploadStream(file.name, { - contentType: file.type || 'binary/octet-stream', - metadata, - }), - ) + // once we are finished, we attach the gridFS Object id on the + // FilesCollection document's meta section and finally unlink the + // upload file from the filesystem + .on( + 'finish', + Meteor.bindEnvironment(ver => { + const property = `versions.${versionName}.meta.gridFsFileId`; - // and we unlink the file from the fs on any error - // that occurred during the upload to prevent zombie files - .on('error', err => { - console.error("[createOnAfterUpload error]", err); - self.unlink(this.collection.findOne(file._id), versionName); // Unlink files from FS - }) + self.collection.update(file._id, { + $set: { + [property]: ver._id.toHexString(), + }, + }); - // once we are finished, we attach the gridFS Object id on the - // FilesCollection document's meta section and finally unlink the - // upload file from the filesystem - .on( - 'finish', - Meteor.bindEnvironment(ver => { - const property = `versions.${versionName}.meta.gridFsFileId`; - - self.collection.update(file._id, { - $set: { - [property]: ver._id.toHexString(), - }, - }); - - self.unlink(this.collection.findOne(file._id), versionName); // Unlink files from FS - }), - ); - }); - }; + self.unlink(self.collection.findOne(file._id), versionName); // Unlink files from FS + }), + ); +};