migration: extract files from old storage and load into new storage

models/attachments_old.js

@@ -0,0 +1,116 @@
+const storeName = 'attachments';
+const defaultStoreOptions = {
+  beforeWrite: fileObj => {
+    if (!fileObj.isImage()) {
+      return {
+        type: 'application/octet-stream',
+      };
+    }
+    return {};
+  },
+};
+let store;
+store = new FS.Store.GridFS(storeName, {
+  // XXX Add a new store for cover thumbnails so we don't load big images in
+  // the general board view
+  // If the uploaded document is not an image we need to enforce browser
+  // download instead of execution. This is particularly important for HTML
+  // files that the browser will just execute if we don't serve them with the
+  // appropriate `application/octet-stream` MIME header which can lead to user
+  // data leaks. I imagine other formats (like PDF) can also be attack vectors.
+  // See https://github.com/wekan/wekan/issues/99
+  // XXX Should we use `beforeWrite` option of CollectionFS instead of
+  // collection-hooks?
+  // We should use `beforeWrite`.
+  ...defaultStoreOptions,
+});
+AttachmentsOld = new FS.Collection('attachments', {
+  stores: [store],
+});
+
+if (Meteor.isServer) {
+  Meteor.startup(() => {
+    AttachmentsOld.files._ensureIndex({ cardId: 1 });
+  });
+
+  AttachmentsOld.allow({
+    insert(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    update(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    remove(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    // We authorize the attachment download either:
+    // - if the board is public, everyone (even unconnected) can download it
+    // - if the board is private, only board members can download it
+    download(userId, doc) {
+      const board = Boards.findOne(doc.boardId);
+      if (board.isPublic()) {
+        return true;
+      } else {
+        return board.hasMember(userId);
+      }
+    },
+
+    fetch: ['boardId'],
+  });
+}
+
+// XXX Enforce a schema for the AttachmentsOld CollectionFS
+
+if (Meteor.isServer) {
+  AttachmentsOld.files.after.insert((userId, doc) => {
+    // If the attachment doesn't have a source field
+    // or its source is different than import
+    if (!doc.source || doc.source !== 'import') {
+      // Add activity about adding the attachment
+      Activities.insert({
+        userId,
+        type: 'card',
+        activityType: 'addAttachment',
+        attachmentId: doc._id,
+        // this preserves the name so that notifications can be meaningful after
+        // this file is removed
+        attachmentName: doc.original.name,
+        boardId: doc.boardId,
+        cardId: doc.cardId,
+        listId: doc.listId,
+        swimlaneId: doc.swimlaneId,
+      });
+    } else {
+      // Don't add activity about adding the attachment as the activity
+      // be imported and delete source field
+      AttachmentsOld.update(
+        {
+          _id: doc._id,
+        },
+        {
+          $unset: {
+            source: '',
+          },
+        },
+      );
+    }
+  });
+
+  AttachmentsOld.files.before.remove((userId, doc) => {
+    Activities.insert({
+      userId,
+      type: 'card',
+      activityType: 'deleteAttachment',
+      attachmentId: doc._id,
+      // this preserves the name so that notifications can be meaningful after
+      // this file is removed
+      attachmentName: doc.original.name,
+      boardId: doc.boardId,
+      cardId: doc.cardId,
+      listId: doc.listId,
+      swimlaneId: doc.swimlaneId,
+    });
+  });
+}
+
+export default AttachmentsOld;

models/avatars_old.js

@@ -0,0 +1,29 @@
+AvatarsOld = new FS.Collection('avatars', {
+  stores: [new FS.Store.GridFS('avatars')],
+  filter: {
+    maxSize: 72000,
+    allow: {
+      contentTypes: ['image/*'],
+    },
+  },
+});
+
+function isOwner(userId, file) {
+  return userId && userId === file.userId;
+}
+
+AvatarsOld.allow({
+  insert: isOwner,
+  update: isOwner,
+  remove: isOwner,
+  download() {
+    return true;
+  },
+  fetch: ['userId'],
+});
+
+AvatarsOld.files.before.insert((userId, doc) => {
+  doc.userId = userId;
+});
+
+export default AvatarsOld;

server/migrations.js

@@ -5,6 +5,9 @@ import Actions from '../models/actions';
 import Activities from '../models/activities';
 import Announcements from '../models/announcements';
 import Attachments from '../models/attachments';
+import AttachmentsOld from '../models/attachments_old';
+import Avatars from '../models/avatars';
+import AvatarsOld from '../models/avatars_old';
 import Boards from '../models/boards';
 import CardComments from '../models/cardComments';
 import Cards from '../models/cards';
@@ -1125,37 +1128,132 @@ Migrations.add('add-card-details-show-lists', () => {
   );
 });
 
-Migrations.add(
-  'adapt-attachments-to-ostrio-files-api-using-meta-and-drp-cfs-leacy',
-  () => {
-    Attachments.find().forEach(file => {
-      Attachments.update(
-        file._id,
-        {
-          $set: {
-            'meta.boardId': file.boardId,
-            'meta.cardId': file.cardId,
-            'meta.listId': file.listId,
-            'meta.swimlaneId': file.swimlaneId,
-          },
-        },
-        noValidate,
-      );
+Migrations.add('migrate-attachments-collectionFS-to-ostrioFiles', () => {
+  AttachmentsOld.find().forEach(function(fileObj) {
+    //console.log('File: ', fileObj.userId);
+
+    // This directory must be writable on server, so a test run first
+    // We are going to copy the files locally, then move them to S3
+    const fileName = `./assets/app/uploads/attachments/${fileObj.name()}`;
+    const newFileName = fileObj.name();
+
+    // This is "example" variable, change it to the userId that you might be using.
+    const userId = fileObj.userId;
+
+    const fileType = fileObj.type();
+    const fileSize = fileObj.size();
+    const fileId = fileObj._id;
+
+    const readStream = fileObj.createReadStream('attachments');
+    const writeStream = fs.createWriteStream(fileName);
+
+    writeStream.on('error', function(err) {
+      console.log('Writing error: ', err, fileName);
     });
-    Attachments.update(
-      {},
-      {
-        $unset: {
-          original: '', // cfs:* legacy
-          copies: '', // cfs:* legacy
-          failures: '', // cfs:* legacy
-          boardId: '',
-          cardId: '',
-          listId: '',
-          swimlaneId: '',
+
+    // Once we have a file, then upload it to our new data storage
+    readStream.on('end', () => {
+      console.log('Ended: ', fileName);
+      // UserFiles is the new Meteor-Files/FilesCollection collection instance
+
+      Attachments.addFile(
+        fileName,
+        {
+          fileName: newFileName,
+          type: fileType,
+          meta: {
+            boardId: fileObj.boardId,
+            cardId: fileObj.cardId,
+            listId: fileObj.listId,
+            swimlaneId: fileObj.swimlaneId,
+          },
+          userId,
+          size: fileSize,
+          fileId,
         },
-      },
-      noValidateMulti,
-    );
-  },
-);
+        (err, fileRef) => {
+          if (err) {
+            console.log(err);
+          } else {
+            console.log('File Inserted: ', fileRef._id);
+            // Set the userId again
+            Attachments.update({ _id: fileRef._id }, { $set: { userId } });
+            fileObj.remove();
+          }
+        },
+        true,
+      ); // proceedAfterUpload
+    });
+
+    readStream.on('error', error => {
+      console.log('Error: ', fileName, error);
+    });
+
+    readStream.pipe(writeStream);
+  });
+});
+
+Migrations.add('migrate-avatars-collectionFS-to-ostrioFiles', () => {
+  AvatarsOld.find().forEach(function(fileObj) {
+    //console.log('File: ', fileObj.userId);
+
+    // This directory must be writable on server, so a test run first
+    // We are going to copy the files locally, then move them to S3
+    const fileName = `./assets/app/uploads/avatars/${fileObj.name()}`;
+    const newFileName = fileObj.name();
+
+    // This is "example" variable, change it to the userId that you might be using.
+    const userId = fileObj.userId;
+
+    const fileType = fileObj.type();
+    const fileSize = fileObj.size();
+    const fileId = fileObj._id;
+
+    const readStream = fileObj.createReadStream('avatars');
+    const writeStream = fs.createWriteStream(fileName);
+
+    writeStream.on('error', function(err) {
+      console.log('Writing error: ', err, fileName);
+    });
+
+    // Once we have a file, then upload it to our new data storage
+    readStream.on('end', () => {
+      console.log('Ended: ', fileName);
+      // UserFiles is the new Meteor-Files/FilesCollection collection instance
+
+      Avatars.addFile(
+        fileName,
+        {
+          fileName: newFileName,
+          type: fileType,
+          meta: {
+            boardId: fileObj.boardId,
+            cardId: fileObj.cardId,
+            listId: fileObj.listId,
+            swimlaneId: fileObj.swimlaneId,
+          },
+          userId,
+          size: fileSize,
+          fileId,
+        },
+        (err, fileRef) => {
+          if (err) {
+            console.log(err);
+          } else {
+            console.log('File Inserted: ', fileRef._id);
+            // Set the userId again
+            Avatars.update({ _id: fileRef._id }, { $set: { userId } });
+            fileObj.remove();
+          }
+        },
+        true,
+      ); // proceedAfterUpload
+    });
+
+    readStream.on('error', error => {
+      console.log('Error: ', fileName, error);
+    });
+
+    readStream.pipe(writeStream);
+  });
+});