[#1317] Update and run email consolidation script

.gitignore

@@ -19,3 +19,4 @@ localhost.key
 privkey.pem
 
 storybook-static
+duplicates.json

server/controllers/__mocks__/aws.controller.js

@@ -1,5 +1,5 @@
 export const getObjectKey = jest.mock();
 export const deleteObjectsFromS3 = jest.fn();
 export const signS3 = jest.fn();
-export const copyObjectInS3 = jest.fn();
+export const copyObjectInS3RequestHandler = jest.fn();
 export const listObjectsInS3ForUser = jest.fn();

server/controllers/aws.controller.js

@@ -98,12 +98,20 @@ export function signS3(req, res) {
   res.json(result);
 }
 
-export function copyObjectInS3(req, res) {
+export function copyObjectInS3(url, userId) {
-  const { url } = req.body;
+  return new Promise((resolve, reject) => {
     const objectKey = getObjectKey(url);
     const fileExtension = getExtension(objectKey);
     const newFilename = uuid.v4() + fileExtension;
-  const userId = req.user.id;
+    const headParams = {
+      Bucket: `${process.env.S3_BUCKET}`,
+      Key: `${objectKey}`
+    };
+    client.s3.headObject(headParams, (headErr) => {
+      if (headErr) {
+        reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
+        return;
+      }
       const params = {
         Bucket: `${process.env.S3_BUCKET}`,
         CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
@@ -112,10 +120,19 @@ export function copyObjectInS3(req, res) {
       };
       const copy = client.copyObject(params);
       copy.on('err', (err) => {
-    console.log(err);
+        reject(err);
       });
       copy.on('end', (data) => {
-    res.json({ url: `${s3Bucket}${userId}/${newFilename}` });
+        resolve(`${s3Bucket}${userId}/${newFilename}`);
+      });
+    });
+  });
+}
+
+export function copyObjectInS3RequestHandler(req, res) {
+  const { url } = req.body;
+  copyObjectInS3(url, req.user.id).then((newUrl) => {
+    res.json({ url: newUrl });
   });
 }
 
@@ -124,7 +141,15 @@ export function moveObjectToUserInS3(url, userId) {
     const objectKey = getObjectKey(url);
     const fileExtension = getExtension(objectKey);
     const newFilename = uuid.v4() + fileExtension;
-    console.log(`${process.env.S3_BUCKET}/${objectKey}`);
+    const headParams = {
+      Bucket: `${process.env.S3_BUCKET}`,
+      Key: `${objectKey}`
+    };
+    client.s3.headObject(headParams, (headErr) => {
+      if (headErr) {
+        reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
+        return;
+      }
       const params = {
         Bucket: `${process.env.S3_BUCKET}`,
         CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
@@ -139,6 +164,7 @@ export function moveObjectToUserInS3(url, userId) {
         resolve(`${s3Bucket}${userId}/${newFilename}`);
       });
     });
+  });
 }
 
 export function listObjectsInS3ForUser(userId) {

server/migrations/emailConsolidation.js

@@ -1,8 +1,9 @@
 import mongoose from 'mongoose';
+import fs from 'fs';
 import User from '../models/user';
 import Project from '../models/project';
 import Collection from '../models/collection';
-import { moveObjectToUserInS3 } from '../controllers/aws.controller';
+import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller';
 import mail from '../utils/mail';
 import { renderAccountConsolidation } from '../views/mail';
 
@@ -65,14 +66,32 @@ const agg = [
 // then, send the email
 // then, figure out how to iterate through all of the users.
 
+// create list of duplicate users
+// User.aggregate(agg).then((result) => {
+//   return fs.writeFile('duplicates.json', JSON.stringify(result), () => {
+//     console.log('File written.');
+//     process.exit(0);
+//   });
+// });
+
 let currentUser = null;
 let duplicates = null;
-User.aggregate(agg).then((result) => {
+
-  console.log(result);
+fs.readFile('duplicates.json', async (err, file) => {
-  const email = result[0]._id;
+  const result = JSON.parse(file);
+  for (let i = 3000; i < result.length; i += 1) {
+    console.log('Index: ', i);
+    const email = result[i]._id;
+    console.log(email);
+    await consolidateAccount(email); // eslint-disable-line
+  }
+  process.exit(0);
+});
+
+async function consolidateAccount(email) {
   return User.find({ email }).collation({ locale: 'en', strength: 2 })
-    .sort({ createdAt: 1 }).exec();
+    .sort({ createdAt: 1 }).exec()
-}).then((result) => {
+    .then((result) => {
       [currentUser, ...duplicates] = result;
       console.log('Current User: ', currentUser._id, ' ', currentUser.email);
       duplicates = duplicates.map(dup => dup._id);
@@ -80,17 +99,34 @@ User.aggregate(agg).then((result) => {
       return Project.find({
         user: { $in: duplicates }
       }).exec();
-}).then((sketches) => {
+    }).then((sketches) => {
       const saveSketchPromises = [];
       sketches.forEach((sketch) => {
+        console.log('SketchId: ', sketch._id);
+        console.log('UserId: ', sketch.user);
         const moveSketchFilesPromises = [];
         sketch.files.forEach((file) => {
-      if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) {
+        // if the file url contains sketch user
+          if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) {
+            if (file.url.includes(sketch.user)) {
               const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
                 .then((newUrl) => {
                   file.url = newUrl;
+                }).catch((err) => {
+                  console.log('Move Error:');
+                  console.log(err);
                 });
               moveSketchFilesPromises.push(fileSavePromise);
+            } else {
+              const fileSavePromise = copyObjectInS3(file.url, currentUser._id)
+                .then((newUrl) => {
+                  file.url = newUrl;
+                }).catch((err) => {
+                  console.log('Copy Error:');
+                  console.log(err);
+                });
+              moveSketchFilesPromises.push(fileSavePromise);
+            }
           }
         });
         const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
@@ -100,20 +136,20 @@ User.aggregate(agg).then((result) => {
         saveSketchPromises.push(sketchSavePromise);
       });
       return Promise.all(saveSketchPromises);
-}).then(() => {
+    }).then(() => {
       console.log('Moved and updated all sketches.');
       return Collection.updateMany(
         { owner: { $in: duplicates } },
         { $set: { owner: ObjectId(currentUser.id) } }
       );
-}).then(() => {
+    }).then(() => {
       console.log('Moved and updated all collections.');
       return User.deleteMany({ _id: { $in: duplicates } });
-}).then(() => {
+    }).then(() => {
       console.log('Deleted other user accounts.');
       currentUser.email = currentUser.email.toLowerCase();
       return currentUser.save();
-}).then(() => {
+    }).then(() => {
       console.log('Migrated email to lowercase.');
       // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
       const mailOptions = renderAccountConsolidation({
@@ -125,13 +161,77 @@ User.aggregate(agg).then((result) => {
         to: currentUser.email,
       });
 
+      return new Promise((resolve, reject) => {
         mail.send(mailOptions, (mailErr, result) => {
           console.log('Sent email.');
-    process.exit(0);
+          if (mailErr) {
+            return reject(mailErr);
+          }
+          return resolve(result);
         });
-});
+      });
+    }).catch((err) => {
+      console.log(err);
+      process.exit(1);
+    });
+}
 
-// ).then((result) => {
+
-//   console.log(result);
+// let duplicates = [
+//   "5ce3d936e0f9df0022d8330c",
+//   "5cff843f091745001e83c070",
+//     "5d246f5db489e6001eaee6e9"
+// ];
+// let currentUser = null;
+// User.deleteMany({ _id: { $in: duplicates } }).then(() => {
+//   return User.findOne({ "email": "Silverstar09@hotmail.com" })
+// }).then((result) => {
+//   currentUser = result;
+//   console.log('Deleted other user accounts.');
+//   currentUser.email = currentUser.email.toLowerCase();
+//   return currentUser.save();
+// }).then(() => {
+//   const mailOptions = renderAccountConsolidation({
+//     body: {
+//       domain: 'https://editor.p5js.org',
+//       username: currentUser.username,
+//       email: currentUser.email
+//     },
+//     to: currentUser.email,
 //   });
 
+//   return new Promise((resolve, reject) => {
+//     mail.send(mailOptions, (mailErr, result) => {
+//       console.log('Sent email.');
+//       if (mailErr) {
+//         return reject(mailErr);
+//       }
+//       return resolve(result);
+//     });
+//   });
+// });
+
+
+// import s3 from '@auth0/s3';
+
+// const client = s3.createClient({
+//   maxAsyncS3: 20,
+//   s3RetryCount: 3,
+//   s3RetryDelay: 1000,
+//   multipartUploadThreshold: 20971520, // this is the default (20 MB)
+//   multipartUploadSize: 15728640, // this is the default (15 MB)
+//   s3Options: {
+//     accessKeyId: `${process.env.AWS_ACCESS_KEY}`,
+//     secretAccessKey: `${process.env.AWS_SECRET_KEY}`,
+//     region: `${process.env.AWS_REGION}`
+//   },
+// });
+
+// const headParams = {
+//   Bucket: `${process.env.S3_BUCKET}`,
+//   Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png"
+// };
+// client.s3.headObject(headParams, (err, data) => {
+//   console.log(err);
+//   console.log(data);
+// });

server/migrations/start.js

@@ -1,7 +1,7 @@
 require('@babel/register');
 require('@babel/polyfill');
 const path = require('path');
-require('dotenv').config({ path: path.resolve('.env') });
+require('dotenv').config({ path: path.resolve('.env.production') });
 require('./emailConsolidation');
 // require('./populateTotalSize');
 // require('./moveBucket');

server/routes/aws.routes.js

@@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated';
 const router = new Router();
 
 router.post('/S3/sign', isAuthenticated, AWSController.signS3);
-router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3);
+router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler);
 router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3);
 router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);