[#1317] Update and run email consolidation script

.gitignore

@@ -19,3 +19,4 @@ localhost.key
 privkey.pem
 
 storybook-static
+duplicates.json

server/controllers/__mocks__/aws.controller.js

@@ -1,5 +1,5 @@
 export const getObjectKey = jest.mock();
 export const deleteObjectsFromS3 = jest.fn();
 export const signS3 = jest.fn();
-export const copyObjectInS3 = jest.fn();
+export const copyObjectInS3RequestHandler = jest.fn();
 export const listObjectsInS3ForUser = jest.fn();

server/controllers/aws.controller.js

@@ -98,24 +98,41 @@ export function signS3(req, res) {
   res.json(result);
 }
 
-export function copyObjectInS3(req, res) {
+export function copyObjectInS3(url, userId) {
-  const { url } = req.body;
+  return new Promise((resolve, reject) => {
-  const objectKey = getObjectKey(url);
+    const objectKey = getObjectKey(url);
-  const fileExtension = getExtension(objectKey);
+    const fileExtension = getExtension(objectKey);
-  const newFilename = uuid.v4() + fileExtension;
+    const newFilename = uuid.v4() + fileExtension;
-  const userId = req.user.id;
+    const headParams = {
-  const params = {
+      Bucket: `${process.env.S3_BUCKET}`,
-    Bucket: `${process.env.S3_BUCKET}`,
+      Key: `${objectKey}`
-    CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
+    };
-    Key: `${userId}/${newFilename}`,
+    client.s3.headObject(headParams, (headErr) => {
-    ACL: 'public-read'
+      if (headErr) {
-  };
+        reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
-  const copy = client.copyObject(params);
+        return;
-  copy.on('err', (err) => {
+      }
-    console.log(err);
+      const params = {
+        Bucket: `${process.env.S3_BUCKET}`,
+        CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
+        Key: `${userId}/${newFilename}`,
+        ACL: 'public-read'
+      };
+      const copy = client.copyObject(params);
+      copy.on('err', (err) => {
+        reject(err);
+      });
+      copy.on('end', (data) => {
+        resolve(`${s3Bucket}${userId}/${newFilename}`);
+      });
+    });
   });
-  copy.on('end', (data) => {
+}
-    res.json({ url: `${s3Bucket}${userId}/${newFilename}` });
+
+export function copyObjectInS3RequestHandler(req, res) {
+  const { url } = req.body;
+  copyObjectInS3(url, req.user.id).then((newUrl) => {
+    res.json({ url: newUrl });
   });
 }
 
@@ -124,19 +141,28 @@ export function moveObjectToUserInS3(url, userId) {
     const objectKey = getObjectKey(url);
     const fileExtension = getExtension(objectKey);
     const newFilename = uuid.v4() + fileExtension;
-    console.log(`${process.env.S3_BUCKET}/${objectKey}`);
+    const headParams = {
-    const params = {
       Bucket: `${process.env.S3_BUCKET}`,
-      CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
+      Key: `${objectKey}`
-      Key: `${userId}/${newFilename}`,
-      ACL: 'public-read'
     };
-    const move = client.moveObject(params);
+    client.s3.headObject(headParams, (headErr) => {
-    move.on('err', (err) => {
+      if (headErr) {
-      reject(err);
+        reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
-    });
+        return;
-    move.on('end', (data) => {
+      }
-      resolve(`${s3Bucket}${userId}/${newFilename}`);
+      const params = {
+        Bucket: `${process.env.S3_BUCKET}`,
+        CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
+        Key: `${userId}/${newFilename}`,
+        ACL: 'public-read'
+      };
+      const move = client.moveObject(params);
+      move.on('err', (err) => {
+        reject(err);
+      });
+      move.on('end', (data) => {
+        resolve(`${s3Bucket}${userId}/${newFilename}`);
+      });
     });
   });
 }

server/migrations/emailConsolidation.js

@@ -1,8 +1,9 @@
 import mongoose from 'mongoose';
+import fs from 'fs';
 import User from '../models/user';
 import Project from '../models/project';
 import Collection from '../models/collection';
-import { moveObjectToUserInS3 } from '../controllers/aws.controller';
+import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller';
 import mail from '../utils/mail';
 import { renderAccountConsolidation } from '../views/mail';
 
@@ -65,73 +66,172 @@ const agg = [
 // then, send the email
 // then, figure out how to iterate through all of the users.
 
-let currentUser = null;
+// create list of duplicate users
-let duplicates = null;
+// User.aggregate(agg).then((result) => {
-User.aggregate(agg).then((result) => {
+//   return fs.writeFile('duplicates.json', JSON.stringify(result), () => {
-  console.log(result);
+//     console.log('File written.');
-  const email = result[0]._id;
+//     process.exit(0);
-  return User.find({ email }).collation({ locale: 'en', strength: 2 })
+//   });
-    .sort({ createdAt: 1 }).exec();
-}).then((result) => {
-  [currentUser, ...duplicates] = result;
-  console.log('Current User: ', currentUser._id, ' ', currentUser.email);
-  duplicates = duplicates.map(dup => dup._id);
-  console.log('Duplicates: ', duplicates);
-  return Project.find({
-    user: { $in: duplicates }
-  }).exec();
-}).then((sketches) => {
-  const saveSketchPromises = [];
-  sketches.forEach((sketch) => {
-    const moveSketchFilesPromises = [];
-    sketch.files.forEach((file) => {
-      if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) {
-        const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
-          .then((newUrl) => {
-            file.url = newUrl;
-          });
-        moveSketchFilesPromises.push(fileSavePromise);
-      }
-    });
-    const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
-      sketch.user = ObjectId(currentUser._id);
-      return sketch.save();
-    });
-    saveSketchPromises.push(sketchSavePromise);
-  });
-  return Promise.all(saveSketchPromises);
-}).then(() => {
-  console.log('Moved and updated all sketches.');
-  return Collection.updateMany(
-    { owner: { $in: duplicates } },
-    { $set: { owner: ObjectId(currentUser.id) } }
-  );
-}).then(() => {
-  console.log('Moved and updated all collections.');
-  return User.deleteMany({ _id: { $in: duplicates } });
-}).then(() => {
-  console.log('Deleted other user accounts.');
-  currentUser.email = currentUser.email.toLowerCase();
-  return currentUser.save();
-}).then(() => {
-  console.log('Migrated email to lowercase.');
-  // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
-  const mailOptions = renderAccountConsolidation({
-    body: {
-      domain: 'https://editor.p5js.org',
-      username: currentUser.username,
-      email: currentUser.email
-    },
-    to: currentUser.email,
-  });
-
-  mail.send(mailOptions, (mailErr, result) => {
-    console.log('Sent email.');
-    process.exit(0);
-  });
-});
-
-// ).then((result) => {
-//   console.log(result);
 // });
 
+let currentUser = null;
+let duplicates = null;
+
+fs.readFile('duplicates.json', async (err, file) => {
+  const result = JSON.parse(file);
+  for (let i = 3000; i < result.length; i += 1) {
+    console.log('Index: ', i);
+    const email = result[i]._id;
+    console.log(email);
+    await consolidateAccount(email); // eslint-disable-line
+  }
+  process.exit(0);
+});
+
+async function consolidateAccount(email) {
+  return User.find({ email }).collation({ locale: 'en', strength: 2 })
+    .sort({ createdAt: 1 }).exec()
+    .then((result) => {
+      [currentUser, ...duplicates] = result;
+      console.log('Current User: ', currentUser._id, ' ', currentUser.email);
+      duplicates = duplicates.map(dup => dup._id);
+      console.log('Duplicates: ', duplicates);
+      return Project.find({
+        user: { $in: duplicates }
+      }).exec();
+    }).then((sketches) => {
+      const saveSketchPromises = [];
+      sketches.forEach((sketch) => {
+        console.log('SketchId: ', sketch._id);
+        console.log('UserId: ', sketch.user);
+        const moveSketchFilesPromises = [];
+        sketch.files.forEach((file) => {
+        // if the file url contains sketch user
+          if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) {
+            if (file.url.includes(sketch.user)) {
+              const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
+                .then((newUrl) => {
+                  file.url = newUrl;
+                }).catch((err) => {
+                  console.log('Move Error:');
+                  console.log(err);
+                });
+              moveSketchFilesPromises.push(fileSavePromise);
+            } else {
+              const fileSavePromise = copyObjectInS3(file.url, currentUser._id)
+                .then((newUrl) => {
+                  file.url = newUrl;
+                }).catch((err) => {
+                  console.log('Copy Error:');
+                  console.log(err);
+                });
+              moveSketchFilesPromises.push(fileSavePromise);
+            }
+          }
+        });
+        const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
+          sketch.user = ObjectId(currentUser._id);
+          return sketch.save();
+        });
+        saveSketchPromises.push(sketchSavePromise);
+      });
+      return Promise.all(saveSketchPromises);
+    }).then(() => {
+      console.log('Moved and updated all sketches.');
+      return Collection.updateMany(
+        { owner: { $in: duplicates } },
+        { $set: { owner: ObjectId(currentUser.id) } }
+      );
+    }).then(() => {
+      console.log('Moved and updated all collections.');
+      return User.deleteMany({ _id: { $in: duplicates } });
+    }).then(() => {
+      console.log('Deleted other user accounts.');
+      currentUser.email = currentUser.email.toLowerCase();
+      return currentUser.save();
+    }).then(() => {
+      console.log('Migrated email to lowercase.');
+      // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
+      const mailOptions = renderAccountConsolidation({
+        body: {
+          domain: 'https://editor.p5js.org',
+          username: currentUser.username,
+          email: currentUser.email
+        },
+        to: currentUser.email,
+      });
+
+      return new Promise((resolve, reject) => {
+        mail.send(mailOptions, (mailErr, result) => {
+          console.log('Sent email.');
+          if (mailErr) {
+            return reject(mailErr);
+          }
+          return resolve(result);
+        });
+      });
+    }).catch((err) => {
+      console.log(err);
+      process.exit(1);
+    });
+}
+
+
+// let duplicates = [
+//   "5ce3d936e0f9df0022d8330c",
+//   "5cff843f091745001e83c070",
+//     "5d246f5db489e6001eaee6e9"
+// ];
+// let currentUser = null;
+// User.deleteMany({ _id: { $in: duplicates } }).then(() => {
+//   return User.findOne({ "email": "Silverstar09@hotmail.com" })
+// }).then((result) => {
+//   currentUser = result;
+//   console.log('Deleted other user accounts.');
+//   currentUser.email = currentUser.email.toLowerCase();
+//   return currentUser.save();
+// }).then(() => {
+//   const mailOptions = renderAccountConsolidation({
+//     body: {
+//       domain: 'https://editor.p5js.org',
+//       username: currentUser.username,
+//       email: currentUser.email
+//     },
+//     to: currentUser.email,
+//   });
+
+//   return new Promise((resolve, reject) => {
+//     mail.send(mailOptions, (mailErr, result) => {
+//       console.log('Sent email.');
+//       if (mailErr) {
+//         return reject(mailErr);
+//       }
+//       return resolve(result);
+//     });
+//   });
+// });
+
+
+// import s3 from '@auth0/s3';
+
+// const client = s3.createClient({
+//   maxAsyncS3: 20,
+//   s3RetryCount: 3,
+//   s3RetryDelay: 1000,
+//   multipartUploadThreshold: 20971520, // this is the default (20 MB)
+//   multipartUploadSize: 15728640, // this is the default (15 MB)
+//   s3Options: {
+//     accessKeyId: `${process.env.AWS_ACCESS_KEY}`,
+//     secretAccessKey: `${process.env.AWS_SECRET_KEY}`,
+//     region: `${process.env.AWS_REGION}`
+//   },
+// });
+
+// const headParams = {
+//   Bucket: `${process.env.S3_BUCKET}`,
+//   Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png"
+// };
+// client.s3.headObject(headParams, (err, data) => {
+//   console.log(err);
+//   console.log(data);
+// });

server/migrations/start.js

@@ -1,7 +1,7 @@
 require('@babel/register');
 require('@babel/polyfill');
 const path = require('path');
-require('dotenv').config({ path: path.resolve('.env') });
+require('dotenv').config({ path: path.resolve('.env.production') });
 require('./emailConsolidation');
 // require('./populateTotalSize');
 // require('./moveBucket');

server/routes/aws.routes.js

@@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated';
 const router = new Router();
 
 router.post('/S3/sign', isAuthenticated, AWSController.signS3);
-router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3);
+router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler);
 router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3);
 router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);