diff --git a/.gitignore b/.gitignore index e36b805f..6fdf06c6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ localhost.key privkey.pem storybook-static +duplicates.json diff --git a/server/controllers/__mocks__/aws.controller.js b/server/controllers/__mocks__/aws.controller.js index 9e5709c4..061072a1 100644 --- a/server/controllers/__mocks__/aws.controller.js +++ b/server/controllers/__mocks__/aws.controller.js @@ -1,5 +1,5 @@ export const getObjectKey = jest.mock(); export const deleteObjectsFromS3 = jest.fn(); export const signS3 = jest.fn(); -export const copyObjectInS3 = jest.fn(); +export const copyObjectInS3RequestHandler = jest.fn(); export const listObjectsInS3ForUser = jest.fn(); diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index a331a98c..a315f1d4 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -98,24 +98,41 @@ export function signS3(req, res) { res.json(result); } -export function copyObjectInS3(req, res) { - const { url } = req.body; - const objectKey = getObjectKey(url); - const fileExtension = getExtension(objectKey); - const newFilename = uuid.v4() + fileExtension; - const userId = req.user.id; - const params = { - Bucket: `${process.env.S3_BUCKET}`, - CopySource: `${process.env.S3_BUCKET}/${objectKey}`, - Key: `${userId}/${newFilename}`, - ACL: 'public-read' - }; - const copy = client.copyObject(params); - copy.on('err', (err) => { - console.log(err); +export function copyObjectInS3(url, userId) { + return new Promise((resolve, reject) => { + const objectKey = getObjectKey(url); + const fileExtension = getExtension(objectKey); + const newFilename = uuid.v4() + fileExtension; + const headParams = { + Bucket: `${process.env.S3_BUCKET}`, + Key: `${objectKey}` + }; + client.s3.headObject(headParams, (headErr) => { + if (headErr) { + reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`)); + return; + } + const params = { + Bucket: `${process.env.S3_BUCKET}`, + CopySource: `${process.env.S3_BUCKET}/${objectKey}`, + Key: `${userId}/${newFilename}`, + ACL: 'public-read' + }; + const copy = client.copyObject(params); + copy.on('err', (err) => { + reject(err); + }); + copy.on('end', (data) => { + resolve(`${s3Bucket}${userId}/${newFilename}`); + }); + }); }); - copy.on('end', (data) => { - res.json({ url: `${s3Bucket}${userId}/${newFilename}` }); +} + +export function copyObjectInS3RequestHandler(req, res) { + const { url } = req.body; + copyObjectInS3(url, req.user.id).then((newUrl) => { + res.json({ url: newUrl }); }); } @@ -124,19 +141,28 @@ export function moveObjectToUserInS3(url, userId) { const objectKey = getObjectKey(url); const fileExtension = getExtension(objectKey); const newFilename = uuid.v4() + fileExtension; - console.log(`${process.env.S3_BUCKET}/${objectKey}`); - const params = { + const headParams = { Bucket: `${process.env.S3_BUCKET}`, - CopySource: `${process.env.S3_BUCKET}/${objectKey}`, - Key: `${userId}/${newFilename}`, - ACL: 'public-read' + Key: `${objectKey}` }; - const move = client.moveObject(params); - move.on('err', (err) => { - reject(err); - }); - move.on('end', (data) => { - resolve(`${s3Bucket}${userId}/${newFilename}`); + client.s3.headObject(headParams, (headErr) => { + if (headErr) { + reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`)); + return; + } + const params = { + Bucket: `${process.env.S3_BUCKET}`, + CopySource: `${process.env.S3_BUCKET}/${objectKey}`, + Key: `${userId}/${newFilename}`, + ACL: 'public-read' + }; + const move = client.moveObject(params); + move.on('err', (err) => { + reject(err); + }); + move.on('end', (data) => { + resolve(`${s3Bucket}${userId}/${newFilename}`); + }); }); }); } diff --git a/server/migrations/emailConsolidation.js b/server/migrations/emailConsolidation.js index d259d918..8343377c 100644 --- a/server/migrations/emailConsolidation.js +++ b/server/migrations/emailConsolidation.js @@ -1,8 +1,9 @@ import mongoose from 'mongoose'; +import fs from 'fs'; import User from '../models/user'; import Project from '../models/project'; import Collection from '../models/collection'; -import { moveObjectToUserInS3 } from '../controllers/aws.controller'; +import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller'; import mail from '../utils/mail'; import { renderAccountConsolidation } from '../views/mail'; @@ -65,73 +66,172 @@ const agg = [ // then, send the email // then, figure out how to iterate through all of the users. -let currentUser = null; -let duplicates = null; -User.aggregate(agg).then((result) => { - console.log(result); - const email = result[0]._id; - return User.find({ email }).collation({ locale: 'en', strength: 2 }) - .sort({ createdAt: 1 }).exec(); -}).then((result) => { - [currentUser, ...duplicates] = result; - console.log('Current User: ', currentUser._id, ' ', currentUser.email); - duplicates = duplicates.map(dup => dup._id); - console.log('Duplicates: ', duplicates); - return Project.find({ - user: { $in: duplicates } - }).exec(); -}).then((sketches) => { - const saveSketchPromises = []; - sketches.forEach((sketch) => { - const moveSketchFilesPromises = []; - sketch.files.forEach((file) => { - if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) { - const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) - .then((newUrl) => { - file.url = newUrl; - }); - moveSketchFilesPromises.push(fileSavePromise); - } - }); - const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => { - sketch.user = ObjectId(currentUser._id); - return sketch.save(); - }); - saveSketchPromises.push(sketchSavePromise); - }); - return Promise.all(saveSketchPromises); -}).then(() => { - console.log('Moved and updated all sketches.'); - return Collection.updateMany( - { owner: { $in: duplicates } }, - { $set: { owner: ObjectId(currentUser.id) } } - ); -}).then(() => { - console.log('Moved and updated all collections.'); - return User.deleteMany({ _id: { $in: duplicates } }); -}).then(() => { - console.log('Deleted other user accounts.'); - currentUser.email = currentUser.email.toLowerCase(); - return currentUser.save(); -}).then(() => { - console.log('Migrated email to lowercase.'); - // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; - const mailOptions = renderAccountConsolidation({ - body: { - domain: 'https://editor.p5js.org', - username: currentUser.username, - email: currentUser.email - }, - to: currentUser.email, - }); - - mail.send(mailOptions, (mailErr, result) => { - console.log('Sent email.'); - process.exit(0); - }); -}); - -// ).then((result) => { -// console.log(result); +// create list of duplicate users +// User.aggregate(agg).then((result) => { +// return fs.writeFile('duplicates.json', JSON.stringify(result), () => { +// console.log('File written.'); +// process.exit(0); +// }); // }); +let currentUser = null; +let duplicates = null; + +fs.readFile('duplicates.json', async (err, file) => { + const result = JSON.parse(file); + for (let i = 3000; i < result.length; i += 1) { + console.log('Index: ', i); + const email = result[i]._id; + console.log(email); + await consolidateAccount(email); // eslint-disable-line + } + process.exit(0); +}); + +async function consolidateAccount(email) { + return User.find({ email }).collation({ locale: 'en', strength: 2 }) + .sort({ createdAt: 1 }).exec() + .then((result) => { + [currentUser, ...duplicates] = result; + console.log('Current User: ', currentUser._id, ' ', currentUser.email); + duplicates = duplicates.map(dup => dup._id); + console.log('Duplicates: ', duplicates); + return Project.find({ + user: { $in: duplicates } + }).exec(); + }).then((sketches) => { + const saveSketchPromises = []; + sketches.forEach((sketch) => { + console.log('SketchId: ', sketch._id); + console.log('UserId: ', sketch.user); + const moveSketchFilesPromises = []; + sketch.files.forEach((file) => { + // if the file url contains sketch user + if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) { + if (file.url.includes(sketch.user)) { + const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) + .then((newUrl) => { + file.url = newUrl; + }).catch((err) => { + console.log('Move Error:'); + console.log(err); + }); + moveSketchFilesPromises.push(fileSavePromise); + } else { + const fileSavePromise = copyObjectInS3(file.url, currentUser._id) + .then((newUrl) => { + file.url = newUrl; + }).catch((err) => { + console.log('Copy Error:'); + console.log(err); + }); + moveSketchFilesPromises.push(fileSavePromise); + } + } + }); + const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => { + sketch.user = ObjectId(currentUser._id); + return sketch.save(); + }); + saveSketchPromises.push(sketchSavePromise); + }); + return Promise.all(saveSketchPromises); + }).then(() => { + console.log('Moved and updated all sketches.'); + return Collection.updateMany( + { owner: { $in: duplicates } }, + { $set: { owner: ObjectId(currentUser.id) } } + ); + }).then(() => { + console.log('Moved and updated all collections.'); + return User.deleteMany({ _id: { $in: duplicates } }); + }).then(() => { + console.log('Deleted other user accounts.'); + currentUser.email = currentUser.email.toLowerCase(); + return currentUser.save(); + }).then(() => { + console.log('Migrated email to lowercase.'); + // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; + const mailOptions = renderAccountConsolidation({ + body: { + domain: 'https://editor.p5js.org', + username: currentUser.username, + email: currentUser.email + }, + to: currentUser.email, + }); + + return new Promise((resolve, reject) => { + mail.send(mailOptions, (mailErr, result) => { + console.log('Sent email.'); + if (mailErr) { + return reject(mailErr); + } + return resolve(result); + }); + }); + }).catch((err) => { + console.log(err); + process.exit(1); + }); +} + + +// let duplicates = [ +// "5ce3d936e0f9df0022d8330c", +// "5cff843f091745001e83c070", +// "5d246f5db489e6001eaee6e9" +// ]; +// let currentUser = null; +// User.deleteMany({ _id: { $in: duplicates } }).then(() => { +// return User.findOne({ "email": "Silverstar09@hotmail.com" }) +// }).then((result) => { +// currentUser = result; +// console.log('Deleted other user accounts.'); +// currentUser.email = currentUser.email.toLowerCase(); +// return currentUser.save(); +// }).then(() => { +// const mailOptions = renderAccountConsolidation({ +// body: { +// domain: 'https://editor.p5js.org', +// username: currentUser.username, +// email: currentUser.email +// }, +// to: currentUser.email, +// }); + +// return new Promise((resolve, reject) => { +// mail.send(mailOptions, (mailErr, result) => { +// console.log('Sent email.'); +// if (mailErr) { +// return reject(mailErr); +// } +// return resolve(result); +// }); +// }); +// }); + + +// import s3 from '@auth0/s3'; + +// const client = s3.createClient({ +// maxAsyncS3: 20, +// s3RetryCount: 3, +// s3RetryDelay: 1000, +// multipartUploadThreshold: 20971520, // this is the default (20 MB) +// multipartUploadSize: 15728640, // this is the default (15 MB) +// s3Options: { +// accessKeyId: `${process.env.AWS_ACCESS_KEY}`, +// secretAccessKey: `${process.env.AWS_SECRET_KEY}`, +// region: `${process.env.AWS_REGION}` +// }, +// }); + +// const headParams = { +// Bucket: `${process.env.S3_BUCKET}`, +// Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png" +// }; +// client.s3.headObject(headParams, (err, data) => { +// console.log(err); +// console.log(data); +// }); diff --git a/server/migrations/start.js b/server/migrations/start.js index 3fc2e6e4..2e0e3d88 100644 --- a/server/migrations/start.js +++ b/server/migrations/start.js @@ -1,7 +1,7 @@ require('@babel/register'); require('@babel/polyfill'); const path = require('path'); -require('dotenv').config({ path: path.resolve('.env') }); +require('dotenv').config({ path: path.resolve('.env.production') }); require('./emailConsolidation'); // require('./populateTotalSize'); // require('./moveBucket'); diff --git a/server/routes/aws.routes.js b/server/routes/aws.routes.js index 850bb190..6bbf3795 100644 --- a/server/routes/aws.routes.js +++ b/server/routes/aws.routes.js @@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated'; const router = new Router(); router.post('/S3/sign', isAuthenticated, AWSController.signS3); -router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3); +router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler); router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3); router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);