r
/
p5.js-web-editor
1
0
Fork 0
Code Issues 1 Pull Requests Releases 1 Wiki Activity

[#1317] Update and run email consolidation script

This commit is contained in:
Cassie Tarakajian 2020-07-17 19:09:23 -04:00
parent cd5c000d6d
commit 2b3be3accc
6 changed files with 227 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -19,3 +19,4 @@ localhost.key
privkey.pem
storybook-static
duplicates.json

2
server/controllers/__mocks__/aws.controller.js
View File

@ -1,5 +1,5 @@
export const getObjectKey = jest.mock();
export const deleteObjectsFromS3 = jest.fn();
export const signS3 = jest.fn();
export const copyObjectInS3 = jest.fn();
export const copyObjectInS3RequestHandler = jest.fn();
export const listObjectsInS3ForUser = jest.fn();

82
server/controllers/aws.controller.js
View File

@ -98,24 +98,41 @@ export function signS3(req, res) {
res.json(result);
}
export function copyObjectInS3(req, res) {
const { url } = req.body;
const objectKey = getObjectKey(url);
const fileExtension = getExtension(objectKey);
const newFilename = uuid.v4() + fileExtension;
const userId = req.user.id;
const params = {
Bucket: `${process.env.S3_BUCKET}`,
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
Key: `${userId}/${newFilename}`,
ACL: 'public-read'
};
const copy = client.copyObject(params);
copy.on('err', (err) => {
console.log(err);
export function copyObjectInS3(url, userId) {
return new Promise((resolve, reject) => {
const objectKey = getObjectKey(url);
const fileExtension = getExtension(objectKey);
const newFilename = uuid.v4() + fileExtension;
const headParams = {
Bucket: `${process.env.S3_BUCKET}`,
Key: `${objectKey}`
};
client.s3.headObject(headParams, (headErr) => {
if (headErr) {
reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
return;
}
const params = {
Bucket: `${process.env.S3_BUCKET}`,
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
Key: `${userId}/${newFilename}`,
ACL: 'public-read'
};
const copy = client.copyObject(params);
copy.on('err', (err) => {
reject(err);
});
copy.on('end', (data) => {
resolve(`${s3Bucket}${userId}/${newFilename}`);
});
});
});
copy.on('end', (data) => {
res.json({ url: `${s3Bucket}${userId}/${newFilename}` });
}
export function copyObjectInS3RequestHandler(req, res) {
const { url } = req.body;
copyObjectInS3(url, req.user.id).then((newUrl) => {
res.json({ url: newUrl });
});
}
@ -124,19 +141,28 @@ export function moveObjectToUserInS3(url, userId) {
const objectKey = getObjectKey(url);
const fileExtension = getExtension(objectKey);
const newFilename = uuid.v4() + fileExtension;
console.log(`${process.env.S3_BUCKET}/${objectKey}`);
const params = {
const headParams = {
Bucket: `${process.env.S3_BUCKET}`,
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
Key: `${userId}/${newFilename}`,
ACL: 'public-read'
Key: `${objectKey}`
};
const move = client.moveObject(params);
move.on('err', (err) => {
reject(err);
});
move.on('end', (data) => {
resolve(`${s3Bucket}${userId}/${newFilename}`);
client.s3.headObject(headParams, (headErr) => {
if (headErr) {
reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`));
return;
}
const params = {
Bucket: `${process.env.S3_BUCKET}`,
CopySource: `${process.env.S3_BUCKET}/${objectKey}`,
Key: `${userId}/${newFilename}`,
ACL: 'public-read'
};
const move = client.moveObject(params);
move.on('err', (err) => {
reject(err);
});
move.on('end', (data) => {
resolve(`${s3Bucket}${userId}/${newFilename}`);
});
});
});
}

238
server/migrations/emailConsolidation.js
View File

@ -1,8 +1,9 @@
import mongoose from 'mongoose';
import fs from 'fs';
import User from '../models/user';
import Project from '../models/project';
import Collection from '../models/collection';
import { moveObjectToUserInS3 } from '../controllers/aws.controller';
import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller';
import mail from '../utils/mail';
import { renderAccountConsolidation } from '../views/mail';
@ -65,73 +66,172 @@ const agg = [
// then, send the email
// then, figure out how to iterate through all of the users.
let currentUser = null;
let duplicates = null;
User.aggregate(agg).then((result) => {
console.log(result);
const email = result[0]._id;
return User.find({ email }).collation({ locale: 'en', strength: 2 })
.sort({ createdAt: 1 }).exec();
}).then((result) => {
[currentUser, ...duplicates] = result;
console.log('Current User: ', currentUser._id, ' ', currentUser.email);
duplicates = duplicates.map(dup => dup._id);
console.log('Duplicates: ', duplicates);
return Project.find({
user: { $in: duplicates }
}).exec();
}).then((sketches) => {
const saveSketchPromises = [];
sketches.forEach((sketch) => {
const moveSketchFilesPromises = [];
sketch.files.forEach((file) => {
if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) {
const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
.then((newUrl) => {
file.url = newUrl;
});
moveSketchFilesPromises.push(fileSavePromise);
}
});
const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
sketch.user = ObjectId(currentUser._id);
return sketch.save();
});
saveSketchPromises.push(sketchSavePromise);
});
return Promise.all(saveSketchPromises);
}).then(() => {
console.log('Moved and updated all sketches.');
return Collection.updateMany(
{ owner: { $in: duplicates } },
{ $set: { owner: ObjectId(currentUser.id) } }
);
}).then(() => {
console.log('Moved and updated all collections.');
return User.deleteMany({ _id: { $in: duplicates } });
}).then(() => {
console.log('Deleted other user accounts.');
currentUser.email = currentUser.email.toLowerCase();
return currentUser.save();
}).then(() => {
console.log('Migrated email to lowercase.');
// const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
const mailOptions = renderAccountConsolidation({
body: {
domain: 'https://editor.p5js.org',
username: currentUser.username,
email: currentUser.email
},
to: currentUser.email,
});
mail.send(mailOptions, (mailErr, result) => {
console.log('Sent email.');
process.exit(0);
});
});
// ).then((result) => {
// console.log(result);
// create list of duplicate users
// User.aggregate(agg).then((result) => {
// return fs.writeFile('duplicates.json', JSON.stringify(result), () => {
// console.log('File written.');
// process.exit(0);
// });
// });
let currentUser = null;
let duplicates = null;
fs.readFile('duplicates.json', async (err, file) => {
const result = JSON.parse(file);
for (let i = 3000; i < result.length; i += 1) {
console.log('Index: ', i);
const email = result[i]._id;
console.log(email);
await consolidateAccount(email); // eslint-disable-line
}
process.exit(0);
});
async function consolidateAccount(email) {
return User.find({ email }).collation({ locale: 'en', strength: 2 })
.sort({ createdAt: 1 }).exec()
.then((result) => {
[currentUser, ...duplicates] = result;
console.log('Current User: ', currentUser._id, ' ', currentUser.email);
duplicates = duplicates.map(dup => dup._id);
console.log('Duplicates: ', duplicates);
return Project.find({
user: { $in: duplicates }
}).exec();
}).then((sketches) => {
const saveSketchPromises = [];
sketches.forEach((sketch) => {
console.log('SketchId: ', sketch._id);
console.log('UserId: ', sketch.user);
const moveSketchFilesPromises = [];
sketch.files.forEach((file) => {
// if the file url contains sketch user
if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) {
if (file.url.includes(sketch.user)) {
const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id)
.then((newUrl) => {
file.url = newUrl;
}).catch((err) => {
console.log('Move Error:');
console.log(err);
});
moveSketchFilesPromises.push(fileSavePromise);
} else {
const fileSavePromise = copyObjectInS3(file.url, currentUser._id)
.then((newUrl) => {
file.url = newUrl;
}).catch((err) => {
console.log('Copy Error:');
console.log(err);
});
moveSketchFilesPromises.push(fileSavePromise);
}
}
});
const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => {
sketch.user = ObjectId(currentUser._id);
return sketch.save();
});
saveSketchPromises.push(sketchSavePromise);
});
return Promise.all(saveSketchPromises);
}).then(() => {
console.log('Moved and updated all sketches.');
return Collection.updateMany(
{ owner: { $in: duplicates } },
{ $set: { owner: ObjectId(currentUser.id) } }
);
}).then(() => {
console.log('Moved and updated all collections.');
return User.deleteMany({ _id: { $in: duplicates } });
}).then(() => {
console.log('Deleted other user accounts.');
currentUser.email = currentUser.email.toLowerCase();
return currentUser.save();
}).then(() => {
console.log('Migrated email to lowercase.');
// const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http';
const mailOptions = renderAccountConsolidation({
body: {
domain: 'https://editor.p5js.org',
username: currentUser.username,
email: currentUser.email
},
to: currentUser.email,
});
return new Promise((resolve, reject) => {
mail.send(mailOptions, (mailErr, result) => {
console.log('Sent email.');
if (mailErr) {
return reject(mailErr);
}
return resolve(result);
});
});
}).catch((err) => {
console.log(err);
process.exit(1);
});
}
// let duplicates = [
// "5ce3d936e0f9df0022d8330c",
// "5cff843f091745001e83c070",
// "5d246f5db489e6001eaee6e9"
// ];
// let currentUser = null;
// User.deleteMany({ _id: { $in: duplicates } }).then(() => {
// return User.findOne({ "email": "Silverstar09@hotmail.com" })
// }).then((result) => {
// currentUser = result;
// console.log('Deleted other user accounts.');
// currentUser.email = currentUser.email.toLowerCase();
// return currentUser.save();
// }).then(() => {
// const mailOptions = renderAccountConsolidation({
// body: {
// domain: 'https://editor.p5js.org',
// username: currentUser.username,
// email: currentUser.email
// },
// to: currentUser.email,
// });
// return new Promise((resolve, reject) => {
// mail.send(mailOptions, (mailErr, result) => {
// console.log('Sent email.');
// if (mailErr) {
// return reject(mailErr);
// }
// return resolve(result);
// });
// });
// });
// import s3 from '@auth0/s3';
// const client = s3.createClient({
// maxAsyncS3: 20,
// s3RetryCount: 3,
// s3RetryDelay: 1000,
// multipartUploadThreshold: 20971520, // this is the default (20 MB)
// multipartUploadSize: 15728640, // this is the default (15 MB)
// s3Options: {
// accessKeyId: `${process.env.AWS_ACCESS_KEY}`,
// secretAccessKey: `${process.env.AWS_SECRET_KEY}`,
// region: `${process.env.AWS_REGION}`
// },
// });
// const headParams = {
// Bucket: `${process.env.S3_BUCKET}`,
// Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png"
// };
// client.s3.headObject(headParams, (err, data) => {
// console.log(err);
// console.log(data);
// });

2
server/migrations/start.js
View File

@ -1,7 +1,7 @@
require('@babel/register');
require('@babel/polyfill');
const path = require('path');
require('dotenv').config({ path: path.resolve('.env') });
require('dotenv').config({ path: path.resolve('.env.production') });
require('./emailConsolidation');
// require('./populateTotalSize');
// require('./moveBucket');

2
server/routes/aws.routes.js
View File

@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated';
const router = new Router();
router.post('/S3/sign', isAuthenticated, AWSController.signS3);
router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3);
router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler);
router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3);
router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);