From ba92e01762ea8db377a9efa174ca29283321013f Mon Sep 17 00:00:00 2001 From: shakti97 Date: Mon, 9 Mar 2020 01:39:01 +0530 Subject: [PATCH 1/9] Fix username/email case issue in login/signup --- server/controllers/user.controller.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index 00f3f022..3275e8c8 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -27,11 +27,15 @@ export function findUserByUsername(username, cb) { const EMAIL_VERIFY_TOKEN_EXPIRY_TIME = Date.now() + (3600000 * 24); // 24 hours export function createUser(req, res, next) { + let { username, email } = req.body; + const { password } = req.body; + username = username.toLowerCase(); + email = email.toLowerCase(); random((tokenError, token) => { const user = new User({ - username: req.body.username, - email: req.body.email, - password: req.body.password, + username, + email, + password, verified: User.EmailConfirmation.Sent, verifiedToken: token, verifiedTokenExpires: EMAIL_VERIFY_TOKEN_EXPIRY_TIME, @@ -40,8 +44,8 @@ export function createUser(req, res, next) { User.findOne( { $or: [ - { email: req.body.email }, - { username: req.body.username } + { email }, + { username } ] }, (err, existingUser) => { @@ -51,7 +55,7 @@ export function createUser(req, res, next) { } if (existingUser) { - const fieldInUse = existingUser.email === req.body.email ? 'Email' : 'Username'; + const fieldInUse = existingUser.email === email ? 'Email' : 'Username'; res.status(422).send({ error: `${fieldInUse} is in use` }); return; } @@ -77,8 +81,8 @@ export function createUser(req, res, next) { mail.send(mailOptions, (mailErr, result) => { // eslint-disable-line no-unused-vars res.json({ - email: req.user.email, - username: req.user.username, + email, + username, preferences: req.user.preferences, verified: req.user.verified, id: req.user._id From 86e299c93626a16f6bbeb6406a0963b6a187025a Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Mon, 6 Apr 2020 15:55:00 -0400 Subject: [PATCH 2/9] Handle both lowercase and mixedcase username/password --- server/controllers/user.controller.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index 00936c4f..c50cefb4 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -39,15 +39,15 @@ export function findUserByUsername(username, cb) { } export function createUser(req, res, next) { - let { username, email } = req.body; + const { username, email } = req.body; const { password } = req.body; - username = username.toLowerCase(); - email = email.toLowerCase(); + const usernameLowerCase = username.toLowerCase(); + const emailLowerCase = email.toLowerCase(); const EMAIL_VERIFY_TOKEN_EXPIRY_TIME = Date.now() + (3600000 * 24); // 24 hours random((tokenError, token) => { const user = new User({ - username, - email, + username: usernameLowerCase, + email: emailLowerCase, password, verified: User.EmailConfirmation.Sent, verifiedToken: token, @@ -57,8 +57,8 @@ export function createUser(req, res, next) { User.findOne( { $or: [ - { email }, - { username } + { email: { $in: [ email, emailLowerCase ]} }, + { username: { $in: [ username, usernameLowerCase ]} } ] }, (err, existingUser) => { @@ -68,7 +68,7 @@ export function createUser(req, res, next) { } if (existingUser) { - const fieldInUse = existingUser.email === email ? 'Email' : 'Username'; + const fieldInUse = existingUser.email.toLowerCase() === emailLowerCase ? 'Email' : 'Username'; res.status(422).send({ error: `${fieldInUse} is in use` }); return; } From ef4a8d7ea12c336da818ecfe38d7fc5aabe35438 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Mon, 6 Apr 2020 19:01:37 -0400 Subject: [PATCH 3/9] Use MongoDB Regexes to query case insensitive username/password --- server/config/passport.js | 2 +- server/controllers/user.controller.js | 14 +++++++------- server/models/user.js | 5 +++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/server/config/passport.js b/server/config/passport.js index 4c739138..3ef7a0e2 100644 --- a/server/config/passport.js +++ b/server/config/passport.js @@ -24,7 +24,7 @@ passport.deserializeUser((id, done) => { * Sign in using Email/Username and Password. */ passport.use(new LocalStrategy({ usernameField: 'email' }, (email, password, done) => { - User.findByMailOrName(email.toLowerCase()) + User.findByMailOrName(email) .then((user) => { // eslint-disable-line consistent-return if (!user) { return done(null, false, { msg: `Email ${email} not found.` }); diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index c50cefb4..d552e648 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -1,5 +1,6 @@ import crypto from 'crypto'; import async from 'async'; +import escapeStringRegexp from 'escape-string-regexp'; import User from '../models/user'; import mail from '../utils/mail'; @@ -41,12 +42,11 @@ export function findUserByUsername(username, cb) { export function createUser(req, res, next) { const { username, email } = req.body; const { password } = req.body; - const usernameLowerCase = username.toLowerCase(); const emailLowerCase = email.toLowerCase(); const EMAIL_VERIFY_TOKEN_EXPIRY_TIME = Date.now() + (3600000 * 24); // 24 hours random((tokenError, token) => { const user = new User({ - username: usernameLowerCase, + username: username, email: emailLowerCase, password, verified: User.EmailConfirmation.Sent, @@ -57,8 +57,8 @@ export function createUser(req, res, next) { User.findOne( { $or: [ - { email: { $in: [ email, emailLowerCase ]} }, - { username: { $in: [ username, usernameLowerCase ]} } + { email: new RegExp(`^${escapeStringRegexp(email)}$`, 'i') }, + { username: new RegExp(`^${escapeStringRegexp(username)}$`, 'i') } ] }, (err, existingUser) => { @@ -106,7 +106,7 @@ export function duplicateUserCheck(req, res) { const checkType = req.query.check_type; const value = req.query[checkType]; const query = {}; - query[checkType] = value; + query[checkType] = new RegExp(`^${escapeStringRegexp(value)}$`, 'i'); User.findOne(query, (err, user) => { if (user) { return res.json({ @@ -151,7 +151,7 @@ export function resetPasswordInitiate(req, res) { async.waterfall([ random, (token, done) => { - User.findOne({ email: req.body.email }, (err, user) => { + User.findOne({ email: req.body.email.toLowerCase() }, (err, user) => { if (!user) { res.json({ success: true, message: 'If the email is registered with the editor, an email has been sent.' }); return; @@ -281,7 +281,7 @@ export function updatePassword(req, res) { } export function userExists(username, callback) { - User.findOne({ username }, (err, user) => ( + User.findOne({ username: new RegExp(`^${escapeStringRegexp(username)}$`, 'i') }, (err, user) => ( user ? callback(true) : callback(false) )); } diff --git a/server/models/user.js b/server/models/user.js index 98c0e1fd..d5e5d8f6 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import escapeStringRegexp from 'escape-string-regexp'; const bcrypt = require('bcrypt-nodejs'); @@ -144,9 +145,9 @@ userSchema.methods.findMatchingKey = function findMatchingKey(candidateKey, cb) userSchema.statics.findByMailOrName = function findByMailOrName(email) { const query = { $or: [{ - email, + email: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), }, { - username: email, + username: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), }], }; return this.findOne(query).exec(); From 9671cd06872a90d23a1349869a0073e5a05565c5 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Mon, 6 Apr 2020 19:02:55 -0400 Subject: [PATCH 4/9] Fix linting errors --- server/controllers/user.controller.js | 2 +- server/models/user.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index d552e648..f06c3d99 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -46,7 +46,7 @@ export function createUser(req, res, next) { const EMAIL_VERIFY_TOKEN_EXPIRY_TIME = Date.now() + (3600000 * 24); // 24 hours random((tokenError, token) => { const user = new User({ - username: username, + username, email: emailLowerCase, password, verified: User.EmailConfirmation.Sent, diff --git a/server/models/user.js b/server/models/user.js index d5e5d8f6..321c2231 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -147,7 +147,7 @@ userSchema.statics.findByMailOrName = function findByMailOrName(email) { $or: [{ email: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), }, { - username: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), + username: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), }], }; return this.findOne(query).exec(); From 15ad07d5cebcc56c17f7d63680f5b08d656b1bbb Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Tue, 14 Jul 2020 18:16:17 -0400 Subject: [PATCH 5/9] [#1314][#1489] Use collation instead of RegEx - Add case insensitive indexes for User.email and User.username - Update user queries by username or email so that they are case insensitive --- server/config/passport.js | 79 ++++++------- .../collectionForUserExists.js | 2 +- .../collection.controller/listCollections.js | 3 +- server/controllers/project.controller.js | 4 +- .../project.controller/getProjectsForUser.js | 2 +- server/controllers/user.controller.js | 106 +++++++++--------- server/models/user.js | 21 ++-- 7 files changed, 110 insertions(+), 107 deletions(-) diff --git a/server/config/passport.js b/server/config/passport.js index 3ef7a0e2..4c436160 100644 --- a/server/config/passport.js +++ b/server/config/passport.js @@ -43,7 +43,7 @@ passport.use(new LocalStrategy({ usernameField: 'email' }, (email, password, don * Authentificate using Basic Auth (Username + Api Key) */ passport.use(new BasicStrategy((userid, key, done) => { - User.findOne({ username: userid }, (err, user) => { // eslint-disable-line consistent-return + User.findOne({ username: userid }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { // eslint-disable-line consistent-return if (err) { return done(err); } if (!user) { return done(null, false); } user.findMatchingKey(key, (innerErr, isMatch, keyDocument) => { @@ -100,7 +100,7 @@ passport.use(new GitHubStrategy({ User.findOne({ email: { $in: emails }, - }, (findByEmailErr, existingEmailUser) => { + }).collation({ locale: 'en', strength: 2 }).exec((findByEmailErr, existingEmailUser) => { if (existingEmailUser) { existingEmailUser.email = existingEmailUser.email || primaryEmail; existingEmailUser.github = profile.id; @@ -143,44 +143,45 @@ passport.use(new GoogleStrategy({ User.findOne({ email: primaryEmail, - }, (findByEmailErr, existingEmailUser) => { + }).collation({ locale: 'en', strength: 2 }).exec((findByEmailErr, existingEmailUser) => { let username = profile._json.emails[0].value.split('@')[0]; - User.findOne({ username }, (findByUsernameErr, existingUsernameUser) => { - if (existingUsernameUser) { - const adj = friendlyWords.predicates[Math.floor(Math.random() * friendlyWords.predicates.length)]; - username = slugify(`${username} ${adj}`); - } - // what if a username is already taken from the display name too? - // then, append a random friendly word? - if (existingEmailUser) { - existingEmailUser.email = existingEmailUser.email || primaryEmail; - existingEmailUser.google = profile._json.emails[0].value; - existingEmailUser.username = existingEmailUser.username || username; - existingEmailUser.tokens.push({ kind: 'google', accessToken }); - existingEmailUser.name = existingEmailUser.name || profile._json.displayName; - existingEmailUser.verified = User.EmailConfirmation.Verified; - existingEmailUser.save((saveErr) => { - if (saveErr) { - console.log(saveErr); - } - done(null, existingEmailUser); - }); - } else { - const user = new User(); - user.email = primaryEmail; - user.google = profile._json.emails[0].value; - user.username = username; - user.tokens.push({ kind: 'google', accessToken }); - user.name = profile._json.displayName; - user.verified = User.EmailConfirmation.Verified; - user.save((saveErr) => { - if (saveErr) { - console.log(saveErr); - } - done(null, user); - }); - } - }); + User.findOne({ username }).collation({ locale: 'en', strength: 2 }) + .exec((findByUsernameErr, existingUsernameUser) => { + if (existingUsernameUser) { + const adj = friendlyWords.predicates[Math.floor(Math.random() * friendlyWords.predicates.length)]; + username = slugify(`${username} ${adj}`); + } + // what if a username is already taken from the display name too? + // then, append a random friendly word? + if (existingEmailUser) { + existingEmailUser.email = existingEmailUser.email || primaryEmail; + existingEmailUser.google = profile._json.emails[0].value; + existingEmailUser.username = existingEmailUser.username || username; + existingEmailUser.tokens.push({ kind: 'google', accessToken }); + existingEmailUser.name = existingEmailUser.name || profile._json.displayName; + existingEmailUser.verified = User.EmailConfirmation.Verified; + existingEmailUser.save((saveErr) => { + if (saveErr) { + console.log(saveErr); + } + done(null, existingEmailUser); + }); + } else { + const user = new User(); + user.email = primaryEmail; + user.google = profile._json.emails[0].value; + user.username = username; + user.tokens.push({ kind: 'google', accessToken }); + user.name = profile._json.displayName; + user.verified = User.EmailConfirmation.Verified; + user.save((saveErr) => { + if (saveErr) { + console.log(saveErr); + } + done(null, user); + }); + } + }); }); }); })); diff --git a/server/controllers/collection.controller/collectionForUserExists.js b/server/controllers/collection.controller/collectionForUserExists.js index e2881fd4..8315d50c 100644 --- a/server/controllers/collection.controller/collectionForUserExists.js +++ b/server/controllers/collection.controller/collectionForUserExists.js @@ -11,7 +11,7 @@ export default function collectionForUserExists(username, collectionId, callback } function findUser() { - return User.findOne({ username }); + return User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec(); } function findCollection(owner) { diff --git a/server/controllers/collection.controller/listCollections.js b/server/controllers/collection.controller/listCollections.js index c71041b3..1def86aa 100644 --- a/server/controllers/collection.controller/listCollections.js +++ b/server/controllers/collection.controller/listCollections.js @@ -3,7 +3,8 @@ import User from '../../models/user'; async function getOwnerUserId(req) { if (req.params.username) { - const user = await User.findOne({ username: req.params.username }); + const user = + await User.findOne({ username: req.params.username }).collation({ locale: 'en', strength: 2 }).exec(); if (user && user._id) { return user._id; } diff --git a/server/controllers/project.controller.js b/server/controllers/project.controller.js index ab4d4a63..9cc58fd2 100644 --- a/server/controllers/project.controller.js +++ b/server/controllers/project.controller.js @@ -64,7 +64,7 @@ export function updateProject(req, res) { export function getProject(req, res) { const { project_id: projectId, username } = req.params; - User.findOne({ username }, (err, user) => { // eslint-disable-line + User.findOne({ username }).collation({ locale: "en", strength: 2 }).exec((err, user) => { // eslint-disable-line if (!user) { return res.status(404).send({ message: 'Project with that username does not exist' }); } @@ -141,7 +141,7 @@ export function projectExists(projectId, callback) { } export function projectForUserExists(username, projectId, callback) { - User.findOne({ username }, (err, user) => { + User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { if (!user) { callback(false); return; diff --git a/server/controllers/project.controller/getProjectsForUser.js b/server/controllers/project.controller/getProjectsForUser.js index 1d9a0e34..453e2ebe 100644 --- a/server/controllers/project.controller/getProjectsForUser.js +++ b/server/controllers/project.controller/getProjectsForUser.js @@ -7,7 +7,7 @@ const UserNotFoundError = createApplicationErrorClass('UserNotFoundError'); function getProjectsForUserName(username) { return new Promise((resolve, reject) => { - User.findOne({ username }, (err, user) => { + User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { if (err) { reject(err); return; diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index f06c3d99..56aa574e 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -1,6 +1,5 @@ import crypto from 'crypto'; import async from 'async'; -import escapeStringRegexp from 'escape-string-regexp'; import User from '../models/user'; import mail from '../utils/mail'; @@ -31,12 +30,9 @@ const random = (done) => { }; export function findUserByUsername(username, cb) { - User.findOne( - { username }, - (err, user) => { - cb(user); - } - ); + User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { + cb(user); + }); } export function createUser(req, res, next) { @@ -54,51 +50,48 @@ export function createUser(req, res, next) { verifiedTokenExpires: EMAIL_VERIFY_TOKEN_EXPIRY_TIME, }); - User.findOne( - { - $or: [ - { email: new RegExp(`^${escapeStringRegexp(email)}$`, 'i') }, - { username: new RegExp(`^${escapeStringRegexp(username)}$`, 'i') } - ] - }, - (err, existingUser) => { - if (err) { - res.status(404).send({ error: err }); - return; - } + User.findOne({ + $or: [ + { email }, + { username } + ] + }).collation({ locale: 'en', strength: 2 }).exec((err, existingUser) => { + if (err) { + res.status(404).send({ error: err }); + return; + } - if (existingUser) { - const fieldInUse = existingUser.email.toLowerCase() === emailLowerCase ? 'Email' : 'Username'; - res.status(422).send({ error: `${fieldInUse} is in use` }); + if (existingUser) { + const fieldInUse = existingUser.email.toLowerCase() === emailLowerCase ? 'Email' : 'Username'; + res.status(422).send({ error: `${fieldInUse} is in use` }); + return; + } + user.save((saveErr) => { + if (saveErr) { + next(saveErr); return; } - user.save((saveErr) => { - if (saveErr) { - next(saveErr); + req.logIn(user, (loginErr) => { + if (loginErr) { + next(loginErr); return; } - req.logIn(user, (loginErr) => { - if (loginErr) { - next(loginErr); - return; - } - const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; - const mailOptions = renderEmailConfirmation({ - body: { - domain: `${protocol}://${req.headers.host}`, - link: `${protocol}://${req.headers.host}/verify?t=${token}` - }, - to: req.user.email, - }); + const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; + const mailOptions = renderEmailConfirmation({ + body: { + domain: `${protocol}://${req.headers.host}`, + link: `${protocol}://${req.headers.host}/verify?t=${token}` + }, + to: req.user.email, + }); - mail.send(mailOptions, (mailErr, result) => { // eslint-disable-line no-unused-vars - res.json(userResponse(req.user)); - }); + mail.send(mailOptions, (mailErr, result) => { // eslint-disable-line no-unused-vars + res.json(userResponse(req.user)); }); }); - } - ); + }); + }); }); } @@ -106,8 +99,8 @@ export function duplicateUserCheck(req, res) { const checkType = req.query.check_type; const value = req.query[checkType]; const query = {}; - query[checkType] = new RegExp(`^${escapeStringRegexp(value)}$`, 'i'); - User.findOne(query, (err, user) => { + query[checkType] = value; + User.findOne(query).collation({ locale: 'en', strength: 2 }).exec((err, user) => { if (user) { return res.json({ exists: true, @@ -151,18 +144,19 @@ export function resetPasswordInitiate(req, res) { async.waterfall([ random, (token, done) => { - User.findOne({ email: req.body.email.toLowerCase() }, (err, user) => { - if (!user) { - res.json({ success: true, message: 'If the email is registered with the editor, an email has been sent.' }); - return; - } - user.resetPasswordToken = token; - user.resetPasswordExpires = Date.now() + 3600000; // 1 hour + User.findOne({ email: req.body.email.toLowerCase() }) + .collation({ locale: 'en', strength: 2 }).exec((err, user) => { + if (!user) { + res.json({ success: true, message: 'If the email is registered with the editor, an email has been sent.' }); + return; + } + user.resetPasswordToken = token; + user.resetPasswordExpires = Date.now() + 3600000; // 1 hour - user.save((saveErr) => { - done(saveErr, token, user); + user.save((saveErr) => { + done(saveErr, token, user); + }); }); - }); }, (token, user, done) => { const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; @@ -281,7 +275,7 @@ export function updatePassword(req, res) { } export function userExists(username, callback) { - User.findOne({ username: new RegExp(`^${escapeStringRegexp(username)}$`, 'i') }, (err, user) => ( + User.findOne(username).collation({ locale: 'en', strength: 2 }).exec((err, user) => ( user ? callback(true) : callback(false) )); } diff --git a/server/models/user.js b/server/models/user.js index 321c2231..1f91c104 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,5 +1,4 @@ import mongoose from 'mongoose'; -import escapeStringRegexp from 'escape-string-regexp'; const bcrypt = require('bcrypt-nodejs'); @@ -143,16 +142,24 @@ userSchema.methods.findMatchingKey = function findMatchingKey(candidateKey, cb) }; userSchema.statics.findByMailOrName = function findByMailOrName(email) { + const isEmail = email.indexOf('@') > -1; + if (isEmail) { + const query = { + email: email.toLowerCase() + }; + // once emails are all lowercase, won't need to do collation + // but maybe it's not even necessary to make all emails lowercase?? + return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(); + } const query = { - $or: [{ - email: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), - }, { - username: new RegExp(`^${escapeStringRegexp(email)}$`, 'i'), - }], + username: email }; - return this.findOne(query).exec(); + return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(); }; userSchema.statics.EmailConfirmation = EmailConfirmationStates; +userSchema.index({ username: 1 }, { collation: { locale: 'en', strength: 2 } }); +userSchema.index({ email: 1 }, { collation: { locale: 'en', strength: 2 } }); + export default mongoose.model('User', userSchema); From 6259f5823378e7252ec00a42ccf3e4134b3bef28 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Wed, 15 Jul 2020 17:33:11 -0400 Subject: [PATCH 6/9] [#1314][#1489] Add static methods to user model - Add new static methods to user model - `findByEmailAndUsername` - renames `findByMailOrName` to `findByEmailOrUsername` - `findByUsername` - `findByEmail` - Reverts case insensitive behavior for username --- server/config/passport.js | 85 +++++++++---------- .../collectionForUserExists.js | 2 +- .../collection.controller/listCollections.js | 2 +- server/controllers/project.controller.js | 4 +- .../project.controller/getProjectsForUser.js | 2 +- server/controllers/user.controller.js | 35 ++++---- server/models/user.js | 81 +++++++++++++++--- 7 files changed, 132 insertions(+), 79 deletions(-) diff --git a/server/config/passport.js b/server/config/passport.js index 4c436160..163db374 100644 --- a/server/config/passport.js +++ b/server/config/passport.js @@ -24,7 +24,7 @@ passport.deserializeUser((id, done) => { * Sign in using Email/Username and Password. */ passport.use(new LocalStrategy({ usernameField: 'email' }, (email, password, done) => { - User.findByMailOrName(email) + User.findByEmailOrUsername(email) .then((user) => { // eslint-disable-line consistent-return if (!user) { return done(null, false, { msg: `Email ${email} not found.` }); @@ -43,7 +43,7 @@ passport.use(new LocalStrategy({ usernameField: 'email' }, (email, password, don * Authentificate using Basic Auth (Username + Api Key) */ passport.use(new BasicStrategy((userid, key, done) => { - User.findOne({ username: userid }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { // eslint-disable-line consistent-return + User.findByUsername(userid, (err, user) => { // eslint-disable-line consistent-return if (err) { return done(err); } if (!user) { return done(null, false); } user.findMatchingKey(key, (innerErr, isMatch, keyDocument) => { @@ -98,9 +98,7 @@ passport.use(new GitHubStrategy({ const emails = getVerifiedEmails(profile.emails); const primaryEmail = getPrimaryEmail(profile.emails); - User.findOne({ - email: { $in: emails }, - }).collation({ locale: 'en', strength: 2 }).exec((findByEmailErr, existingEmailUser) => { + User.findByEmail(emails, (findByEmailErr, existingEmailUser) => { if (existingEmailUser) { existingEmailUser.email = existingEmailUser.email || primaryEmail; existingEmailUser.github = profile.id; @@ -141,47 +139,44 @@ passport.use(new GoogleStrategy({ const primaryEmail = profile._json.emails[0].value; - User.findOne({ - email: primaryEmail, - }).collation({ locale: 'en', strength: 2 }).exec((findByEmailErr, existingEmailUser) => { + User.findByEmail(primaryEmail, (findByEmailErr, existingEmailUser) => { let username = profile._json.emails[0].value.split('@')[0]; - User.findOne({ username }).collation({ locale: 'en', strength: 2 }) - .exec((findByUsernameErr, existingUsernameUser) => { - if (existingUsernameUser) { - const adj = friendlyWords.predicates[Math.floor(Math.random() * friendlyWords.predicates.length)]; - username = slugify(`${username} ${adj}`); - } - // what if a username is already taken from the display name too? - // then, append a random friendly word? - if (existingEmailUser) { - existingEmailUser.email = existingEmailUser.email || primaryEmail; - existingEmailUser.google = profile._json.emails[0].value; - existingEmailUser.username = existingEmailUser.username || username; - existingEmailUser.tokens.push({ kind: 'google', accessToken }); - existingEmailUser.name = existingEmailUser.name || profile._json.displayName; - existingEmailUser.verified = User.EmailConfirmation.Verified; - existingEmailUser.save((saveErr) => { - if (saveErr) { - console.log(saveErr); - } - done(null, existingEmailUser); - }); - } else { - const user = new User(); - user.email = primaryEmail; - user.google = profile._json.emails[0].value; - user.username = username; - user.tokens.push({ kind: 'google', accessToken }); - user.name = profile._json.displayName; - user.verified = User.EmailConfirmation.Verified; - user.save((saveErr) => { - if (saveErr) { - console.log(saveErr); - } - done(null, user); - }); - } - }); + User.findByUsername(username, (findByUsernameErr, existingUsernameUser) => { + if (existingUsernameUser) { + const adj = friendlyWords.predicates[Math.floor(Math.random() * friendlyWords.predicates.length)]; + username = slugify(`${username} ${adj}`); + } + // what if a username is already taken from the display name too? + // then, append a random friendly word? + if (existingEmailUser) { + existingEmailUser.email = existingEmailUser.email || primaryEmail; + existingEmailUser.google = profile._json.emails[0].value; + existingEmailUser.username = existingEmailUser.username || username; + existingEmailUser.tokens.push({ kind: 'google', accessToken }); + existingEmailUser.name = existingEmailUser.name || profile._json.displayName; + existingEmailUser.verified = User.EmailConfirmation.Verified; + existingEmailUser.save((saveErr) => { + if (saveErr) { + console.log(saveErr); + } + done(null, existingEmailUser); + }); + } else { + const user = new User(); + user.email = primaryEmail; + user.google = profile._json.emails[0].value; + user.username = username; + user.tokens.push({ kind: 'google', accessToken }); + user.name = profile._json.displayName; + user.verified = User.EmailConfirmation.Verified; + user.save((saveErr) => { + if (saveErr) { + console.log(saveErr); + } + done(null, user); + }); + } + }); }); }); })); diff --git a/server/controllers/collection.controller/collectionForUserExists.js b/server/controllers/collection.controller/collectionForUserExists.js index 8315d50c..4dd2d40f 100644 --- a/server/controllers/collection.controller/collectionForUserExists.js +++ b/server/controllers/collection.controller/collectionForUserExists.js @@ -11,7 +11,7 @@ export default function collectionForUserExists(username, collectionId, callback } function findUser() { - return User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec(); + return User.findByUsername(username); } function findCollection(owner) { diff --git a/server/controllers/collection.controller/listCollections.js b/server/controllers/collection.controller/listCollections.js index 1def86aa..b1a60ee9 100644 --- a/server/controllers/collection.controller/listCollections.js +++ b/server/controllers/collection.controller/listCollections.js @@ -4,7 +4,7 @@ import User from '../../models/user'; async function getOwnerUserId(req) { if (req.params.username) { const user = - await User.findOne({ username: req.params.username }).collation({ locale: 'en', strength: 2 }).exec(); + await User.findByUsername(req.params.username); if (user && user._id) { return user._id; } diff --git a/server/controllers/project.controller.js b/server/controllers/project.controller.js index 9cc58fd2..7f118c23 100644 --- a/server/controllers/project.controller.js +++ b/server/controllers/project.controller.js @@ -64,7 +64,7 @@ export function updateProject(req, res) { export function getProject(req, res) { const { project_id: projectId, username } = req.params; - User.findOne({ username }).collation({ locale: "en", strength: 2 }).exec((err, user) => { // eslint-disable-line + User.findByUsername(username, (err, user) => { // eslint-disable-line if (!user) { return res.status(404).send({ message: 'Project with that username does not exist' }); } @@ -141,7 +141,7 @@ export function projectExists(projectId, callback) { } export function projectForUserExists(username, projectId, callback) { - User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { + User.findByUsername(username, (err, user) => { if (!user) { callback(false); return; diff --git a/server/controllers/project.controller/getProjectsForUser.js b/server/controllers/project.controller/getProjectsForUser.js index 453e2ebe..7b563b2c 100644 --- a/server/controllers/project.controller/getProjectsForUser.js +++ b/server/controllers/project.controller/getProjectsForUser.js @@ -7,7 +7,7 @@ const UserNotFoundError = createApplicationErrorClass('UserNotFoundError'); function getProjectsForUserName(username) { return new Promise((resolve, reject) => { - User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { + User.findByUsername(username, (err, user) => { if (err) { reject(err); return; diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index 56aa574e..0ff791c6 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -30,7 +30,7 @@ const random = (done) => { }; export function findUserByUsername(username, cb) { - User.findOne({ username }).collation({ locale: 'en', strength: 2 }).exec((err, user) => { + User.findByUsername(username, (err, user) => { cb(user); }); } @@ -50,12 +50,7 @@ export function createUser(req, res, next) { verifiedTokenExpires: EMAIL_VERIFY_TOKEN_EXPIRY_TIME, }); - User.findOne({ - $or: [ - { email }, - { username } - ] - }).collation({ locale: 'en', strength: 2 }).exec((err, existingUser) => { + User.findByEmailAndUsername(email, username, (err, existingUser) => { if (err) { res.status(404).send({ error: err }); return; @@ -100,6 +95,9 @@ export function duplicateUserCheck(req, res) { const value = req.query[checkType]; const query = {}; query[checkType] = value; + // Don't want to use findByEmailOrUsername here, because in this case we do + // want to use case-insensitive search for usernames to prevent username + // duplicates, which overrides the default behavior. User.findOne(query).collation({ locale: 'en', strength: 2 }).exec((err, user) => { if (user) { return res.json({ @@ -144,19 +142,18 @@ export function resetPasswordInitiate(req, res) { async.waterfall([ random, (token, done) => { - User.findOne({ email: req.body.email.toLowerCase() }) - .collation({ locale: 'en', strength: 2 }).exec((err, user) => { - if (!user) { - res.json({ success: true, message: 'If the email is registered with the editor, an email has been sent.' }); - return; - } - user.resetPasswordToken = token; - user.resetPasswordExpires = Date.now() + 3600000; // 1 hour + User.findByEmail(req.body.email, (err, user) => { + if (!user) { + res.json({ success: true, message: 'If the email is registered with the editor, an email has been sent.' }); + return; + } + user.resetPasswordToken = token; + user.resetPasswordExpires = Date.now() + 3600000; // 1 hour - user.save((saveErr) => { - done(saveErr, token, user); - }); + user.save((saveErr) => { + done(saveErr, token, user); }); + }); }, (token, user, done) => { const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; @@ -275,7 +272,7 @@ export function updatePassword(req, res) { } export function userExists(username, callback) { - User.findOne(username).collation({ locale: 'en', strength: 2 }).exec((err, user) => ( + User.findByUsername(username, (err, user) => ( user ? callback(true) : callback(false) )); } diff --git a/server/models/user.js b/server/models/user.js index 1f91c104..c4097e87 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -141,20 +141,81 @@ userSchema.methods.findMatchingKey = function findMatchingKey(candidateKey, cb) if (!foundOne) cb('Matching API key not found !', false, null); }; -userSchema.statics.findByMailOrName = function findByMailOrName(email) { - const isEmail = email.indexOf('@') > -1; - if (isEmail) { - const query = { - email: email.toLowerCase() +/** + * + * Queries User collection by email and returns one User document. + * + * @param {string|string[]} email - Email string or array of email strings + * @callback [cb] - Optional error-first callback that passes User document + * @return {Promise} - Returns Promise fulfilled by User document + */ +userSchema.statics.findByEmail = function findByEmail(email, cb) { + let query; + if (Array.isArray(email)) { + query = { + email: { $in: email } + }; + } else { + query = { + email }; - // once emails are all lowercase, won't need to do collation - // but maybe it's not even necessary to make all emails lowercase?? - return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(); } + // Email addresses should be case-insensitive unique + // In MongoDB, you must use collation in order to do a case-insensitive query + return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(cb); +}; + +/** + * + * Queries User collection by username and returns one User document. + * + * @param {string} username - Username string + * @callback [cb] - Optional error-first callback that passes User document + * @return {Promise} - Returns Promise fulfilled by User document + */ +userSchema.statics.findByUsername = function findByUsername(username, cb) { const query = { - username: email + username }; - return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(); + return this.findOne(query, cb); +}; + +/** + * + * Queries User collection using email or username with optional callback. + * This function will determine automatically whether the data passed is + * a username or email. + * + * @param {string} value - Email or username + * @callback [cb] - Optional error-first callback that passes User document + * @return {Promise} - Returns Promise fulfilled by User document + */ +userSchema.statics.findByEmailOrUsername = function findByEmailOrUsername(value, cb) { + const isEmail = value.indexOf('@') > -1; + if (isEmail) { + return this.findByEmail(value, cb); + } + return this.findByUsername(value, cb); +}; + +/** + * + * Queries User collection, performing a MongoDB logical or with the email + * and username (i.e. if either one matches, will return the first document). + * + * @param {string} email + * @param {string} username + * @callback [cb] - Optional error-first callback that passes User document + * @return {Promise} - Returns Promise fulfilled by User document + */ +userSchema.statics.findByEmailAndUsername = function findByEmailAndUsername(email, username, cb) { + const query = { + $or: [ + { email }, + { username } + ] + }; + return this.findOne(query).collation({ locale: 'en', strength: 2 }).exec(cb); }; userSchema.statics.EmailConfirmation = EmailConfirmationStates; From b126a69c110d25da78bf34ae140eb8fb8392246e Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Thu, 16 Jul 2020 14:25:02 -0400 Subject: [PATCH 7/9] [#1317] Begin writing duplicate email script --- server/controllers/aws.controller.js | 21 +++++ server/migrations/emailConsolidation.js | 101 ++++++++++++++++++++++++ server/migrations/start.js | 3 +- 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 server/migrations/emailConsolidation.js diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index 0b7d5d85..6c0cb13b 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -119,6 +119,27 @@ export function copyObjectInS3(req, res) { }); } +export function moveObjectToUserInS3(url, userId) { + return new Promise((resolve, reject) => { + const objectKey = getObjectKey(url); + const fileExtension = getExtension(objectKey); + const newFilename = uuid.v4() + fileExtension; + const params = { + Bucket: `${process.env.S3_BUCKET}`, + CopySource: `${process.env.S3_BUCKET}/${objectKey}`, + Key: `${userId}/${newFilename}`, + ACL: 'public-read' + }; + const move = client.moveObject(params); + move.on('err', (err) => { + reject(err); + }); + move.on('end', (data) => { + resolve(`${s3Bucket}${userId}/${newFilename}`); + }); + }); +} + export function listObjectsInS3ForUser(userId) { let assets = []; return new Promise((resolve) => { diff --git a/server/migrations/emailConsolidation.js b/server/migrations/emailConsolidation.js new file mode 100644 index 00000000..7b8dd1d8 --- /dev/null +++ b/server/migrations/emailConsolidation.js @@ -0,0 +1,101 @@ +import mongoose from 'mongoose'; +import User from '../models/user'; +import Project from '../models/project'; +import Collection from '../models/collection'; +import { moveObjectToUserInS3 } from '../controllers/aws.controller'; + + +const mongoConnectionString = process.env.MONGO_URL; +const { ObjectId } = mongoose.Types; +// Connect to MongoDB +mongoose.Promise = global.Promise; +mongoose.connect(mongoConnectionString, { useNewUrlParser: true, useUnifiedTopology: true }); +mongoose.set('useCreateIndex', true); +mongoose.connection.on('error', () => { + console.error('MongoDB Connection Error. Please make sure that MongoDB is running.'); + process.exit(1); +}); + +/* + * Requires the MongoDB Node.js Driver + * https://mongodb.github.io/node-mongodb-native + */ + +const agg = [ + { + $project: { + email: { + $toLower: [ + '$email' + ] + } + } + }, { + $group: { + _id: '$email', + total: { + $sum: 1 + } + } + }, { + $match: { + total: { + $gt: 1 + } + } + }, { + $sort: { + total: -1 + } + } +]; + +let currentUser = null; +let duplicates = null; +User.aggregate(agg).then((result) => { + const email = result[0]._id; + return User.find({ email }).collation({ locale: 'en', strength: 2 }) + .sort({ createdAt: 1 }).exec(); +}).then((result) => { + [currentUser, ...duplicates] = result; + duplicates = duplicates.map(dup => dup._id); + console.log(duplicates); + return Project.find({ + user: { $in: duplicates } + }).exec(); +}).then((sketches) => { + const saveSketchPromises = []; + sketches.forEach((sketch) => { + const moveSketchFilesPromises = []; + sketch.files.forEach((file) => { + if (file.url.includes('assets.editor.p5js.org')) { + const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) + .then((newUrl) => { + file.url = newUrl; + }); + moveSketchFilesPromises.push(fileSavePromise); + } + }); + const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => { + sketch.user = ObjectId(currentUser._id); + return sketch.save(); + }); + saveSketchPromises.push(sketchSavePromise); + }); + return Promise.all(saveSketchPromises); + // iterate through the results + // check if any files are on AWS + // if so, move them to the right user bucket + // then, update the user to currentUser + // then, after updating all of the projects + // also update the collections + // delete other users + // update user email so it is all lowercase + // then, send the email +}).then(() => Collection.updateMany( + { owner: { $in: duplicates } }, + { $set: { owner: ObjectId(currentUser.id) } } +)).then(() => User.deleteMany({ _id: { $in: duplicates } })).catch((err) => { + console.log(err); +}); + diff --git a/server/migrations/start.js b/server/migrations/start.js index a3127620..3fc2e6e4 100644 --- a/server/migrations/start.js +++ b/server/migrations/start.js @@ -2,6 +2,7 @@ require('@babel/register'); require('@babel/polyfill'); const path = require('path'); require('dotenv').config({ path: path.resolve('.env') }); -require('./populateTotalSize'); +require('./emailConsolidation'); +// require('./populateTotalSize'); // require('./moveBucket'); // require('./truncate'); From cd5c000d6db3790920e3c551d5fd91645e6af550 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Thu, 16 Jul 2020 19:37:37 -0400 Subject: [PATCH 8/9] [#1317] Update email consolidation script - Update the emailConsolidation script so that it will de-duplicate the first duplicated user - Sends email to user alerting them that the consolidation has taken place. --- server/controllers/aws.controller.js | 1 + server/migrations/emailConsolidation.js | 68 ++++++++++++++++----- server/views/consolidationMailLayout.js | 78 +++++++++++++++++++++++++ server/views/mail.js | 42 +++++++++++++ 4 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 server/views/consolidationMailLayout.js diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index 32122c71..a331a98c 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -124,6 +124,7 @@ export function moveObjectToUserInS3(url, userId) { const objectKey = getObjectKey(url); const fileExtension = getExtension(objectKey); const newFilename = uuid.v4() + fileExtension; + console.log(`${process.env.S3_BUCKET}/${objectKey}`); const params = { Bucket: `${process.env.S3_BUCKET}`, CopySource: `${process.env.S3_BUCKET}/${objectKey}`, diff --git a/server/migrations/emailConsolidation.js b/server/migrations/emailConsolidation.js index 7b8dd1d8..d259d918 100644 --- a/server/migrations/emailConsolidation.js +++ b/server/migrations/emailConsolidation.js @@ -3,6 +3,8 @@ import User from '../models/user'; import Project from '../models/project'; import Collection from '../models/collection'; import { moveObjectToUserInS3 } from '../controllers/aws.controller'; +import mail from '../utils/mail'; +import { renderAccountConsolidation } from '../views/mail'; const mongoConnectionString = process.env.MONGO_URL; @@ -50,16 +52,31 @@ const agg = [ } ]; + +// steps to make this work +// iterate through the results +// check if any files are on AWS +// if so, move them to the right user bucket +// then, update the user to currentUser +// then, after updating all of the projects +// also update the collections +// delete other users +// update user email so it is all lowercase +// then, send the email +// then, figure out how to iterate through all of the users. + let currentUser = null; let duplicates = null; User.aggregate(agg).then((result) => { + console.log(result); const email = result[0]._id; return User.find({ email }).collation({ locale: 'en', strength: 2 }) .sort({ createdAt: 1 }).exec(); }).then((result) => { [currentUser, ...duplicates] = result; + console.log('Current User: ', currentUser._id, ' ', currentUser.email); duplicates = duplicates.map(dup => dup._id); - console.log(duplicates); + console.log('Duplicates: ', duplicates); return Project.find({ user: { $in: duplicates } }).exec(); @@ -68,7 +85,7 @@ User.aggregate(agg).then((result) => { sketches.forEach((sketch) => { const moveSketchFilesPromises = []; sketch.files.forEach((file) => { - if (file.url.includes('assets.editor.p5js.org')) { + if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) { const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) .then((newUrl) => { file.url = newUrl; @@ -83,19 +100,38 @@ User.aggregate(agg).then((result) => { saveSketchPromises.push(sketchSavePromise); }); return Promise.all(saveSketchPromises); - // iterate through the results - // check if any files are on AWS - // if so, move them to the right user bucket - // then, update the user to currentUser - // then, after updating all of the projects - // also update the collections - // delete other users - // update user email so it is all lowercase - // then, send the email -}).then(() => Collection.updateMany( - { owner: { $in: duplicates } }, - { $set: { owner: ObjectId(currentUser.id) } } -)).then(() => User.deleteMany({ _id: { $in: duplicates } })).catch((err) => { - console.log(err); +}).then(() => { + console.log('Moved and updated all sketches.'); + return Collection.updateMany( + { owner: { $in: duplicates } }, + { $set: { owner: ObjectId(currentUser.id) } } + ); +}).then(() => { + console.log('Moved and updated all collections.'); + return User.deleteMany({ _id: { $in: duplicates } }); +}).then(() => { + console.log('Deleted other user accounts.'); + currentUser.email = currentUser.email.toLowerCase(); + return currentUser.save(); +}).then(() => { + console.log('Migrated email to lowercase.'); + // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; + const mailOptions = renderAccountConsolidation({ + body: { + domain: 'https://editor.p5js.org', + username: currentUser.username, + email: currentUser.email + }, + to: currentUser.email, + }); + + mail.send(mailOptions, (mailErr, result) => { + console.log('Sent email.'); + process.exit(0); + }); }); +// ).then((result) => { +// console.log(result); +// }); + diff --git a/server/views/consolidationMailLayout.js b/server/views/consolidationMailLayout.js new file mode 100644 index 00000000..fa986024 --- /dev/null +++ b/server/views/consolidationMailLayout.js @@ -0,0 +1,78 @@ +export default ({ + domain, + headingText, + greetingText, + messageText, + username, + email, + message2Text, + resetPasswordLink, + directLinkText, + resetPasswordText, + noteText, + meta +}) => ( + ` + + + + + + + + + + + + + + + + + + + + ${headingText} + + + + + + + + ${greetingText} + + + ${messageText} + + + Username: ${username} + + + Email: ${email} + + + ${message2Text} + + + ${resetPasswordText} + + + + + + + + ${directLinkText} + + ${domain}/${resetPasswordLink} + + ${noteText} + + + + + + +` +); diff --git a/server/views/mail.js b/server/views/mail.js index 43aaf3cb..07e5c5e5 100644 --- a/server/views/mail.js +++ b/server/views/mail.js @@ -1,5 +1,47 @@ import renderMjml from '../utils/renderMjml'; import mailLayout from './mailLayout'; +import consolidationMailLayout from './consolidationMailLayout'; + +export const renderAccountConsolidation = (data) => { + const subject = 'p5.js Web Editor Account Consolidation'; + const templateOptions = { + domain: data.body.domain, + headingText: 'Account Consolidation', + greetingText: 'Hello,', + messageText: `You're receiving this message because you previous registered for the + p5.js Web Editor + using the same email address multiple times. In order to fix bugs and prevent future bugs, + your accounts have been consolidated to the first account you created. You can login with + the following email and username:`, + username: data.body.username, + email: data.body.email, + message2Text: `All of your sketches and collections have been preserved and have not been modified. + If you have forgotten your password you can reset it:`, + resetPasswordLink: 'reset-password', + resetPasswordText: 'Reset Password', + directLinkText: 'Or copy and paste the following URL into your browser:', + noteText: `We are grateful for your patience and understanding. Thank you for supporting p5.js and the + p5.js Web Editor!`, + meta: { + keywords: 'p5.js, p5.js web editor, web editor, processing, code editor', + description: 'A web editor for p5.js, a JavaScript library with the goal' + + ' of making coding accessible to artists, designers, educators, and beginners.' + } + }; + + // Return MJML string + const template = consolidationMailLayout(templateOptions); + + // Render MJML to HTML string + const html = renderMjml(template); + + // Return options to send mail + return Object.assign( + {}, + data, + { html, subject }, + ); +}; export const renderResetPassword = (data) => { const subject = 'p5.js Web Editor Password Reset'; From 2b3be3acccedbd81ed0bd15301df9be6f66b9ca4 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Fri, 17 Jul 2020 19:09:23 -0400 Subject: [PATCH 9/9] [#1317] Update and run email consolidation script --- .gitignore | 1 + .../controllers/__mocks__/aws.controller.js | 2 +- server/controllers/aws.controller.js | 82 +++--- server/migrations/emailConsolidation.js | 238 +++++++++++++----- server/migrations/start.js | 2 +- server/routes/aws.routes.js | 2 +- 6 files changed, 227 insertions(+), 100 deletions(-) diff --git a/.gitignore b/.gitignore index e36b805f..6fdf06c6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ localhost.key privkey.pem storybook-static +duplicates.json diff --git a/server/controllers/__mocks__/aws.controller.js b/server/controllers/__mocks__/aws.controller.js index 9e5709c4..061072a1 100644 --- a/server/controllers/__mocks__/aws.controller.js +++ b/server/controllers/__mocks__/aws.controller.js @@ -1,5 +1,5 @@ export const getObjectKey = jest.mock(); export const deleteObjectsFromS3 = jest.fn(); export const signS3 = jest.fn(); -export const copyObjectInS3 = jest.fn(); +export const copyObjectInS3RequestHandler = jest.fn(); export const listObjectsInS3ForUser = jest.fn(); diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index a331a98c..a315f1d4 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -98,24 +98,41 @@ export function signS3(req, res) { res.json(result); } -export function copyObjectInS3(req, res) { - const { url } = req.body; - const objectKey = getObjectKey(url); - const fileExtension = getExtension(objectKey); - const newFilename = uuid.v4() + fileExtension; - const userId = req.user.id; - const params = { - Bucket: `${process.env.S3_BUCKET}`, - CopySource: `${process.env.S3_BUCKET}/${objectKey}`, - Key: `${userId}/${newFilename}`, - ACL: 'public-read' - }; - const copy = client.copyObject(params); - copy.on('err', (err) => { - console.log(err); +export function copyObjectInS3(url, userId) { + return new Promise((resolve, reject) => { + const objectKey = getObjectKey(url); + const fileExtension = getExtension(objectKey); + const newFilename = uuid.v4() + fileExtension; + const headParams = { + Bucket: `${process.env.S3_BUCKET}`, + Key: `${objectKey}` + }; + client.s3.headObject(headParams, (headErr) => { + if (headErr) { + reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`)); + return; + } + const params = { + Bucket: `${process.env.S3_BUCKET}`, + CopySource: `${process.env.S3_BUCKET}/${objectKey}`, + Key: `${userId}/${newFilename}`, + ACL: 'public-read' + }; + const copy = client.copyObject(params); + copy.on('err', (err) => { + reject(err); + }); + copy.on('end', (data) => { + resolve(`${s3Bucket}${userId}/${newFilename}`); + }); + }); }); - copy.on('end', (data) => { - res.json({ url: `${s3Bucket}${userId}/${newFilename}` }); +} + +export function copyObjectInS3RequestHandler(req, res) { + const { url } = req.body; + copyObjectInS3(url, req.user.id).then((newUrl) => { + res.json({ url: newUrl }); }); } @@ -124,19 +141,28 @@ export function moveObjectToUserInS3(url, userId) { const objectKey = getObjectKey(url); const fileExtension = getExtension(objectKey); const newFilename = uuid.v4() + fileExtension; - console.log(`${process.env.S3_BUCKET}/${objectKey}`); - const params = { + const headParams = { Bucket: `${process.env.S3_BUCKET}`, - CopySource: `${process.env.S3_BUCKET}/${objectKey}`, - Key: `${userId}/${newFilename}`, - ACL: 'public-read' + Key: `${objectKey}` }; - const move = client.moveObject(params); - move.on('err', (err) => { - reject(err); - }); - move.on('end', (data) => { - resolve(`${s3Bucket}${userId}/${newFilename}`); + client.s3.headObject(headParams, (headErr) => { + if (headErr) { + reject(new Error(`Object with key ${process.env.S3_BUCKET}/${objectKey} does not exist.`)); + return; + } + const params = { + Bucket: `${process.env.S3_BUCKET}`, + CopySource: `${process.env.S3_BUCKET}/${objectKey}`, + Key: `${userId}/${newFilename}`, + ACL: 'public-read' + }; + const move = client.moveObject(params); + move.on('err', (err) => { + reject(err); + }); + move.on('end', (data) => { + resolve(`${s3Bucket}${userId}/${newFilename}`); + }); }); }); } diff --git a/server/migrations/emailConsolidation.js b/server/migrations/emailConsolidation.js index d259d918..8343377c 100644 --- a/server/migrations/emailConsolidation.js +++ b/server/migrations/emailConsolidation.js @@ -1,8 +1,9 @@ import mongoose from 'mongoose'; +import fs from 'fs'; import User from '../models/user'; import Project from '../models/project'; import Collection from '../models/collection'; -import { moveObjectToUserInS3 } from '../controllers/aws.controller'; +import { moveObjectToUserInS3, copyObjectInS3 } from '../controllers/aws.controller'; import mail from '../utils/mail'; import { renderAccountConsolidation } from '../views/mail'; @@ -65,73 +66,172 @@ const agg = [ // then, send the email // then, figure out how to iterate through all of the users. -let currentUser = null; -let duplicates = null; -User.aggregate(agg).then((result) => { - console.log(result); - const email = result[0]._id; - return User.find({ email }).collation({ locale: 'en', strength: 2 }) - .sort({ createdAt: 1 }).exec(); -}).then((result) => { - [currentUser, ...duplicates] = result; - console.log('Current User: ', currentUser._id, ' ', currentUser.email); - duplicates = duplicates.map(dup => dup._id); - console.log('Duplicates: ', duplicates); - return Project.find({ - user: { $in: duplicates } - }).exec(); -}).then((sketches) => { - const saveSketchPromises = []; - sketches.forEach((sketch) => { - const moveSketchFilesPromises = []; - sketch.files.forEach((file) => { - if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE)) { - const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) - .then((newUrl) => { - file.url = newUrl; - }); - moveSketchFilesPromises.push(fileSavePromise); - } - }); - const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => { - sketch.user = ObjectId(currentUser._id); - return sketch.save(); - }); - saveSketchPromises.push(sketchSavePromise); - }); - return Promise.all(saveSketchPromises); -}).then(() => { - console.log('Moved and updated all sketches.'); - return Collection.updateMany( - { owner: { $in: duplicates } }, - { $set: { owner: ObjectId(currentUser.id) } } - ); -}).then(() => { - console.log('Moved and updated all collections.'); - return User.deleteMany({ _id: { $in: duplicates } }); -}).then(() => { - console.log('Deleted other user accounts.'); - currentUser.email = currentUser.email.toLowerCase(); - return currentUser.save(); -}).then(() => { - console.log('Migrated email to lowercase.'); - // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; - const mailOptions = renderAccountConsolidation({ - body: { - domain: 'https://editor.p5js.org', - username: currentUser.username, - email: currentUser.email - }, - to: currentUser.email, - }); - - mail.send(mailOptions, (mailErr, result) => { - console.log('Sent email.'); - process.exit(0); - }); -}); - -// ).then((result) => { -// console.log(result); +// create list of duplicate users +// User.aggregate(agg).then((result) => { +// return fs.writeFile('duplicates.json', JSON.stringify(result), () => { +// console.log('File written.'); +// process.exit(0); +// }); // }); +let currentUser = null; +let duplicates = null; + +fs.readFile('duplicates.json', async (err, file) => { + const result = JSON.parse(file); + for (let i = 3000; i < result.length; i += 1) { + console.log('Index: ', i); + const email = result[i]._id; + console.log(email); + await consolidateAccount(email); // eslint-disable-line + } + process.exit(0); +}); + +async function consolidateAccount(email) { + return User.find({ email }).collation({ locale: 'en', strength: 2 }) + .sort({ createdAt: 1 }).exec() + .then((result) => { + [currentUser, ...duplicates] = result; + console.log('Current User: ', currentUser._id, ' ', currentUser.email); + duplicates = duplicates.map(dup => dup._id); + console.log('Duplicates: ', duplicates); + return Project.find({ + user: { $in: duplicates } + }).exec(); + }).then((sketches) => { + const saveSketchPromises = []; + sketches.forEach((sketch) => { + console.log('SketchId: ', sketch._id); + console.log('UserId: ', sketch.user); + const moveSketchFilesPromises = []; + sketch.files.forEach((file) => { + // if the file url contains sketch user + if (file.url && file.url.includes(process.env.S3_BUCKET_URL_BASE) && !file.url.includes(currentUser._id)) { + if (file.url.includes(sketch.user)) { + const fileSavePromise = moveObjectToUserInS3(file.url, currentUser._id) + .then((newUrl) => { + file.url = newUrl; + }).catch((err) => { + console.log('Move Error:'); + console.log(err); + }); + moveSketchFilesPromises.push(fileSavePromise); + } else { + const fileSavePromise = copyObjectInS3(file.url, currentUser._id) + .then((newUrl) => { + file.url = newUrl; + }).catch((err) => { + console.log('Copy Error:'); + console.log(err); + }); + moveSketchFilesPromises.push(fileSavePromise); + } + } + }); + const sketchSavePromise = Promise.all(moveSketchFilesPromises).then(() => { + sketch.user = ObjectId(currentUser._id); + return sketch.save(); + }); + saveSketchPromises.push(sketchSavePromise); + }); + return Promise.all(saveSketchPromises); + }).then(() => { + console.log('Moved and updated all sketches.'); + return Collection.updateMany( + { owner: { $in: duplicates } }, + { $set: { owner: ObjectId(currentUser.id) } } + ); + }).then(() => { + console.log('Moved and updated all collections.'); + return User.deleteMany({ _id: { $in: duplicates } }); + }).then(() => { + console.log('Deleted other user accounts.'); + currentUser.email = currentUser.email.toLowerCase(); + return currentUser.save(); + }).then(() => { + console.log('Migrated email to lowercase.'); + // const protocol = process.env.NODE_ENV === 'production' ? 'https' : 'http'; + const mailOptions = renderAccountConsolidation({ + body: { + domain: 'https://editor.p5js.org', + username: currentUser.username, + email: currentUser.email + }, + to: currentUser.email, + }); + + return new Promise((resolve, reject) => { + mail.send(mailOptions, (mailErr, result) => { + console.log('Sent email.'); + if (mailErr) { + return reject(mailErr); + } + return resolve(result); + }); + }); + }).catch((err) => { + console.log(err); + process.exit(1); + }); +} + + +// let duplicates = [ +// "5ce3d936e0f9df0022d8330c", +// "5cff843f091745001e83c070", +// "5d246f5db489e6001eaee6e9" +// ]; +// let currentUser = null; +// User.deleteMany({ _id: { $in: duplicates } }).then(() => { +// return User.findOne({ "email": "Silverstar09@hotmail.com" }) +// }).then((result) => { +// currentUser = result; +// console.log('Deleted other user accounts.'); +// currentUser.email = currentUser.email.toLowerCase(); +// return currentUser.save(); +// }).then(() => { +// const mailOptions = renderAccountConsolidation({ +// body: { +// domain: 'https://editor.p5js.org', +// username: currentUser.username, +// email: currentUser.email +// }, +// to: currentUser.email, +// }); + +// return new Promise((resolve, reject) => { +// mail.send(mailOptions, (mailErr, result) => { +// console.log('Sent email.'); +// if (mailErr) { +// return reject(mailErr); +// } +// return resolve(result); +// }); +// }); +// }); + + +// import s3 from '@auth0/s3'; + +// const client = s3.createClient({ +// maxAsyncS3: 20, +// s3RetryCount: 3, +// s3RetryDelay: 1000, +// multipartUploadThreshold: 20971520, // this is the default (20 MB) +// multipartUploadSize: 15728640, // this is the default (15 MB) +// s3Options: { +// accessKeyId: `${process.env.AWS_ACCESS_KEY}`, +// secretAccessKey: `${process.env.AWS_SECRET_KEY}`, +// region: `${process.env.AWS_REGION}` +// }, +// }); + +// const headParams = { +// Bucket: `${process.env.S3_BUCKET}`, +// Key: "5c9de807f6bccf0017da7927/8b9d95ae-7ddd-452a-b398-672392c4ac43.png" +// }; +// client.s3.headObject(headParams, (err, data) => { +// console.log(err); +// console.log(data); +// }); diff --git a/server/migrations/start.js b/server/migrations/start.js index 3fc2e6e4..2e0e3d88 100644 --- a/server/migrations/start.js +++ b/server/migrations/start.js @@ -1,7 +1,7 @@ require('@babel/register'); require('@babel/polyfill'); const path = require('path'); -require('dotenv').config({ path: path.resolve('.env') }); +require('dotenv').config({ path: path.resolve('.env.production') }); require('./emailConsolidation'); // require('./populateTotalSize'); // require('./moveBucket'); diff --git a/server/routes/aws.routes.js b/server/routes/aws.routes.js index 850bb190..6bbf3795 100644 --- a/server/routes/aws.routes.js +++ b/server/routes/aws.routes.js @@ -5,7 +5,7 @@ import isAuthenticated from '../utils/isAuthenticated'; const router = new Router(); router.post('/S3/sign', isAuthenticated, AWSController.signS3); -router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3); +router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3RequestHandler); router.delete('/S3/:userId?/:objectKey', isAuthenticated, AWSController.deleteObjectFromS3); router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler);