From 36d20281dbfc40b0422e2f87161a4ac38f759c16 Mon Sep 17 00:00:00 2001 From: Cassie Tarakajian Date: Thu, 8 Aug 2019 17:34:49 -0400 Subject: [PATCH] add script to add total size to all user accounts, add totalSize to user model, return totalSize from user api request --- server/controllers/aws.controller.js | 87 +++++++++++++++----------- server/controllers/user.controller.js | 3 +- server/migrations/populateTotalSize.js | 28 +++++++++ server/migrations/start.js | 6 +- server/models/user.js | 3 +- server/routes/aws.routes.js | 2 +- 6 files changed, 88 insertions(+), 41 deletions(-) create mode 100644 server/migrations/populateTotalSize.js diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index 3cdd7dfc..e8744959 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -108,54 +108,67 @@ export function copyObjectInS3(req, res) { }); } -export function listObjectsInS3ForUser(req, res) { - const { username } = req.user; - findUserByUsername(username, (user) => { - const userId = user.id; +export function listObjectsInS3ForUser(userId) { + let assets = []; + return new Promise((resolve) => { const params = { s3Params: { Bucket: `${process.env.S3_BUCKET}`, Prefix: `${userId}/` } }; - let assets = []; client.listObjects(params) .on('data', (data) => { assets = assets.concat(data.Contents.map(object => ({ key: object.Key, size: object.Size }))); }) .on('end', () => { - const projectAssets = []; - getProjectsForUserId(userId).then((projects) => { - let totalSize = 0; - assets.forEach((asset) => { - const name = asset.key.split('/').pop(); - const foundAsset = { - key: asset.key, - name, - size: asset.size, - url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}` - }; - totalSize += asset.size; - projects.some((project) => { - let found = false; - project.files.some((file) => { - if (!file.url) return false; - if (file.url.includes(asset.key)) { - found = true; - foundAsset.name = file.name; - foundAsset.sketchName = project.name; - foundAsset.sketchId = project.id; - foundAsset.url = file.url; - return true; - } - return false; - }); - return found; - }); - projectAssets.push(foundAsset); - }); - res.json({ assets: projectAssets, totalSize }); - }); + resolve(); }); + }).then(() => { + return getProjectsForUserId(userId); + }).then((projects) => { + const projectAssets = []; + let totalSize = 0; + assets.forEach((asset) => { + const name = asset.key.split('/').pop(); + const foundAsset = { + key: asset.key, + name, + size: asset.size, + url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}` + }; + totalSize += asset.size; + projects.some((project) => { + let found = false; + project.files.some((file) => { + if (!file.url) return false; + if (file.url.includes(asset.key)) { + found = true; + foundAsset.name = file.name; + foundAsset.sketchName = project.name; + foundAsset.sketchId = project.id; + foundAsset.url = file.url; + return true; + } + return false; + }); + return found; + }); + projectAssets.push(foundAsset); + }); + return Promise.resolve({ assets: projectAssets, totalSize}); + }).catch((err) => { + console.log('got an error'); + console.log(err); + }); +} + +export function listObjectsInS3ForUserRequestHandler(req, res) { + const { username } = req.user; + findUserByUsername(username, (user) => { + const userId = user.id; + listObjectsInS3ForUser(userId).then((objects) => { + res.json(objects); + }); }); } diff --git a/server/controllers/user.controller.js b/server/controllers/user.controller.js index 9c0d5076..3ffdb026 100644 --- a/server/controllers/user.controller.js +++ b/server/controllers/user.controller.js @@ -17,7 +17,8 @@ export function userResponse(user) { preferences: user.preferences, apiKeys: user.apiKeys, verified: user.verified, - id: user._id + id: user._id, + totalSize: req.user.totalSize }; } diff --git a/server/migrations/populateTotalSize.js b/server/migrations/populateTotalSize.js new file mode 100644 index 00000000..6551eceb --- /dev/null +++ b/server/migrations/populateTotalSize.js @@ -0,0 +1,28 @@ +import mongoose from 'mongoose'; + +import User from '../models/user'; +import { listObjectsInS3ForUser } from '../controllers/aws.controller'; + +// Connect to MongoDB +mongoose.Promise = global.Promise; +mongoose.connect(process.env.MONGO_URL, { useMongoClient: true }); +mongoose.connection.on('error', () => { + console.error('MongoDB Connection Error. Please make sure that MongoDB is running.'); + process.exit(1); +}); + +User.find({}, {}, { timeout: true }).cursor().eachAsync((user) => { + console.log(user.id); + if (user.totalSize !== undefined) { + console.log('Already updated size for user: ' + user.username); + return Promise.resolve(); + } + return listObjectsInS3ForUser(user.id).then((objects) => { + return User.findByIdAndUpdate(user.id, { $set: { totalSize: objects.totalSize } }); + }).then(() => { + console.log('Updated new total size for user: ' + user.username); + }); +}).then(() => { + console.log('Done iterating over every user'); + process.exit(0); +}); \ No newline at end of file diff --git a/server/migrations/start.js b/server/migrations/start.js index 0c68e0eb..a3127620 100644 --- a/server/migrations/start.js +++ b/server/migrations/start.js @@ -1,3 +1,7 @@ require('@babel/register'); require('@babel/polyfill'); -require('./truncate'); +const path = require('path'); +require('dotenv').config({ path: path.resolve('.env') }); +require('./populateTotalSize'); +// require('./moveBucket'); +// require('./truncate'); diff --git a/server/models/user.js b/server/models/user.js index 0a1a4dc3..98c0e1fd 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -66,7 +66,8 @@ const userSchema = new Schema({ soundOutput: { type: Boolean, default: false }, theme: { type: String, default: 'light' }, autorefresh: { type: Boolean, default: false } - } + }, + totalSize: { type: Number, default: 0 } }, { timestamps: true, usePushEach: true }); /** diff --git a/server/routes/aws.routes.js b/server/routes/aws.routes.js index 0bda0f8b..f73e4fa4 100644 --- a/server/routes/aws.routes.js +++ b/server/routes/aws.routes.js @@ -7,6 +7,6 @@ const router = new Router(); router.post('/S3/sign', isAuthenticated, AWSController.signS3); router.post('/S3/copy', isAuthenticated, AWSController.copyObjectInS3); router.delete('/S3/:object_key', isAuthenticated, AWSController.deleteObjectFromS3); -router.get('/S3/objects', AWSController.listObjectsInS3ForUser); +router.get('/S3/objects', AWSController.listObjectsInS3ForUserRequestHandler); export default router;