r/p5.js-web-editor
1
0
Fork 0
Code Issues 1 Pull requests Releases 1 Wiki Activity

finish moving migration of projects to username folder to chunks

Browse source
This commit is contained in:
Cassie Tarakajian 2018-01-08 14:26:51 -05:00
parent 899b5c9710
commit d58d8ea0bd
1 changed files with 56 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
server/migrations/s3UnderUser.js
View file

@ -29,36 +29,60 @@ const CHUNK = 100;
Project.count({}).exec().then((numProjects) => { Project.count({}).exec().then((numProjects) => {
console.log(numProjects); console.log(numProjects);
let index = 0; let index = 0;
}) async.whilst(
Project.find({}, (err, projects) => { () => {
projects.forEach((project, projectIndex) => { return index < numProjects;
if (!project.user) return; },
const userId = project.user.valueOf(); (whilstCb) => {
project.files.forEach((file, fileIndex) => { Project.find({}).skip(index).limit(CHUNK).exec((err, projects) => {
if (file.url && file.url.includes(process.env.S3_BUCKET) && !file.url.includes(userId)) { async.eachSeries(projects, (project, cb) => {
const key = file.url.split('/').pop(); if (!project.user) {
console.log(key); cb();
const params = { return;
Bucket: `${process.env.S3_BUCKET}`, }
CopySource: `${process.env.S3_BUCKET}/${key}`, const userId = project.user.valueOf();
Key: `${userId}/${key}` console.log(project.name);
}; async.eachSeries(project.files, (file, fileCb) => {
try { if (file.url && file.url.includes(process.env.S3_BUCKET) && !file.url.includes(userId)) {
client.moveObject(params) const key = file.url.split('/').pop();
.on('err', (err) => { console.log(key);
console.log(err); const params = {
}) Bucket: `${process.env.S3_BUCKET}`,
.on('end', () => { CopySource: `${process.env.S3_BUCKET}/${key}`,
file.url = (process.env.S3_BUCKET_URL_BASE || Key: `${userId}/${key}`
`https://s3-${process.env.AWS_REGION}.amazonaws.com/${process.env.S3_BUCKET}`) + `/${userId}/${key}`; };
project.save((err, savedProject) => { try {
console.log(`updated file ${key}`); client.moveObject(params)
}); .on('err', (err) => {
console.log(err);
})
.on('end', () => {
file.url = (process.env.S3_BUCKET_URL_BASE ||
`https://s3-${process.env.AWS_REGION}.amazonaws.com/${process.env.S3_BUCKET}`) + `/${userId}/${key}`;
project.save((err, savedProject) => {
console.log(`updated file ${key}`);
fileCb();
});
});
} catch(e) {
console.log(e);
fileCb(e);
}
} else {
fileCb();
}
}, () => {
cb();
}); });
} catch(e) { }, () => {
console.log(e); index += CHUNK;
} whilstCb();
} });
}); });
}); },
() => {
console.log('finished processing all documents.');
process.exit(0);
}
);
}); });