start to add file to delete unused files from S3

This commit is contained in:
Cassie Tarakajian 2016-12-13 20:07:02 -05:00
parent 11325626f1
commit 5711c3b4da
2 changed files with 63 additions and 10 deletions

View file

@ -31,7 +31,9 @@ function localIntercept(file, options = {}) {
} }
export function dropzoneAcceptCallback(file, done) { export function dropzoneAcceptCallback(file, done) {
return () => { return (dispatch, getState) => {
const state = getState();
console.log(state);
// for text files and small files // for text files and small files
// check mime type // check mime type
// if text, local interceptor // if text, local interceptor

View file

@ -1,4 +1,6 @@
import mongoose from 'mongoose'; import mongoose from 'mongoose';
import path from 'path';
require('dotenv').config({path: path.resolve('.env')});
const ObjectId = mongoose.Types.ObjectId; const ObjectId = mongoose.Types.ObjectId;
mongoose.connect('mongodb://localhost:27017/p5js-web-editor'); mongoose.connect('mongodb://localhost:27017/p5js-web-editor');
mongoose.connection.on('error', () => { mongoose.connection.on('error', () => {
@ -9,6 +11,55 @@ mongoose.connection.on('error', () => {
import Project from '../models/project'; import Project from '../models/project';
import User from '../models/user'; import User from '../models/user';
import s3 from 's3';
let client = s3.createClient({
maxAsyncS3: 20,
s3RetryCount: 3,
s3RetryDelay: 1000,
multipartUploadThreshold: 20971520, // this is the default (20 MB)
multipartUploadSize: 15728640, // this is the default (15 MB)
s3Options: {
accessKeyId: `${process.env.AWS_ACCESS_KEY}`,
secretAccessKey: `${process.env.AWS_SECRET_KEY}`,
region: 'us-west-2'
},
});
const s3Files = [];
Project.find({})
.exec((err, projects) => {
projects.forEach((project, projectIndex) => {
project.files.forEach((file, fileIndex) => {
if (file.url && !file.url.includes("https://rawgit.com/")) {
s3Files.push(file.url.split('/').pop());
}
});
});
console.log(s3Files);
console.log(s3Files.length);
});
const uploadedFiles = [];
const params = {'s3Params': {'Bucket': `${process.env.S3_BUCKET}`}};
let objectsResponse = client.listObjects(params);
objectsResponse.on('data', function(objects) {
objects.Contents.forEach(object => {
uploadedFiles.push(object.Key);
});
});
objectsResponse.on('end', () => {
console.log(uploadedFiles.length);
uploadedFiles.forEach(fileKey => {
if (s3Files.indexOf(fileKey) === -1) {
//delete file
console.log("would delete file: ", fileKey);
}
});
});
// let projectsNotToUpdate; // let projectsNotToUpdate;
// Project.find({'files.name': 'root'}) // Project.find({'files.name': 'root'})
// .exec((err, projects) => { // .exec((err, projects) => {
@ -126,12 +177,12 @@ import User from '../models/user';
// }); // });
// }); // });
User.find({}) // User.find({})
.exec((err, users) => { // .exec((err, users) => {
users.forEach(user => { // users.forEach(user => {
user.preferences.autorefresh = false; // user.preferences.autorefresh = false;
user.save((err, savedUser) => { // user.save((err, savedUser) => {
console.log('user saved'); // console.log('user saved');
}); // });
}); // });
}); // });