p5.js-web-editor/server/scripts/examples-ml5.js

456 lines
12 KiB
JavaScript

import fs from 'fs';
import rp from 'request-promise';
import Q from 'q';
import mongoose from 'mongoose';
import objectID from 'bson-objectid';
import shortid from 'shortid';
import User from '../models/user';
import Project from '../models/project';
// TODO: Change branchName if necessary
const branchName = 'release';
const branchRef = `?ref=${branchName}`;
const baseUrl = 'https://api.github.com/repos/ml5js/ml5-examples/contents';
const clientId = process.env.GITHUB_ID;
const clientSecret = process.env.GITHUB_SECRET;
const editorUsername = process.env.ML5_EXAMPLES_USERNAME;
const headers = {
'User-Agent': 'p5js-web-editor/0.0.1'
};
const requestOptions = {
url: baseUrl,
qs: {
client_id: clientId,
client_secret: clientSecret
},
method: 'GET',
headers,
json: true
};
const mongoConnectionString = process.env.MONGO_URL;
mongoose.connect(mongoConnectionString, { useNewUrlParser: true, useUnifiedTopology: true });
mongoose.set('useCreateIndex', true);
mongoose.connection.on('error', () => {
console.error('MongoDB Connection Error. Please make sure that MongoDB is running.');
process.exit(1);
});
/**
* ---------------------------------------------------------
* --------------------- helper functions --------------------
* ---------------------------------------------------------
*/
/**
* fatten a nested array
*/
function flatten(list) {
return list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []);
}
/**
* STEP 1: Get the top level cateogories
*/
async function getCategories() {
try {
const options = Object.assign({}, requestOptions);
options.url = `${options.url}/p5js${branchRef}`;
const results = await rp(options);
return results;
} catch (err) {
return err;
}
}
/**
* STEP 2: Get the examples for each category
* e.g. Posenet:
* - /posenet_image_single
* - /posenet_part_selection
*/
async function getCategoryExamples(sketchRootList) {
const output = [];
const sketchRootCategories = sketchRootList.map(async (categories) => {
// let options = Object.assign({url: `${requestOptions.url}/${categories.path}${branchRef}`}, requestOptions)
const options = Object.assign({}, requestOptions);
options.url = `${options.url}${categories.path}${branchRef}`;
// console.log(options)
const sketchDirs = await rp(options);
const result = flatten(sketchDirs);
return result;
});
const sketchList = await Q.all(sketchRootCategories);
sketchList.forEach((sketch) => {
sketch.forEach((item) => {
if (item.type === 'dir') output.push(item);
});
});
return output;
}
/**
* STEP 3.1: Recursively get the tree of files for each directory
* @param parentObject - one sketch directory object
*/
async function traverseSketchTree(parentObject) {
const output = Object.assign({}, parentObject);
if (parentObject.type !== 'dir') {
return output;
}
// let options = `https://api.github.com/repos/ml5js/ml5-examples/contents/${sketches.path}${branchRef}`
const options = Object.assign({}, requestOptions);
options.url = `${options.url}${parentObject.path}${branchRef}`;
output.tree = await rp(options);
output.tree = output.tree.map(file => traverseSketchTree(file));
output.tree = await Q.all(output.tree);
return output;
}
/**
* STEP 3.2: Traverse the sketchtree for all of the sketches
* @param {*} categoryExamples - all of the categories in an array
*/
async function traverseSketchTreeAll(categoryExamples) {
// const sketches = categoryExamples.map(async sketch => await traverseSketchTree(sketch));
const sketches = categoryExamples.map(async sketch => traverseSketchTree(sketch));
const result = await Q.all(sketches);
return result;
}
/**
* Traverse the tree and format into parent child relation
* @param {*} parentObject
*/
function traverseAndFormat(parentObject) {
const parent = Object.assign({}, parentObject);
if (!parentObject.tree) {
const newid = objectID().toHexString();
// returns the files
return {
name: parent.name,
url: parent.download_url,
content: null,
id: newid,
_id: newid,
fileType: 'file'
};
}
const subdir = parentObject.tree.map((item) => {
const newid = objectID().toHexString();
if (!item.tree) {
// returns the files
return {
name: item.name,
url: item.download_url,
content: null,
id: newid,
_id: newid,
fileType: 'file'
};
}
const feat = {
name: item.name,
id: newid,
_id: newid,
fileType: 'folder',
children: traverseAndFormat(item)
};
return feat;
});
return subdir;
}
/**
* Traverse the tree and flatten for project.files[]
* @param {*} projectFileTree
*/
function traverseAndFlatten(projectFileTree) {
const r = objectID().toHexString();
const projectRoot = {
name: 'root',
id: r,
_id: r,
children: [],
fileType: 'folder'
};
let currentParent;
const output = projectFileTree.reduce(
(result, item, idx) => {
if (idx < projectFileTree.length) {
projectRoot.children.push(item.id);
}
if (item.fileType === 'file') {
if (item.name === 'sketch.js') {
item.isSelectedFile = true;
}
result.push(item);
}
// here's where the magic happens *twinkles*
if (item.fileType === 'folder') {
// recursively go down the tree of children
currentParent = traverseAndFlatten(item.children);
// the above will return an array of the children files
// concatenate that with the results
result = result.concat(currentParent); // eslint-disable-line no-param-reassign
// since we want to get the children ids,
// we can map the child ids to the current item
// then push that to our result array to get
// our flat files array.
item.children = item.children.map(child => child.id);
result.push(item);
}
return result;
},
[projectRoot]
);
// Kind of hacky way to remove all roots other than the starting one
let counter = 0;
output.forEach((item, idx) => {
if (item.name === 'root') {
if (counter === 0) {
counter += 1;
} else {
output.splice(idx, 1);
counter += 1;
}
}
});
return output;
}
/**
* STEP 4
* Take a parent directory and prepare it for injestion!
* @param {*} sketch
* @param {*} user
*/
function formatSketchForStorage(sketch, user) {
const newProject = new Project({
_id: shortid.generate(),
name: sketch.name,
user: user._id,
files: [] // <== add files to this array as file objects and add _id reference to children of root
});
let projectFiles = traverseAndFormat(sketch);
projectFiles = traverseAndFlatten(projectFiles);
newProject.files = projectFiles;
return newProject;
}
/**
* format all the sketches using the formatSketchForStorage()
*/
function formatSketchForStorageAll(sketchWithItems, user) {
let sketchList = sketchWithItems.slice(0);
sketchList = sketchList.map(sketch => formatSketchForStorage(sketch, user));
return sketchList;
}
/**
* Get all the content for the relevant files in project.files[]
* @param {*} projectObject
*/
async function fetchSketchContent(projectObject) {
const output = Object.assign({}, JSON.parse(JSON.stringify(projectObject)));
const newFiles = output.files.map(async (item, i) => {
// if it is an html or js file
if (
(item.fileType === 'file' && item.name.endsWith('.html')) ||
item.name.endsWith('.js')
) {
const options = Object.assign({}, requestOptions);
options.url = `${item.url}`;
if (
options.url !== undefined ||
options.url !== null ||
options.url !== ''
) {
item.content = await rp(options);
// NOTE: remove the URL property if there's content
// Otherwise the p5 editor will try to pull from that url
if (item.content !== null) delete item.url;
}
return item;
// if it is NOT an html or js file
}
if (item.url) {
const cdnRef = `https://cdn.jsdelivr.net/gh/ml5js/ml5-examples@${branchName}${
item.url.split(branchName)[1]
}`;
item.content = cdnRef;
item.url = cdnRef;
}
return item;
});
output.files = await Q.all(newFiles);
return output;
}
/**
* STEP 5
* Get all the content for the relevant files in project.files[] for all sketches
* @param {*} formattedSketchList
*/
async function fetchSketchContentAll(formattedSketchList) {
let output = formattedSketchList.slice(0);
output = output.map(async item => fetchSketchContent(item));
output = await Q.all(output);
return output;
}
/**
* STEP 6
* Remove existing projects, then fill the db
* @param {*} filledProjectList
* @param {*} user
*/
async function createProjectsInP5User(filledProjectList, user) {
const userProjects = await Project.find({ user: user._id });
const removeProjects = userProjects.map(async project => Project.remove({ _id: project._id }));
await Q.all(removeProjects);
console.log('deleted old projects!');
const newProjects = filledProjectList.map(async (project) => {
const item = new Project(project);
console.log(`saving ${project.name}`);
await item.save();
});
await Q.all(newProjects);
console.log(`Projects saved to User: ${editorUsername}!`);
}
/**
* STEP 0
* CHECK if user exists, ifnot create one
*
*/
async function checkP5User() {
const user = await User.findOne({ username: editorUsername });
if (!user) {
const ml5user = new User({
username: editorUsername,
email: process.env.ML5_EXAMPLES_EMAIL,
password: process.env.ML5_EXAMPLES_PASS
});
await ml5user.save((saveErr) => {
if (saveErr) throw saveErr;
console.log(`Created a user p5${ml5user}`);
});
}
}
/**
* ---------------------------------------------------------
* --------------------- main ------------------------------
* ---------------------------------------------------------
*/
/**
* MAKE
* Get all the sketches from the ml5-examples repo
* Get the p5 examples
* Dive down into each sketch and get all the files
* Format the sketch files to be save to the db
* Delete existing and save
*/
async function make() {
await checkP5User();
// Get the user
const user = await User.findOne({
username: editorUsername
});
// Get the categories and their examples
const categories = await getCategories();
const categoryExamples = await getCategoryExamples(categories);
const examplesWithResourceTree = await traverseSketchTreeAll(categoryExamples);
const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
const filledProjectList = await fetchSketchContentAll(formattedSketchList);
await createProjectsInP5User(filledProjectList, user);
console.log('done!');
process.exit();
}
/**
* TEST - same as make except reads from file for testing purposes
* Get all the sketches from the ml5-examples repo
* Get the p5 examples
* Dive down into each sketch and get all the files
* Format the sketch files to be save to the db
* Delete existing and save
*/
async function test() {
await checkP5User();
// Get the user
const user = await User.findOne({
username: editorUsername
});
// read from file while testing
const examplesWithResourceTree = JSON.parse(fs.readFileSync('./ml5-examplesWithResourceTree.json'));
const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
const filledProjectList = await fetchSketchContentAll(formattedSketchList);
await createProjectsInP5User(filledProjectList, user);
console.log('done!');
process.exit();
}
/**
* ---------------------------------------------------------
* --------------------- Run -------------------------------
* ---------------------------------------------------------
* Usage:
* If you're testing, change the make() function to test()
* ensure when testing that you've saved some JSON outputs to
* read from so you don't have to make a billion requests all the time
*
* $ GITHUB_ID=<....> GITHUB_SECRET=<...> NODE_ENV=development npm run fetch-examples-ml5
* $ GITHUB_ID=<....> GITHUB_SECRET=<...> npm run fetch-examples-ml5
*/
if (process.env.NODE_ENV === 'development') {
// test()
make(); // replace with test() if you don't want to run all the fetch functions over and over
} else {
make();
}