Script to fetch ml5 examples from Github Repo (#1051)
* replaced () with {} to fix implicit return error
* added first version of fetching generative-design examples
* ignore local testing files
* formatting
* updated examples-gg-latest
* updated examples-gg-latest.js
- data files not served via rawgit - hallelujah!
- added jquery
* updated p5 version
* refactoring and code cleanup
* added comment
* comment out link to svgFiles - unused
* moved commented code
* fixed conflicts
* linted examples-gg-latest
* removed console.log of response.data to prevent logging user data to console
* fixed linting error
* initial commit for setting up automated ml5 example fetching
* rm logs from .env replaced with placeholder
* added functions for retrieving all assets from examples sketches
* added recursive walk through to get all directories and files
* added functions t format files for making project
* added full working test
* added comments and formated code
* added comments
* set username at to of code
* added process.exit() on complete
* added linting and fixed errors
* rm await in return
* added es lint disable for reduce()
* rm package-lock.json
* reset package-lock.json to master
* "updated .env.example with dummy logins"
* updated .env.example for consistency
* added p5 user checking
* fixed linting issues
* add webpack config to build ml5 example fetching bundle
This commit is contained in:
Joey Lee
Cassie Tarakajian
parent
538a41c617
commit
a5753b5e4c
|
|
@ -6,9 +6,12 @@ EMAIL_SENDER=<transactional-email-sender>
|
|||
EMAIL_VERIFY_SECRET_TOKEN=whatever_you_want_this_to_be_it_only_matters_for_production
|
||||
EXAMPLE_USER_EMAIL=examples@p5js.org
|
||||
EXAMPLE_USER_PASSWORD=hellop5js
|
||||
GG_EXAMPLES_USERNAME=generativedesign
|
||||
GG_EXAMPLES_EMAIL=benedikt.gross@generative-gestaltung.de
|
||||
GG_EXAMPLES_PASS=generativedesign
|
||||
GG_EXAMPLES_USERNAME=generative-design
|
||||
ML5_EXAMPLES_USERNAME=ml5
|
||||
ML5_EXAMPLES_EMAIL=examples@ml5js.org
|
||||
ML5_EXAMPLES_PASS=helloml5
|
||||
GITHUB_ID=<your-github-client-id>
|
||||
GITHUB_SECRET=<your-github-client-secret>
|
||||
GOOGLE_ID=<your-google-client-id> (use google+ api)
|
||||
|
|
|
|||
|
|
@ -16,8 +16,10 @@
|
|||
"test:ci": "npm run lint",
|
||||
"fetch-examples": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples.js",
|
||||
"fetch-examples-gg": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples-gg.js",
|
||||
"fetch-examples-ml5": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples-ml5.js",
|
||||
"fetch-examples:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples.bundle.js",
|
||||
"fetch-examples-gg:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples-gg.bundle.js",
|
||||
"fetch-examples-ml5:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples-ml5.bundle.js",
|
||||
"heroku-postbuild": "touch .env; npm run build"
|
||||
},
|
||||
"jest": {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,457 @@
|
|||
import fs from 'fs';
|
||||
import rp from 'request-promise';
|
||||
import Q from 'q';
|
||||
import mongoose from 'mongoose';
|
||||
import objectID from 'bson-objectid';
|
||||
import shortid from 'shortid';
|
||||
import User from '../models/user';
|
||||
import Project from '../models/project';
|
||||
|
||||
// TODO: Change branchName if necessary
|
||||
const branchName = 'release';
|
||||
const branchRef = `?ref=${branchName}`;
|
||||
const baseUrl = 'https://api.github.com/repos/ml5js/ml5-examples/contents';
|
||||
const clientId = process.env.GITHUB_ID;
|
||||
const clientSecret = process.env.GITHUB_SECRET;
|
||||
const editorUsername = process.env.ML5_EXAMPLES_USERNAME;
|
||||
const headers = {
|
||||
'User-Agent': 'p5js-web-editor/0.0.1'
|
||||
};
|
||||
|
||||
const requestOptions = {
|
||||
url: baseUrl,
|
||||
qs: {
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret
|
||||
},
|
||||
method: 'GET',
|
||||
headers,
|
||||
json: true
|
||||
};
|
||||
|
||||
const mongoConnectionString = process.env.MONGO_URL;
|
||||
mongoose.connect(mongoConnectionString, {
|
||||
useMongoClient: true
|
||||
});
|
||||
mongoose.connection.on('error', () => {
|
||||
console.error('MongoDB Connection Error. Please make sure that MongoDB is running.');
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
/**
|
||||
* ---------------------------------------------------------
|
||||
* --------------------- helper functions --------------------
|
||||
* ---------------------------------------------------------
|
||||
*/
|
||||
|
||||
/**
|
||||
* fatten a nested array
|
||||
*/
|
||||
function flatten(list) {
|
||||
return list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []);
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 1: Get the top level cateogories
|
||||
*/
|
||||
async function getCategories() {
|
||||
try {
|
||||
const options = Object.assign({}, requestOptions);
|
||||
options.url = `${options.url}/p5js${branchRef}`;
|
||||
const results = await rp(options);
|
||||
|
||||
return results;
|
||||
} catch (err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 2: Get the examples for each category
|
||||
* e.g. Posenet:
|
||||
* - /posenet_image_single
|
||||
* - /posenet_part_selection
|
||||
*/
|
||||
async function getCategoryExamples(sketchRootList) {
|
||||
const output = [];
|
||||
const sketchRootCategories = sketchRootList.map(async (categories) => {
|
||||
// let options = Object.assign({url: `${requestOptions.url}/${categories.path}${branchRef}`}, requestOptions)
|
||||
const options = Object.assign({}, requestOptions);
|
||||
options.url = `${options.url}${categories.path}${branchRef}`;
|
||||
// console.log(options)
|
||||
const sketchDirs = await rp(options);
|
||||
const result = flatten(sketchDirs);
|
||||
|
||||
return result;
|
||||
});
|
||||
|
||||
const sketchList = await Q.all(sketchRootCategories);
|
||||
|
||||
sketchList.forEach((sketch) => {
|
||||
sketch.forEach((item) => {
|
||||
if (item.type === 'dir') output.push(item);
|
||||
});
|
||||
});
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 3.1: Recursively get the tree of files for each directory
|
||||
* @param parentObject - one sketch directory object
|
||||
*/
|
||||
async function traverseSketchTree(parentObject) {
|
||||
const output = Object.assign({}, parentObject);
|
||||
|
||||
if (parentObject.type !== 'dir') {
|
||||
return output;
|
||||
}
|
||||
// let options = `https://api.github.com/repos/ml5js/ml5-examples/contents/${sketches.path}${branchRef}`
|
||||
const options = Object.assign({}, requestOptions);
|
||||
options.url = `${options.url}${parentObject.path}${branchRef}`;
|
||||
|
||||
output.tree = await rp(options);
|
||||
|
||||
output.tree = output.tree.map(file => traverseSketchTree(file));
|
||||
|
||||
output.tree = await Q.all(output.tree);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 3.2: Traverse the sketchtree for all of the sketches
|
||||
* @param {*} categoryExamples - all of the categories in an array
|
||||
*/
|
||||
async function traverseSketchTreeAll(categoryExamples) {
|
||||
// const sketches = categoryExamples.map(async sketch => await traverseSketchTree(sketch));
|
||||
const sketches = categoryExamples.map(async sketch => traverseSketchTree(sketch));
|
||||
|
||||
const result = await Q.all(sketches);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Traverse the tree and format into parent child relation
|
||||
* @param {*} parentObject
|
||||
*/
|
||||
function traverseAndFormat(parentObject) {
|
||||
const parent = Object.assign({}, parentObject);
|
||||
|
||||
if (!parentObject.tree) {
|
||||
const newid = objectID().toHexString();
|
||||
// returns the files
|
||||
return {
|
||||
name: parent.name,
|
||||
url: parent.download_url,
|
||||
content: null,
|
||||
id: newid,
|
||||
_id: newid,
|
||||
fileType: 'file'
|
||||
};
|
||||
}
|
||||
|
||||
const subdir = parentObject.tree.map((item) => {
|
||||
const newid = objectID().toHexString();
|
||||
if (!item.tree) {
|
||||
// returns the files
|
||||
return {
|
||||
name: item.name,
|
||||
url: item.download_url,
|
||||
content: null,
|
||||
id: newid,
|
||||
_id: newid,
|
||||
fileType: 'file'
|
||||
};
|
||||
}
|
||||
|
||||
const feat = {
|
||||
name: item.name,
|
||||
id: newid,
|
||||
_id: newid,
|
||||
fileType: 'folder',
|
||||
children: traverseAndFormat(item)
|
||||
};
|
||||
return feat;
|
||||
});
|
||||
return subdir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Traverse the tree and flatten for project.files[]
|
||||
* @param {*} projectFileTree
|
||||
*/
|
||||
function traverseAndFlatten(projectFileTree) {
|
||||
const r = objectID().toHexString();
|
||||
|
||||
const projectRoot = {
|
||||
name: 'root',
|
||||
id: r,
|
||||
_id: r,
|
||||
children: [],
|
||||
fileType: 'folder'
|
||||
};
|
||||
|
||||
let currentParent;
|
||||
|
||||
const output = projectFileTree.reduce(
|
||||
(result, item, idx) => {
|
||||
if (idx < projectFileTree.length) {
|
||||
projectRoot.children.push(item.id);
|
||||
}
|
||||
|
||||
if (item.fileType === 'file') {
|
||||
if (item.name === 'sketch.js') {
|
||||
item.isSelectedFile = true;
|
||||
}
|
||||
result.push(item);
|
||||
}
|
||||
|
||||
// here's where the magic happens *twinkles*
|
||||
if (item.fileType === 'folder') {
|
||||
// recursively go down the tree of children
|
||||
currentParent = traverseAndFlatten(item.children);
|
||||
// the above will return an array of the children files
|
||||
// concatenate that with the results
|
||||
result = result.concat(currentParent); // eslint-disable-line no-param-reassign
|
||||
// since we want to get the children ids,
|
||||
// we can map the child ids to the current item
|
||||
// then push that to our result array to get
|
||||
// our flat files array.
|
||||
item.children = item.children.map(child => child.id);
|
||||
result.push(item);
|
||||
}
|
||||
|
||||
return result;
|
||||
},
|
||||
[projectRoot]
|
||||
);
|
||||
|
||||
// Kind of hacky way to remove all roots other than the starting one
|
||||
let counter = 0;
|
||||
output.forEach((item, idx) => {
|
||||
if (item.name === 'root') {
|
||||
if (counter === 0) {
|
||||
counter += 1;
|
||||
} else {
|
||||
output.splice(idx, 1);
|
||||
counter += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 4
|
||||
* Take a parent directory and prepare it for injestion!
|
||||
* @param {*} sketch
|
||||
* @param {*} user
|
||||
*/
|
||||
function formatSketchForStorage(sketch, user) {
|
||||
const newProject = new Project({
|
||||
_id: shortid.generate(),
|
||||
name: sketch.name,
|
||||
user: user._id,
|
||||
files: [] // <== add files to this array as file objects and add _id reference to children of root
|
||||
});
|
||||
|
||||
let projectFiles = traverseAndFormat(sketch);
|
||||
projectFiles = traverseAndFlatten(projectFiles);
|
||||
newProject.files = projectFiles;
|
||||
return newProject;
|
||||
}
|
||||
|
||||
/**
|
||||
* format all the sketches using the formatSketchForStorage()
|
||||
*/
|
||||
function formatSketchForStorageAll(sketchWithItems, user) {
|
||||
let sketchList = sketchWithItems.slice(0);
|
||||
|
||||
sketchList = sketchList.map(sketch => formatSketchForStorage(sketch, user));
|
||||
|
||||
return sketchList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all the content for the relevant files in project.files[]
|
||||
* @param {*} projectObject
|
||||
*/
|
||||
async function fetchSketchContent(projectObject) {
|
||||
const output = Object.assign({}, JSON.parse(JSON.stringify(projectObject)));
|
||||
|
||||
const newFiles = output.files.map(async (item, i) => {
|
||||
// if it is an html or js file
|
||||
if (
|
||||
(item.fileType === 'file' && item.name.endsWith('.html')) ||
|
||||
item.name.endsWith('.js')
|
||||
) {
|
||||
const options = Object.assign({}, requestOptions);
|
||||
options.url = `${item.url}`;
|
||||
|
||||
if (
|
||||
options.url !== undefined ||
|
||||
options.url !== null ||
|
||||
options.url !== ''
|
||||
) {
|
||||
item.content = await rp(options);
|
||||
// NOTE: remove the URL property if there's content
|
||||
// Otherwise the p5 editor will try to pull from that url
|
||||
if (item.content !== null) delete item.url;
|
||||
}
|
||||
|
||||
return item;
|
||||
// if it is NOT an html or js file
|
||||
}
|
||||
|
||||
if (item.url) {
|
||||
const cdnRef = `https://cdn.jsdelivr.net/gh/ml5js/ml5-examples@${branchName}${
|
||||
item.url.split(branchName)[1]
|
||||
}`;
|
||||
item.content = cdnRef;
|
||||
item.url = cdnRef;
|
||||
}
|
||||
|
||||
return item;
|
||||
});
|
||||
|
||||
output.files = await Q.all(newFiles);
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 5
|
||||
* Get all the content for the relevant files in project.files[] for all sketches
|
||||
* @param {*} formattedSketchList
|
||||
*/
|
||||
async function fetchSketchContentAll(formattedSketchList) {
|
||||
let output = formattedSketchList.slice(0);
|
||||
|
||||
output = output.map(async item => fetchSketchContent(item));
|
||||
|
||||
output = await Q.all(output);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 6
|
||||
* Remove existing projects, then fill the db
|
||||
* @param {*} filledProjectList
|
||||
* @param {*} user
|
||||
*/
|
||||
async function createProjectsInP5User(filledProjectList, user) {
|
||||
const userProjects = await Project.find({ user: user._id });
|
||||
const removeProjects = userProjects.map(async project => Project.remove({ _id: project._id }));
|
||||
await Q.all(removeProjects);
|
||||
console.log('deleted old projects!');
|
||||
|
||||
const newProjects = filledProjectList.map(async (project) => {
|
||||
const item = new Project(project);
|
||||
console.log(`saving ${project.name}`);
|
||||
await item.save();
|
||||
});
|
||||
await Q.all(newProjects);
|
||||
console.log(`Projects saved to User: ${editorUsername}!`);
|
||||
}
|
||||
|
||||
/**
|
||||
* STEP 0
|
||||
* CHECK if user exists, ifnot create one
|
||||
*
|
||||
*/
|
||||
async function checkP5User() {
|
||||
const user = await User.findOne({ username: editorUsername });
|
||||
|
||||
if (!user) {
|
||||
const ml5user = new User({
|
||||
username: editorUsername,
|
||||
email: process.env.ML5_EXAMPLES_EMAIL,
|
||||
password: process.env.ML5_EXAMPLES_PASS
|
||||
});
|
||||
|
||||
await ml5user.save((saveErr) => {
|
||||
if (saveErr) throw saveErr;
|
||||
console.log(`Created a user p5${ml5user}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* ---------------------------------------------------------
|
||||
* --------------------- main ------------------------------
|
||||
* ---------------------------------------------------------
|
||||
*/
|
||||
|
||||
/**
|
||||
* MAKE
|
||||
* Get all the sketches from the ml5-examples repo
|
||||
* Get the p5 examples
|
||||
* Dive down into each sketch and get all the files
|
||||
* Format the sketch files to be save to the db
|
||||
* Delete existing and save
|
||||
*/
|
||||
async function make() {
|
||||
await checkP5User();
|
||||
// Get the user
|
||||
const user = await User.findOne({
|
||||
username: editorUsername
|
||||
});
|
||||
// Get the categories and their examples
|
||||
const categories = await getCategories();
|
||||
const categoryExamples = await getCategoryExamples(categories);
|
||||
const examplesWithResourceTree = await traverseSketchTreeAll(categoryExamples);
|
||||
const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
|
||||
const filledProjectList = await fetchSketchContentAll(formattedSketchList);
|
||||
await createProjectsInP5User(filledProjectList, user);
|
||||
console.log('done!');
|
||||
process.exit();
|
||||
}
|
||||
|
||||
/**
|
||||
* TEST - same as make except reads from file for testing purposes
|
||||
* Get all the sketches from the ml5-examples repo
|
||||
* Get the p5 examples
|
||||
* Dive down into each sketch and get all the files
|
||||
* Format the sketch files to be save to the db
|
||||
* Delete existing and save
|
||||
*/
|
||||
async function test() {
|
||||
await checkP5User();
|
||||
// Get the user
|
||||
const user = await User.findOne({
|
||||
username: editorUsername
|
||||
});
|
||||
|
||||
// read from file while testing
|
||||
const examplesWithResourceTree = JSON.parse(fs.readFileSync('./ml5-examplesWithResourceTree.json'));
|
||||
|
||||
const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
|
||||
|
||||
const filledProjectList = await fetchSketchContentAll(formattedSketchList);
|
||||
await createProjectsInP5User(filledProjectList, user);
|
||||
console.log('done!');
|
||||
process.exit();
|
||||
}
|
||||
|
||||
/**
|
||||
* ---------------------------------------------------------
|
||||
* --------------------- Run -------------------------------
|
||||
* ---------------------------------------------------------
|
||||
* Usage:
|
||||
* If you're testing, change the make() function to test()
|
||||
* ensure when testing that you've saved some JSON outputs to
|
||||
* read from so you don't have to make a billion requests all the time
|
||||
*
|
||||
* $ GITHUB_ID=<....> GITHUB_SECRET=<...> NODE_ENV=development npm run fetch-examples-ml5
|
||||
* $ GITHUB_ID=<....> GITHUB_SECRET=<...> npm run fetch-examples-ml5
|
||||
*/
|
||||
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
// test()
|
||||
make(); // replace with test() if you don't want to run all the fetch functions over and over
|
||||
} else {
|
||||
make();
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
require('babel-register');
|
||||
require('babel-polyfill');
|
||||
const dotenv = require('dotenv');
|
||||
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
dotenv.config();
|
||||
}
|
||||
require('./examples-ml5.js');
|
||||
|
|
@ -66,6 +66,51 @@ module.exports = [{
|
|||
],
|
||||
},
|
||||
|
||||
module: {
|
||||
loaders: [
|
||||
{
|
||||
test: /\.js$/,
|
||||
exclude: /node_modules/,
|
||||
loader: 'babel-loader',
|
||||
query: {
|
||||
presets: [
|
||||
'react',
|
||||
'env',
|
||||
'stage-0',
|
||||
],
|
||||
plugins: [
|
||||
[
|
||||
'babel-plugin-webpack-loaders', {
|
||||
'config': path.resolve(__dirname, './config.babel.js'),
|
||||
"verbose": false
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
entry: path.resolve(__dirname, '../server/scripts/fetch-examples-ml5.js'),
|
||||
|
||||
output: {
|
||||
path: path.resolve(__dirname, '../dist/'),
|
||||
filename: 'fetch-examples-ml5.bundle.js'
|
||||
},
|
||||
|
||||
target: 'node',
|
||||
|
||||
externals: [nodeExternals()],
|
||||
|
||||
resolve: {
|
||||
extensions: ['*', '.js', '.jsx'],
|
||||
modules: [
|
||||
'client',
|
||||
'node_modules',
|
||||
],
|
||||
},
|
||||
|
||||
module: {
|
||||
loaders: [
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue