Script to fetch ml5 examples from Github Repo (#1051)

* replaced () with {} to fix implicit return error

* added first version of fetching generative-design examples

* ignore local testing files

* formatting

* updated examples-gg-latest

* updated examples-gg-latest.js

- data files not served via rawgit - hallelujah!
- added jquery

* updated p5 version

* refactoring and code cleanup

* added comment

* comment out link to svgFiles - unused

* moved commented code

* fixed conflicts

* linted examples-gg-latest

* removed console.log of response.data to prevent logging user data to console

* fixed linting error

* initial commit for setting up automated ml5 example fetching

* rm logs from .env replaced with placeholder

* added functions for retrieving all assets from examples sketches

* added recursive walk through to get all directories and files

* added functions t format files for making project

* added full working test

* added comments and formated code

* added comments

* set username at to of code

* added process.exit() on complete

* added linting and fixed errors

* rm await in return

* added es lint disable for reduce()

* rm package-lock.json

* reset package-lock.json to master

* "updated .env.example with dummy logins"

* updated .env.example for consistency

* added p5 user checking

* fixed linting issues

* add webpack config to build ml5 example fetching bundle

.env.example

@@ -6,9 +6,12 @@ EMAIL_SENDER=<transactional-email-sender>
 EMAIL_VERIFY_SECRET_TOKEN=whatever_you_want_this_to_be_it_only_matters_for_production
 EXAMPLE_USER_EMAIL=examples@p5js.org
 EXAMPLE_USER_PASSWORD=hellop5js
+GG_EXAMPLES_USERNAME=generativedesign
 GG_EXAMPLES_EMAIL=benedikt.gross@generative-gestaltung.de
 GG_EXAMPLES_PASS=generativedesign
-GG_EXAMPLES_USERNAME=generative-design
+ML5_EXAMPLES_USERNAME=ml5
+ML5_EXAMPLES_EMAIL=examples@ml5js.org
+ML5_EXAMPLES_PASS=helloml5
 GITHUB_ID=<your-github-client-id>
 GITHUB_SECRET=<your-github-client-secret>
 GOOGLE_ID=<your-google-client-id> (use google+ api)

package.json

@@ -16,8 +16,10 @@
     "test:ci": "npm run lint",
     "fetch-examples": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples.js",
     "fetch-examples-gg": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples-gg.js",
+    "fetch-examples-ml5": "cross-env NODE_ENV=development node ./server/scripts/fetch-examples-ml5.js",
     "fetch-examples:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples.bundle.js",
     "fetch-examples-gg:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples-gg.bundle.js",
+    "fetch-examples-ml5:prod": "cross-env NODE_ENV=production node ./dist/fetch-examples-ml5.bundle.js",
     "heroku-postbuild": "touch .env; npm run build"
   },
   "jest": {

server/scripts/examples-ml5.js

@@ -0,0 +1,457 @@
+import fs from 'fs';
+import rp from 'request-promise';
+import Q from 'q';
+import mongoose from 'mongoose';
+import objectID from 'bson-objectid';
+import shortid from 'shortid';
+import User from '../models/user';
+import Project from '../models/project';
+
+// TODO: Change branchName if necessary
+const branchName = 'release';
+const branchRef = `?ref=${branchName}`;
+const baseUrl = 'https://api.github.com/repos/ml5js/ml5-examples/contents';
+const clientId = process.env.GITHUB_ID;
+const clientSecret = process.env.GITHUB_SECRET;
+const editorUsername = process.env.ML5_EXAMPLES_USERNAME;
+const headers = {
+  'User-Agent': 'p5js-web-editor/0.0.1'
+};
+
+const requestOptions = {
+  url: baseUrl,
+  qs: {
+    client_id: clientId,
+    client_secret: clientSecret
+  },
+  method: 'GET',
+  headers,
+  json: true
+};
+
+const mongoConnectionString = process.env.MONGO_URL;
+mongoose.connect(mongoConnectionString, {
+  useMongoClient: true
+});
+mongoose.connection.on('error', () => {
+  console.error('MongoDB Connection Error. Please make sure that MongoDB is running.');
+  process.exit(1);
+});
+
+/**
+ * ---------------------------------------------------------
+ * --------------------- helper functions --------------------
+ * ---------------------------------------------------------
+ */
+
+/**
+ * fatten a nested array
+ */
+function flatten(list) {
+  return list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []);
+}
+
+/**
+ * STEP 1: Get the top level cateogories
+ */
+async function getCategories() {
+  try {
+    const options = Object.assign({}, requestOptions);
+    options.url = `${options.url}/p5js${branchRef}`;
+    const results = await rp(options);
+
+    return results;
+  } catch (err) {
+    return err;
+  }
+}
+
+/**
+ * STEP 2: Get the examples for each category
+ * e.g. Posenet:
+ *  - /posenet_image_single
+ *  - /posenet_part_selection
+ */
+async function getCategoryExamples(sketchRootList) {
+  const output = [];
+  const sketchRootCategories = sketchRootList.map(async (categories) => {
+    // let options = Object.assign({url: `${requestOptions.url}/${categories.path}${branchRef}`}, requestOptions)
+    const options = Object.assign({}, requestOptions);
+    options.url = `${options.url}${categories.path}${branchRef}`;
+    // console.log(options)
+    const sketchDirs = await rp(options);
+    const result = flatten(sketchDirs);
+
+    return result;
+  });
+
+  const sketchList = await Q.all(sketchRootCategories);
+
+  sketchList.forEach((sketch) => {
+    sketch.forEach((item) => {
+      if (item.type === 'dir') output.push(item);
+    });
+  });
+
+  return output;
+}
+
+/**
+ *  STEP 3.1: Recursively get the tree of files for each directory
+ *  @param parentObject - one sketch directory object
+ */
+async function traverseSketchTree(parentObject) {
+  const output = Object.assign({}, parentObject);
+
+  if (parentObject.type !== 'dir') {
+    return output;
+  }
+  // let options = `https://api.github.com/repos/ml5js/ml5-examples/contents/${sketches.path}${branchRef}`
+  const options = Object.assign({}, requestOptions);
+  options.url = `${options.url}${parentObject.path}${branchRef}`;
+
+  output.tree = await rp(options);
+
+  output.tree = output.tree.map(file => traverseSketchTree(file));
+
+  output.tree = await Q.all(output.tree);
+
+  return output;
+}
+
+/**
+ * STEP 3.2: Traverse the sketchtree for all of the sketches
+ * @param {*} categoryExamples - all of the categories in an array
+ */
+async function traverseSketchTreeAll(categoryExamples) {
+  // const sketches = categoryExamples.map(async sketch => await traverseSketchTree(sketch));
+  const sketches = categoryExamples.map(async sketch => traverseSketchTree(sketch));
+
+  const result = await Q.all(sketches);
+  return result;
+}
+
+/**
+ * Traverse the tree and format into parent child relation
+ * @param {*} parentObject
+ */
+function traverseAndFormat(parentObject) {
+  const parent = Object.assign({}, parentObject);
+
+  if (!parentObject.tree) {
+    const newid = objectID().toHexString();
+    // returns the files
+    return {
+      name: parent.name,
+      url: parent.download_url,
+      content: null,
+      id: newid,
+      _id: newid,
+      fileType: 'file'
+    };
+  }
+
+  const subdir = parentObject.tree.map((item) => {
+    const newid = objectID().toHexString();
+    if (!item.tree) {
+      // returns the files
+      return {
+        name: item.name,
+        url: item.download_url,
+        content: null,
+        id: newid,
+        _id: newid,
+        fileType: 'file'
+      };
+    }
+
+    const feat = {
+      name: item.name,
+      id: newid,
+      _id: newid,
+      fileType: 'folder',
+      children: traverseAndFormat(item)
+    };
+    return feat;
+  });
+  return subdir;
+}
+
+/**
+ * Traverse the tree and flatten for project.files[]
+ * @param {*} projectFileTree
+ */
+function traverseAndFlatten(projectFileTree) {
+  const r = objectID().toHexString();
+
+  const projectRoot = {
+    name: 'root',
+    id: r,
+    _id: r,
+    children: [],
+    fileType: 'folder'
+  };
+
+  let currentParent;
+
+  const output = projectFileTree.reduce(
+    (result, item, idx) => {
+      if (idx < projectFileTree.length) {
+        projectRoot.children.push(item.id);
+      }
+
+      if (item.fileType === 'file') {
+        if (item.name === 'sketch.js') {
+          item.isSelectedFile = true;
+        }
+        result.push(item);
+      }
+
+      // here's where the magic happens *twinkles*
+      if (item.fileType === 'folder') {
+        // recursively go down the tree of children
+        currentParent = traverseAndFlatten(item.children);
+        // the above will return an array of the children files
+        // concatenate that with the results
+        result = result.concat(currentParent); // eslint-disable-line no-param-reassign
+        // since we want to get the children ids,
+        // we can map the child ids to the current item
+        // then push that to our result array to get
+        // our flat files array.
+        item.children = item.children.map(child => child.id);
+        result.push(item);
+      }
+
+      return result;
+    },
+    [projectRoot]
+  );
+
+  // Kind of hacky way to remove all roots other than the starting one
+  let counter = 0;
+  output.forEach((item, idx) => {
+    if (item.name === 'root') {
+      if (counter === 0) {
+        counter += 1;
+      } else {
+        output.splice(idx, 1);
+        counter += 1;
+      }
+    }
+  });
+
+  return output;
+}
+
+/**
+ * STEP 4
+ * Take a parent directory and prepare it for injestion!
+ * @param {*} sketch
+ * @param {*} user
+ */
+function formatSketchForStorage(sketch, user) {
+  const newProject = new Project({
+    _id: shortid.generate(),
+    name: sketch.name,
+    user: user._id,
+    files: [] // <== add files to this array as file objects and add _id reference to children of root
+  });
+
+  let projectFiles = traverseAndFormat(sketch);
+  projectFiles = traverseAndFlatten(projectFiles);
+  newProject.files = projectFiles;
+  return newProject;
+}
+
+/**
+ * format all the sketches using the formatSketchForStorage()
+ */
+function formatSketchForStorageAll(sketchWithItems, user) {
+  let sketchList = sketchWithItems.slice(0);
+
+  sketchList = sketchList.map(sketch => formatSketchForStorage(sketch, user));
+
+  return sketchList;
+}
+
+/**
+ * Get all the content for the relevant files in project.files[]
+ * @param {*} projectObject
+ */
+async function fetchSketchContent(projectObject) {
+  const output = Object.assign({}, JSON.parse(JSON.stringify(projectObject)));
+
+  const newFiles = output.files.map(async (item, i) => {
+    // if it is an html or js file
+    if (
+      (item.fileType === 'file' && item.name.endsWith('.html')) ||
+      item.name.endsWith('.js')
+    ) {
+      const options = Object.assign({}, requestOptions);
+      options.url = `${item.url}`;
+
+      if (
+        options.url !== undefined ||
+        options.url !== null ||
+        options.url !== ''
+      ) {
+        item.content = await rp(options);
+        // NOTE: remove the URL property if there's content
+        // Otherwise the p5 editor will try to pull from that url
+        if (item.content !== null) delete item.url;
+      }
+
+      return item;
+      // if it is NOT an html or js file
+    }
+
+    if (item.url) {
+      const cdnRef = `https://cdn.jsdelivr.net/gh/ml5js/ml5-examples@${branchName}${
+        item.url.split(branchName)[1]
+      }`;
+      item.content = cdnRef;
+      item.url = cdnRef;
+    }
+
+    return item;
+  });
+
+  output.files = await Q.all(newFiles);
+  return output;
+}
+
+/**
+ * STEP 5
+ * Get all the content for the relevant files in project.files[] for all sketches
+ * @param {*} formattedSketchList
+ */
+async function fetchSketchContentAll(formattedSketchList) {
+  let output = formattedSketchList.slice(0);
+
+  output = output.map(async item => fetchSketchContent(item));
+
+  output = await Q.all(output);
+
+  return output;
+}
+
+/**
+ * STEP 6
+ * Remove existing projects, then fill the db
+ * @param {*} filledProjectList
+ * @param {*} user
+ */
+async function createProjectsInP5User(filledProjectList, user) {
+  const userProjects = await Project.find({ user: user._id });
+  const removeProjects = userProjects.map(async project => Project.remove({ _id: project._id }));
+  await Q.all(removeProjects);
+  console.log('deleted old projects!');
+
+  const newProjects = filledProjectList.map(async (project) => {
+    const item = new Project(project);
+    console.log(`saving ${project.name}`);
+    await item.save();
+  });
+  await Q.all(newProjects);
+  console.log(`Projects saved to User: ${editorUsername}!`);
+}
+
+/**
+ * STEP 0
+ * CHECK if user exists, ifnot create one
+ *
+ */
+async function checkP5User() {
+  const user = await User.findOne({ username: editorUsername });
+
+  if (!user) {
+    const ml5user = new User({
+      username: editorUsername,
+      email: process.env.ML5_EXAMPLES_EMAIL,
+      password: process.env.ML5_EXAMPLES_PASS
+    });
+
+    await ml5user.save((saveErr) => {
+      if (saveErr) throw saveErr;
+      console.log(`Created a user p5${ml5user}`);
+    });
+  }
+}
+
+
+/**
+ * ---------------------------------------------------------
+ * --------------------- main ------------------------------
+ * ---------------------------------------------------------
+ */
+
+/**
+ * MAKE
+ * Get all the sketches from the ml5-examples repo
+ * Get the p5 examples
+ * Dive down into each sketch and get all the files
+ * Format the sketch files to be save to the db
+ * Delete existing and save
+ */
+async function make() {
+  await checkP5User();
+  // Get the user
+  const user = await User.findOne({
+    username: editorUsername
+  });
+  // Get the categories and their examples
+  const categories = await getCategories();
+  const categoryExamples = await getCategoryExamples(categories);
+  const examplesWithResourceTree = await traverseSketchTreeAll(categoryExamples);
+  const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
+  const filledProjectList = await fetchSketchContentAll(formattedSketchList);
+  await createProjectsInP5User(filledProjectList, user);
+  console.log('done!');
+  process.exit();
+}
+
+/**
+ * TEST - same as make except reads from file for testing purposes
+ * Get all the sketches from the ml5-examples repo
+ * Get the p5 examples
+ * Dive down into each sketch and get all the files
+ * Format the sketch files to be save to the db
+ * Delete existing and save
+ */
+async function test() {
+  await checkP5User();
+  // Get the user
+  const user = await User.findOne({
+    username: editorUsername
+  });
+
+  // read from file while testing
+  const examplesWithResourceTree = JSON.parse(fs.readFileSync('./ml5-examplesWithResourceTree.json'));
+
+  const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
+
+  const filledProjectList = await fetchSketchContentAll(formattedSketchList);
+  await createProjectsInP5User(filledProjectList, user);
+  console.log('done!');
+  process.exit();
+}
+
+/**
+ * ---------------------------------------------------------
+ * --------------------- Run -------------------------------
+ * ---------------------------------------------------------
+ * Usage:
+ * If you're testing, change the make() function to test()
+ * ensure when testing that you've saved some JSON outputs to
+ * read from so you don't have to make a billion requests all the time
+ *
+ * $ GITHUB_ID=<....> GITHUB_SECRET=<...> NODE_ENV=development npm run fetch-examples-ml5
+ * $ GITHUB_ID=<....> GITHUB_SECRET=<...> npm run fetch-examples-ml5
+ */
+
+if (process.env.NODE_ENV === 'development') {
+  // test()
+  make(); // replace with test() if you don't want to run all the fetch functions over and over
+} else {
+  make();
+}

server/scripts/fetch-examples-ml5.js

@@ -0,0 +1,8 @@
+require('babel-register');
+require('babel-polyfill');
+const dotenv = require('dotenv');
+
+if (process.env.NODE_ENV === 'development') {
+  dotenv.config();
+}
+require('./examples-ml5.js');

webpack/config.examples.js

@@ -66,6 +66,51 @@ module.exports = [{
     ],
   },
 
+  module: {
+    loaders: [
+      {
+        test: /\.js$/,
+        exclude: /node_modules/,
+        loader: 'babel-loader',
+        query: {
+          presets: [
+            'react',
+            'env',
+            'stage-0',
+          ],
+          plugins: [
+            [
+              'babel-plugin-webpack-loaders', {
+                'config': path.resolve(__dirname, './config.babel.js'),
+                "verbose": false
+              }
+            ]
+          ]
+        },
+      }
+    ],
+  },
+},
+{
+  entry: path.resolve(__dirname, '../server/scripts/fetch-examples-ml5.js'),
+
+  output: {
+    path: path.resolve(__dirname, '../dist/'),
+    filename: 'fetch-examples-ml5.bundle.js'
+  },
+
+  target: 'node',
+
+  externals: [nodeExternals()],
+
+  resolve: {
+    extensions: ['*', '.js', '.jsx'],
+    modules: [
+      'client',
+      'node_modules',
+    ],
+  },
+
   module: {
     loaders: [
       {