p5.js-web-editor/server/scripts/examples-ml5.js

import fs from 'fs';
import rp from 'request-promise';
import Q from 'q';
import mongoose from 'mongoose';
import objectID from 'bson-objectid';
import shortid from 'shortid';
import User from '../models/user';
import Project from '../models/project';

// TODO: Change branchName if necessary
const branchName = 'release';
const branchRef = `?ref=${branchName}`;
const baseUrl = 'https://api.github.com/repos/ml5js/ml5-examples/contents';
const clientId = process.env.GITHUB_ID;
const clientSecret = process.env.GITHUB_SECRET;
const editorUsername = process.env.ML5_EXAMPLES_USERNAME;
const headers = {
  'User-Agent': 'p5js-web-editor/0.0.1'
};

const requestOptions = {
  url: baseUrl,
  qs: {
    client_id: clientId,
    client_secret: clientSecret
  },
  method: 'GET',
  headers,
  json: true
};

const mongoConnectionString = process.env.MONGO_URL;
mongoose.connect(mongoConnectionString, { useNewUrlParser: true, useUnifiedTopology: true });
mongoose.set('useCreateIndex', true);
mongoose.connection.on('error', () => {
  console.error('MongoDB Connection Error. Please make sure that MongoDB is running.');
  process.exit(1);
});

/**
 * ---------------------------------------------------------
 * --------------------- helper functions --------------------
 * ---------------------------------------------------------
 */

/**
 * fatten a nested array
 */
function flatten(list) {
  return list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []);
}

/**
 * STEP 1: Get the top level cateogories
 */
async function getCategories() {
  try {
    const options = Object.assign({}, requestOptions);
    options.url = `${options.url}/p5js${branchRef}`;
    const results = await rp(options);

    return results;
  } catch (err) {
    return err;
  }
}

/**
 * STEP 2: Get the examples for each category
 * e.g. Posenet:
 *  - /posenet_image_single
 *  - /posenet_part_selection
 */
async function getCategoryExamples(sketchRootList) {
  const output = [];
  const sketchRootCategories = sketchRootList.map(async (categories) => {
    // let options = Object.assign({url: `${requestOptions.url}/${categories.path}${branchRef}`}, requestOptions)
    const options = Object.assign({}, requestOptions);
    options.url = `${options.url}${categories.path}${branchRef}`;
    // console.log(options)
    const sketchDirs = await rp(options);
    const result = flatten(sketchDirs);

    return result;
  });

  const sketchList = await Q.all(sketchRootCategories);

  sketchList.forEach((sketch) => {
    sketch.forEach((item) => {
      if (item.type === 'dir') output.push(item);
    });
  });

  return output;
}

/**
 *  STEP 3.1: Recursively get the tree of files for each directory
 *  @param parentObject - one sketch directory object
 */
async function traverseSketchTree(parentObject) {
  const output = Object.assign({}, parentObject);

  if (parentObject.type !== 'dir') {
    return output;
  }
  // let options = `https://api.github.com/repos/ml5js/ml5-examples/contents/${sketches.path}${branchRef}`
  const options = Object.assign({}, requestOptions);
  options.url = `${options.url}${parentObject.path}${branchRef}`;

  output.tree = await rp(options);

  output.tree = output.tree.map(file => traverseSketchTree(file));

  output.tree = await Q.all(output.tree);

  return output;
}

/**
 * STEP 3.2: Traverse the sketchtree for all of the sketches
 * @param {*} categoryExamples - all of the categories in an array
 */
async function traverseSketchTreeAll(categoryExamples) {
  // const sketches = categoryExamples.map(async sketch => await traverseSketchTree(sketch));
  const sketches = categoryExamples.map(async sketch => traverseSketchTree(sketch));

  const result = await Q.all(sketches);
  return result;
}

/**
 * Traverse the tree and format into parent child relation
 * @param {*} parentObject
 */
function traverseAndFormat(parentObject) {
  const parent = Object.assign({}, parentObject);

  if (!parentObject.tree) {
    const newid = objectID().toHexString();
    // returns the files
    return {
      name: parent.name,
      url: parent.download_url,
      content: null,
      id: newid,
      _id: newid,
      fileType: 'file'
    };
  }

  const subdir = parentObject.tree.map((item) => {
    const newid = objectID().toHexString();
    if (!item.tree) {
      // returns the files
      return {
        name: item.name,
        url: item.download_url,
        content: null,
        id: newid,
        _id: newid,
        fileType: 'file'
      };
    }

    const feat = {
      name: item.name,
      id: newid,
      _id: newid,
      fileType: 'folder',
      children: traverseAndFormat(item)
    };
    return feat;
  });
  return subdir;
}

/**
 * Traverse the tree and flatten for project.files[]
 * @param {*} projectFileTree
 */
function traverseAndFlatten(projectFileTree) {
  const r = objectID().toHexString();

  const projectRoot = {
    name: 'root',
    id: r,
    _id: r,
    children: [],
    fileType: 'folder'
  };

  let currentParent;

  const output = projectFileTree.reduce(
    (result, item, idx) => {
      if (idx < projectFileTree.length) {
        projectRoot.children.push(item.id);
      }

      if (item.fileType === 'file') {
        if (item.name === 'sketch.js') {
          item.isSelectedFile = true;
        }
        result.push(item);
      }

      // here's where the magic happens *twinkles*
      if (item.fileType === 'folder') {
        // recursively go down the tree of children
        currentParent = traverseAndFlatten(item.children);
        // the above will return an array of the children files
        // concatenate that with the results
        result = result.concat(currentParent); // eslint-disable-line no-param-reassign
        // since we want to get the children ids,
        // we can map the child ids to the current item
        // then push that to our result array to get
        // our flat files array.
        item.children = item.children.map(child => child.id);
        result.push(item);
      }

      return result;
    },
    [projectRoot]
  );

  // Kind of hacky way to remove all roots other than the starting one
  let counter = 0;
  output.forEach((item, idx) => {
    if (item.name === 'root') {
      if (counter === 0) {
        counter += 1;
      } else {
        output.splice(idx, 1);
        counter += 1;
      }
    }
  });

  return output;
}

/**
 * STEP 4
 * Take a parent directory and prepare it for injestion!
 * @param {*} sketch
 * @param {*} user
 */
function formatSketchForStorage(sketch, user) {
  const newProject = new Project({
    _id: shortid.generate(),
    name: sketch.name,
    user: user._id,
    files: [] // <== add files to this array as file objects and add _id reference to children of root
  });

  let projectFiles = traverseAndFormat(sketch);
  projectFiles = traverseAndFlatten(projectFiles);
  newProject.files = projectFiles;
  return newProject;
}

/**
 * format all the sketches using the formatSketchForStorage()
 */
function formatSketchForStorageAll(sketchWithItems, user) {
  let sketchList = sketchWithItems.slice(0);

  sketchList = sketchList.map(sketch => formatSketchForStorage(sketch, user));

  return sketchList;
}

/**
 * Get all the content for the relevant files in project.files[]
 * @param {*} projectObject
 */
async function fetchSketchContent(projectObject) {
  const output = Object.assign({}, JSON.parse(JSON.stringify(projectObject)));

  const newFiles = output.files.map(async (item, i) => {
    // if it is an html or js file
    if (
      (item.fileType === 'file' && item.name.endsWith('.html')) ||
      item.name.endsWith('.js')
    ) {
      const options = Object.assign({}, requestOptions);
      options.url = `${item.url}`;

      if (
        options.url !== undefined ||
        options.url !== null ||
        options.url !== ''
      ) {
        item.content = await rp(options);
        // NOTE: remove the URL property if there's content
        // Otherwise the p5 editor will try to pull from that url
        if (item.content !== null) delete item.url;
      }

      return item;
      // if it is NOT an html or js file
    }

    if (item.url) {
      const cdnRef = `https://cdn.jsdelivr.net/gh/ml5js/ml5-examples@${branchName}${
        item.url.split(branchName)[1]
      }`;
      item.content = cdnRef;
      item.url = cdnRef;
    }

    return item;
  });

  output.files = await Q.all(newFiles);
  return output;
}

/**
 * STEP 5
 * Get all the content for the relevant files in project.files[] for all sketches
 * @param {*} formattedSketchList
 */
async function fetchSketchContentAll(formattedSketchList) {
  let output = formattedSketchList.slice(0);

  output = output.map(async item => fetchSketchContent(item));

  output = await Q.all(output);

  return output;
}

/**
 * STEP 6
 * Remove existing projects, then fill the db
 * @param {*} filledProjectList
 * @param {*} user
 */
async function createProjectsInP5User(filledProjectList, user) {
  const userProjects = await Project.find({ user: user._id });
  const removeProjects = userProjects.map(async project => Project.remove({ _id: project._id }));
  await Q.all(removeProjects);
  console.log('deleted old projects!');

  const newProjects = filledProjectList.map(async (project) => {
    const item = new Project(project);
    console.log(`saving ${project.name}`);
    await item.save();
  });
  await Q.all(newProjects);
  console.log(`Projects saved to User: ${editorUsername}!`);
}

/**
 * STEP 0
 * CHECK if user exists, ifnot create one
 *
 */
async function checkP5User() {
  const user = await User.findOne({ username: editorUsername });

  if (!user) {
    const ml5user = new User({
      username: editorUsername,
      email: process.env.ML5_EXAMPLES_EMAIL,
      password: process.env.ML5_EXAMPLES_PASS
    });

    await ml5user.save((saveErr) => {
      if (saveErr) throw saveErr;
      console.log(`Created a user p5${ml5user}`);
    });
  }
}


/**
 * ---------------------------------------------------------
 * --------------------- main ------------------------------
 * ---------------------------------------------------------
 */

/**
 * MAKE
 * Get all the sketches from the ml5-examples repo
 * Get the p5 examples
 * Dive down into each sketch and get all the files
 * Format the sketch files to be save to the db
 * Delete existing and save
 */
async function make() {
  await checkP5User();
  // Get the user
  const user = await User.findOne({
    username: editorUsername
  });
  // Get the categories and their examples
  const categories = await getCategories();
  const categoryExamples = await getCategoryExamples(categories);
  const examplesWithResourceTree = await traverseSketchTreeAll(categoryExamples);
  const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);
  const filledProjectList = await fetchSketchContentAll(formattedSketchList);
  await createProjectsInP5User(filledProjectList, user);
  console.log('done!');
  process.exit();
}

/**
 * TEST - same as make except reads from file for testing purposes
 * Get all the sketches from the ml5-examples repo
 * Get the p5 examples
 * Dive down into each sketch and get all the files
 * Format the sketch files to be save to the db
 * Delete existing and save
 */
async function test() {
  await checkP5User();
  // Get the user
  const user = await User.findOne({
    username: editorUsername
  });

  // read from file while testing
  const examplesWithResourceTree = JSON.parse(fs.readFileSync('./ml5-examplesWithResourceTree.json'));

  const formattedSketchList = formatSketchForStorageAll(examplesWithResourceTree, user);

  const filledProjectList = await fetchSketchContentAll(formattedSketchList);
  await createProjectsInP5User(filledProjectList, user);
  console.log('done!');
  process.exit();
}

/**
 * ---------------------------------------------------------
 * --------------------- Run -------------------------------
 * ---------------------------------------------------------
 * Usage:
 * If you're testing, change the make() function to test()
 * ensure when testing that you've saved some JSON outputs to
 * read from so you don't have to make a billion requests all the time
 *
 * $ GITHUB_ID=<....> GITHUB_SECRET=<...> NODE_ENV=development npm run fetch-examples-ml5
 * $ GITHUB_ID=<....> GITHUB_SECRET=<...> npm run fetch-examples-ml5
 */

if (process.env.NODE_ENV === 'development') {
  // test()
  make(); // replace with test() if you don't want to run all the fetch functions over and over
} else {
  make();
}