From 6faf52f61eae9304ad3c1b91a99274229ee6312c Mon Sep 17 00:00:00 2001 From: Joey Lee Date: Thu, 8 Feb 2018 16:02:35 -0500 Subject: [PATCH] Added script to download and save all generative-design project examples to p5-web-editor (#526) * replaced () with {} to fix implicit return error * added first version of fetching generative-design examples * ignore local testing files * formatting * updated examples-gg-latest * updated examples-gg-latest.js - data files not served via rawgit - hallelujah! - added jquery * updated p5 version * refactoring and code cleanup * added comment * comment out link to svgFiles - unused * moved commented code * fixed conflicts * linted examples-gg-latest --- fetch-examples-gg.js | 4 + package.json | 1 + server/examples-gg-latest.js | 577 +++++++++++++++++++++++++++++++++++ 3 files changed, 582 insertions(+) create mode 100644 fetch-examples-gg.js create mode 100644 server/examples-gg-latest.js diff --git a/fetch-examples-gg.js b/fetch-examples-gg.js new file mode 100644 index 00000000..fc354c2c --- /dev/null +++ b/fetch-examples-gg.js @@ -0,0 +1,4 @@ +require('babel-register'); +require('babel-polyfill'); +require('dotenv').config(); +require('./server/examples-gg-latest.js'); diff --git a/package.json b/package.json index dc58e150..563019fc 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "build": "cross-env NODE_ENV=production webpack --config webpack.config.prod.js --progress", "test": "npm run lint", "fetch-examples": "node fetch-examples.js", + "fetch-examples-gg": "node fetch-examples-gg.js", "postinstall": "git submodule update --remote --recursive" }, "main": "index.js", diff --git a/server/examples-gg-latest.js b/server/examples-gg-latest.js new file mode 100644 index 00000000..3fa6b2a1 --- /dev/null +++ b/server/examples-gg-latest.js @@ -0,0 +1,577 @@ +import rp from 'request-promise'; +import Q from 'q'; +import mongoose from 'mongoose'; +import objectID from 'bson-objectid'; +import shortid from 'shortid'; +import eachSeries from 'async/eachSeries'; +import User from './models/user'; +import Project from './models/project'; + +// TODO: change to true when testing! +const testMake = false; + +const defaultHTML = + ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`; + +const defaultCSS = + `html, body { + padding: 0; + margin: 0; +} + +canvas { + vertical-align: top; +} +`; + +// TODO: Change branchName if necessary +const branchName = 'gg4editor'; +const branchRef = `?ref=${branchName}`; +const clientId = process.env.GITHUB_ID; +const clientSecret = process.env.GITHUB_SECRET; + +const headers = { 'User-Agent': 'p5js-web-editor/0.0.1' }; + +mongoose.connect(process.env.MONGO_URL); +mongoose.connection.on('error', () => { + console.error('MongoDB Connection Error. Please make sure that MongoDB is running.'); + process.exit(1); +}); + + +/* --- Helper functions --- */ +const flatten = function flatten(list) { + return list.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []); +}; + +const insert = function insert(_mainString, _insString, _pos) { + let insString = _insString; + const mainString = _mainString; + let pos = _pos; + + if (typeof (pos) === 'undefined') { + pos = 0; + } + if (typeof (insString) === 'undefined') { + insString = ''; + } + return mainString.slice(0, pos) + insString + mainString.slice(pos); +}; + +// TEMP: GATHER DATA FROM STATIC FILE +// - to save time use local json file for now - +const fs = require('fs'); +// gg-github-retrieval.json +// gg-github-newProjects.json +function retrieveDataTemp(fName) { + return new Promise((resolve, reject) => { + const ggdata = `${__dirname}/${fName}`; + resolve(JSON.parse(fs.readFileSync(ggdata))); + }); +} + +/* --- data processing --- */ +// 1. first get the top level directories P and M +// https://api.github.com/repos/generative-design/Code-Package-p5.js/contents?ref=pre-release +function getCodePackage() { + const sketchRootList = []; + const options = { + // url: 'https://api.github.com/repos/generative-design/Code-Package-p5.js/contents', + url: `https://api.github.com/repos/generative-design/Code-Package-p5.js/contents${branchRef}`, + // url: 'https://api.github.com/repos/generative-design/Code-Package-p5.js/contents?ref=pre-release', + qs: { + client_id: clientId, + client_secret: clientSecret + }, + method: 'GET', + headers + }; + + return rp(options).then((res) => { + const json = JSON.parse(res); + + json.forEach((metadata) => { + if (metadata.name.endsWith('P') === true || metadata.name.endsWith('M') === true) { + sketchRootList.push(metadata); + } + }); + + return sketchRootList; + }).catch((err) => { + throw err; + }); +} + +// 2. get the list of all the top-level sketch directories in P and M +function getSketchDirectories(sketchRootList) { + // console.log(sketchRootList); + + return Q.all(sketchRootList.map((sketches) => { + // console.log(sketches) + const options = { + url: `https://api.github.com/repos/generative-design/Code-Package-p5.js/contents/${sketches.path}${branchRef}`, + qs: { + client_id: clientId, + client_secret: clientSecret + }, + method: 'GET', + headers + }; + + return rp(options).then((res) => { + const sketchDirs = flatten(JSON.parse(res)); + + return sketchDirs; + }).catch((err) => { + throw err; + }); + }) + + ).then((output) => { + const sketchList = []; + output.forEach((l) => { + l.forEach((i) => { + if (i.type === 'dir') { sketchList.push(i); } + }); + }); + + return sketchList; + }); +} + + +// 3. For each sketch item in the sketchList, append the tree contents to each item +function appendSketchItemLinks(sketchList) { + return Q.all(sketchList.map((sketches) => { + const options = { + // url: `${sketches.url}?client_id=${clientId}&client_secret=${clientSecret}`, + url: `https://api.github.com/repos/generative-design/Code-Package-p5.js/contents/${sketches.path}${branchRef}`, + qs: { + client_id: clientId, + client_secret: clientSecret + }, + method: 'GET', + headers + }; + + return rp(options).then((res) => { + const sketchItems = JSON.parse(res); + sketches.tree = sketchItems; + + return sketchList; + }); + })); +} + +// 4. for each sketch item +function getSketchItems(sketchList) { + const completeSketchPkg = []; + + /* eslint-disable */ + return Q.all(sketchList[0].map(sketch => Q.all(sketch.tree.map((item) => { + if (item.name === 'data') { + const options = { + url: `https://api.github.com/repos/generative-design/Code-Package-p5.js/contents/${item.path}${branchRef}`, + qs: { + client_id: clientId, + client_secret: clientSecret + }, + method: 'GET', + headers + }; + + return rp(options).then((res) => { + sketch.data = JSON.parse(res); + return sketch; + }).catch((err) => { + throw err; + }); + } + // pass + })))).then(() => sketchList[0]); + /* eslint-enable */ +} + +function formatSketchForStorage(sketch, user) { + // get the sketch download url + function getSketchDownloadUrl(_sketch) { + let downloadUrl = ''; + _sketch.tree.forEach((item) => { + if (item.name === 'sketch.js') { + downloadUrl += item.download_url; + } + }); + return downloadUrl; + } + + // create a new project template + const a = objectID().toHexString(); + const b = objectID().toHexString(); + const c = objectID().toHexString(); + const r = objectID().toHexString(); + + + const newProject = new Project({ + name: sketch.name, + user: user._id, + files: [{ + name: 'root', + id: r, + _id: r, + children: [a, b, c], + fileType: 'folder' + }, + { + name: 'sketch.js', + content: getSketchDownloadUrl(sketch), + id: a, + _id: a, + isSelectedFile: true, + fileType: 'file', + children: [] + }, + { + name: 'index.html', + content: defaultHTML, + id: b, + _id: b, + fileType: 'file', + children: [] + }, + { + name: 'style.css', + content: defaultCSS, + id: c, + _id: c, + fileType: 'file', + children: [] + } + ], + _id: shortid.generate() + + }); + + + // get any additional js files url + // TODO: this could probably be optimized - so many loops! + function addAdditionalJs(_sketch, newProjectObject) { + const output = newProjectObject.files; + + _sketch.tree.forEach((item) => { + if (item.name.endsWith('.js') === true && item.name !== 'sketch.js') { + const itemId = objectID().toHexString(); + + const projectItem = { + name: item.name, + content: item.download_url, + id: itemId, + _id: itemId, + fileType: 'file', + children: [] + }; + + // push the projectItem to the files array + output.push(projectItem); + // add the ID to the root children id array + output[0].children.push(projectItem.id); + // add the JS reference to the defaultHTML + output[2].content = insert(output[2].content, ``, output[2].content.search('')); + } + }); + + // if there's output return it, if not pass + if (output.length > 0) { + return output; + } + console.log('ERR with addAdditionalJs!'); + return null; + } + newProject.files = addAdditionalJs(sketch, newProject); + + // fill the data folder with the data files for each sketch + function fillDataFolder(_sketch, newProjectObject) { + const dataInProject = _sketch.data || []; + const output = newProjectObject.files; + + // create a data folder if there are data in the project + if (dataInProject.length > 0) { + const objId = objectID().toHexString(); + + const dataFolderIndex = output.length; + output.push({ + name: 'data', + id: objId, + _id: objId, + children: [], + fileType: 'folder' + }); + // add assets folder inside root + output[0].children.push(objId); + + // for all the data, stuff them into the folder + dataInProject.forEach((item) => { + const fileID = objectID().toHexString(); + output.push({ + name: item.name, + url: item.download_url, + content: null, + id: fileID, + _id: fileID, + children: [], + fileType: 'file' + }); + // console.log(`create data: ${item.name}`); + // add asset file inside the newly created assets folder at last position + output[dataFolderIndex].children.push(fileID); + }); + } + return output; + } + newProject.files = fillDataFolder(sketch, newProject); + + // return the newProject + // console.log(newProject); + return newProject; +} + +// format all the sketches using the formatSketchForStorage() +function formatAllSketches(sketchList) { + return new Promise((resolve, reject) => { + User.findOne({ username: 'generative-design' }, (err, user) => { + const output = []; + sketchList.forEach((sketch) => { + const newProject = formatSketchForStorage(sketch, user); + output.push(newProject); + }); + resolve(output); + }); + }); +} + + +// get all the sketch data content and download to the newProjects array +function getAllSketchContent(newProjectList) { + /* eslint-disable */ + return Q.all(newProjectList.map(newProject => Q.all(newProject.files.map((sketchFile, i) => { + /* + sketchFile.name.endsWith(".mp4") !== true && + sketchFile.name.endsWith(".ogg") !== true && + sketchFile.name.endsWith(".otf") !== true && + sketchFile.name.endsWith(".ttf") !== true && + sketchFile.name.endsWith(".vvt") !== true && + sketchFile.name.endsWith(".jpg") !== true && + sketchFile.name.endsWith(".png") !== true && + sketchFile.name.endsWith(".svg") !== true + */ + + if (sketchFile.fileType === 'file' && + sketchFile.content != null && + sketchFile.name.endsWith('.html') !== true && + sketchFile.name.endsWith('.css') !== true && + sketchFile.name.endsWith('.js') === true + ) { + const options = { + url: newProject.files[i].content, + qs: { + client_id: clientId, + client_secret: clientSecret + }, + method: 'GET', + headers + }; + + // console.log("CONVERT ME!") + return rp(options).then((res) => { + newProject.files[i].content = res; + return newProject; + }).catch((err) => { + throw err; + }); + } + if (newProject.files[i].url) { + return new Promise((resolve, reject) => { + console.log(sketchFile.name); + // https://cdn.rawgit.com/opensourcedesign/fonts/2f220059/gnu-freefont_freesans/FreeSans.otf?raw=true + // "https://raw.githubusercontent.com/generative-design/Code-Package-p5.js/gg4editor/01_P/P_3_2_1_01/data/FreeSans.otf", + const rawGitRef = `https://cdn.rawgit.com/${newProject.files[i].url.split('.com/')[1]}`; + sketchFile.content = rawGitRef; + sketchFile.url = rawGitRef; + + // replace ref in sketch.js ==> should serve from the file? + // newProject.files[1].content = newProject.files[1].content.replace(`'data/${sketchFile.name}'`, `'${rawGitRef}'`); + resolve(newProject); + }); + } + })).catch((err) => { + throw err; + }))).then(() => newProjectList); + /* eslint-enable */ +} + +// Save the project in the db +function createProjectsInP5user(newProjectList) { + User.findOne({ username: 'generative-design' }, (err, user) => { + if (err) throw err; + + eachSeries(newProjectList, (newProject, sketchCallback) => { + newProject.save((saveErr, savedProject) => { + if (saveErr) throw saveErr; + console.log(`Created a new project in p5 user: ${savedProject.name}`); + sketchCallback(); + }); + }); + }).catch((err) => { + throw err; + }); +} + + +/* --- Main --- */ +// remove any of the old files and add the new stuffs to the UI +function getp5User() { + User.findOne({ username: 'generative-design' }, (err, user) => { + if (err) throw err; + + let ggUser = user; + if (!ggUser) { + ggUser = new User({ + username: process.env.GG_EXAMPLES_USERNAME, + email: process.env.GG_EXAMPLES_PASS, + password: process.env.GG_EXAMPLES_EMAIL + }); + ggUser.save((saveErr) => { + if (saveErr) throw saveErr; + console.log(`Created a user p5${ggUser}`); + }); + } + + Project.find({ user: ggUser._id }, (projectsErr, projects) => { + // if there are already some sketches, delete them + console.log('Deleting old projects...'); + projects.forEach((project) => { + Project.remove({ _id: project._id }, (removeErr) => { + if (removeErr) throw removeErr; + }); + }); + }); + + + if (testMake === true) { + // Run for Testing + // Run for production + return getCodePackage() + .then(getSketchDirectories) + .then(appendSketchItemLinks) + .then(getSketchItems) + // .then(saveRetrievalToFile) + .then(formatAllSketches) + .then(getAllSketchContent) + // .then(saveNewProjectsToFile) + .then(createProjectsInP5user); + } + // Run for production + return getCodePackage() + .then(getSketchDirectories) + .then(appendSketchItemLinks) + .then(getSketchItems) + .then(formatAllSketches) + .then(getAllSketchContent) + .then(createProjectsInP5user); + }); +} +// Run the entire process +getp5User(); + + +/* --- Tester Functions --- */ +/** * Tester Functions - IGNORE BELOW +@ below are functions for testing +output etc +** */ +// formatSketchForStorage(P_2_1_1_04); +// formatSketchForStorage(M_1_5_04); + +// function linkToSvgFiles(newProjectList){ +// return Q.all(newProjectList.map(newProject => Q.all(newProject.files.map((sketchFile, i) => { + +// return new Promise( (resolve, reject) => { +// if( sketchFile.fileType == 'file' && +// sketchFile.name.endsWith(".svg") == true ){ +// // handle cases for svgs + +// const rawGitRef = `https://cdn.rawgit.com/${newProject.files[i].url.split(".com/")[1]}`; +// sketchFile.content = rawGitRef; +// newProject.files[1].content = newProject.files[1].content.replace(`'data/${sketchFile.name}'`, `'${rawGitRef}'`); + +// resolve(newProject) +// } +// }) + +// })).catch((err) => { +// throw err +// }))).then(() => { + +// return newProjectList; +// }) +// } + +// checking function +function doNext(output) { + console.log(JSON.stringify(output)); + console.log(output.length); +} + +// save output to terminal +function saveRetrievalToFile(output) { + return new Promise((resolve, reject) => { + fs.writeFileSync('server/gg-github-raw.json', JSON.stringify(output)); + resolve(output); + }); +} + +// save output to terminal +function saveNewProjectsToFile(output) { + return new Promise((resolve, reject) => { + fs.writeFileSync('server/gg-github-newProjects.json', JSON.stringify(output)); + resolve(output); + }); +} +// test make without deleting all projects etc +// function make() { +// return retrieveDataTemp('gg-github-retrieval.json') +// .then(formatAllSketches) +// .then(getAllSketchContent) +// .then(linkToFontFiles) +// .then(saveToFile); +// } +// make();