r/diversions
1
0
Fork 0
Code Pull requests Releases Activity

WIP with most necessary features

Browse source
This commit is contained in:
Ruben van de Ven 2020-06-18 15:15:37 +02:00
parent 62b6717d1b
commit 58bcc0729f
3 changed files with 327 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
dataset/*
interview/*.mp3

79
index.html
View file

@ -11,10 +11,22 @@
height: 100%; height: 100%;
font-family: sans-serif; font-family: sans-serif;
} }
body.paused canvas{
opacity: .6;
}
canvas{ canvas{
margin:0; margin:0;
width:100vw; width:100vw;
height:100vh; height:100vh;
opacity: 1;
transition: opacity 1s;
}
body.paused #subtitles{
color:darkgray;
} }
/* audio{ /* audio{
@ -29,7 +41,48 @@
color:white; color:white;
text-shadow: black 0 0 10px;; text-shadow: black 0 0 10px;;
text-align: center; text-align: center;
font-size: 20px; font-size: 25px;
}
#annotation{
position: absolute;
top:0;
left:0;
padding: 5px;
background:white;
}
#annotation img{
height: 300px;
}
#annotation .description{
display:none;
}
#time{
position: fixed;
bottom: 0;
right: 0;
padding: 5px;
color:white;
}
#workmsg{
display:none;
}
body.paused #workmsg{
position:absolute;
top: calc(50vh - 20px);
text-align: center;
color:gray;
display: block;
width: 520px;
left: calc(50% - 150px);
font-size: 30px;
} }
</style> </style>
@ -41,14 +94,22 @@
<!-- Will automatically be injected if needed: <script src=https://preview.babylonjs.com/inspector/babylon.inspector.bundle.js></script> --> <!-- Will automatically be injected if needed: <script src=https://preview.babylonjs.com/inspector/babylon.inspector.bundle.js></script> -->
</head> </head>
<body> <body class='paused'>
<canvas id="renderCanvas" touch-action="none"></canvas> <!--touch-action="none" for best results from PEP--> <canvas id="renderCanvas" touch-action="none"></canvas> <!--touch-action="none" for best results from PEP-->
<div id='annotation'> <div id='annotation'>
<template id="annotationContent"> <template id="annotationContentTemplate">
<img id='img'> <img class='img'>
<input type='number' id='weight'> <div class='name'></div>
<div class='description'></div>
Position of image in parade:
<input type='range' min="0" max="110" class='weight'>
</template> </template>
<div id='annotationContent'>
<!-- contents of template will come here through the javascript -->
</div>
<input type='button' value='Next image' id='nextButton'>
</div> </div>
<audio id='interview' <audio id='interview'
@ -64,6 +125,14 @@
</div> </div>
<div id='workmsg'>
<h2>Annotate to listen...</h2>
</div>
<div id='time'>
</div>
<script src="https://cdn.babylonjs.com/babylon.js"></script> <script src="https://cdn.babylonjs.com/babylon.js"></script>
<script src="https://preview.babylonjs.com/loaders/babylonjs.loaders.min.js"></script> <script src="https://preview.babylonjs.com/loaders/babylonjs.loaders.min.js"></script>

337
parade.js
View file

@ -1,6 +1,14 @@
var parade; var parade;
var interview_sorter; var interview_sorter;
var config = {
speedFactor: 0.01, // speed of the shapes
distanceFactor: 20, // distance between the shapes
startOffset: 0.001, // starting position
shapeWidth: 100, // 8 shape of animation: width
shapeHeight: 1300 // 8 shape of animation: height;
}
// polyfill // polyfill
window.performance = window.performance || {}; window.performance = window.performance || {};
performance.now = (function() { performance.now = (function() {
@ -26,7 +34,7 @@ fetch(request)
console.debug(data); console.debug(data);
// loaded data from json, initialise interface // loaded data from json, initialise interface
const canvas = document.getElementById("renderCanvas"); const canvas = document.getElementById("renderCanvas");
parade = new Parade(canvas, data['image']); parade = new Parade(canvas, data['image'], config);
parade.createScene(); parade.createScene();
// parade.scene.debugLayer.show(); // parade.scene.debugLayer.show();
}).catch(error => { }).catch(error => {
@ -35,24 +43,61 @@ fetch(request)
class Parade { class Parade {
constructor(canvasEl, images) { constructor(canvasEl, images, config) {
this.speedFactor = -.05; // animation parameters:
this.distanceFactor = 20; this.config = config;
this.startOffset = 800;
this.maxStepSize = 1;
this.prevRenderTime = null;
this.canvasEl = canvasEl; this.canvasEl = canvasEl;
this.engine = new BABYLON.Engine(canvasEl, true); this.engine = new BABYLON.Engine(canvasEl, true); // create 3d engine
this.isPlaying = false;
// initialise image objects:
this.images = [];
this.imagePositions = []; // index: position, value: image-index
let storedPositions = localStorage.getItem('imagePositions');
if(storedPositions) {
this.imagePositions = JSON.parse(storedPositions);
}
// else or invalid:
if(!storedPositions || this.imagePositions.length != images.length) {
// fill with default: order of the JSON file
for (let index = 0; index < images.length; index++) {
this.imagePositions[index] = index;
}
}
for(let i in images) {
i = parseInt(i);
let pos = this.imagePositions.indexOf(i);
// console.log('pos', , pos);
let img = new ParadeImage(images[i], pos);
this.images.push(img);
// for testing:
// if(this.images.length > 5)
// break;
}
this.setupSubtitles();
this.setupSorter();
// Watch for browser/canvas resize events
window.addEventListener("resize", function () {
this.engine.resize();
}.bind(this));
//Register a render loop to repeatedly render the scene
this.engine.runRenderLoop(this.renderLoop.bind(this));
}
setupSubtitles() {
this.audioEl = document.getElementById('interview'); // audio player this.audioEl = document.getElementById('interview'); // audio player
this.subtitleEl = document.getElementById('subtitles'); // the subtitle div this.subtitleEl = document.getElementById('subtitles'); // the subtitle div
this.trackEl = this.audioEl.querySelector('track'); // the subtitle track this.trackEl = this.audioEl.querySelector('track'); // the subtitle track
this.timeEl = document.getElementById('time'); // the subtitle track
// set up subtitles: // set up subtitles:
this.trackEl.addEventListener("cuechange", (event) => { this.trackEl.addEventListener("cuechange", (event) => {
console.log(event);
let cues = event.target.track.activeCues; let cues = event.target.track.activeCues;
let content = ""; let content = "";
for(let cue of cues) { for(let cue of cues) {
@ -62,22 +107,115 @@ class Parade {
}); });
// initialise image objects: this.audioEl.addEventListener('timeupdate', this.updateTimestring.bind(this));
this.images = []; this.updateTimestring();
for(let i in images) {
this.images.push(new ParadeImage(images[i], i));
// for testing:
// if(this.images.length > 5)
// break;
} }
// Watch for browser/canvas resize events updateTimestring() {
window.addEventListener("resize", function () { let now = this.audioEl.currentTime;
this.engine.resize(); let duration = this.audioEl.duration;
}.bind(this));
//Register a render loop to repeatedly render the scene let timestring = String(Math.floor(now/60)).padStart(2, '0') + ":"
this.engine.runRenderLoop(this.renderLoop.bind(this)); + String(Math.floor(now)%60).padStart(2, '0')
+ " / " + String(Math.floor(duration/60)).padStart(2, '0') + ":"
+ String(Math.floor(duration)%60).padStart(2, '0');
this.timeEl.innerHTML = timestring;
}
setupSorter() {
this.sorterTemplate = document.getElementById('annotationContentTemplate');
this.sorter = document.getElementById('annotation');
this.pauseTimeout = null;
this.imagesToSort = this.images.slice(); // we will keep track of images that needs weighing (clone by reference)
let nextButton = document.getElementById('nextButton');
nextButton.addEventListener('click', this.nextSorterImage.bind(this));
// set maximum to last image index
this.sorterTemplate.content.querySelector('.weight').max = this.images.length - 1;
this.changeSorterImage();
}
changeSorterImage() {
// pick random image
this.imageToSort = this.imagesToSort[Math.floor(Math.random()*this.imagesToSort.length)];
// this.imageToSort = this.imagesToSort[0]; // always the first instead of random
let clone = this.sorterTemplate.content.cloneNode(true);
let imgEl = clone.querySelector(".img");
let descriptionEl = clone.querySelector(".description");
let nameEl = clone.querySelector(".name");
let weightEl = clone.querySelector(".weight");
imgEl.src = this.imageToSort.getImageUrl();
nameEl.innerHTML = this.imageToSort.info['image'];
descriptionEl.innerHTML = this.imageToSort.info['set'];
// weightEl.value = parseInt(image.info['weight']);
weightEl.value = parseInt(this.imageToSort.position); // don't use weight, but the image's actual position
weightEl.addEventListener('input', this.changeSorterWeight.bind(this));
let sorterContent = this.sorter.querySelector('#annotationContent');
sorterContent.innerHTML = "";
sorterContent.appendChild(clone);
}
changeSorterWeight(ev) {
this.play();
// make sure nr is never too big.
let newIndex = Math.min(this.images.length -1 , parseInt(ev.target.value));
this.setImagePosition(this.imageToSort, newIndex);
}
setImagePosition(image, new_position) {
let id = this.images.indexOf(image);
let old_position = this.imagePositions.indexOf(id);
// move from old to new position in the array, see https://stackoverflow.com/a/5306832
this.imagePositions.splice(
new_position, 0,
this.imagePositions.splice(old_position, 1)[0]
);
// update positions of image objects
for(let i in this.images) {
i = parseInt(i);
this.images[i].setPosition(this.imagePositions.indexOf(i), this.t + 2000);
}
// save:
localStorage.setItem('imagePositions', JSON.stringify(this.imagePositions));
}
nextSorterImage() {
this.play();
let i = this.imagesToSort.indexOf(this.imageToSort);
this.imagesToSort.splice(i, 1); // remove from images to position
this.changeSorterImage();
}
play() {
// play audio for n seconds:
// extend timeout if it's already playing
const duration = 10; // seconds
if(this.pauseTimeout) {
clearTimeout(this.pauseTimeout);
}
this.pauseTimeout = setTimeout(() => {
this.pause();
}, duration * 1000);
this.audioEl.play();
document.body.classList.remove('paused');
this.isPlaying = true;
}
pause() {
// pause audio and animation
this.audioEl.pause();
document.body.classList.add('paused');
this.isPlaying = false;
} }
renderLoop() { renderLoop() {
@ -89,43 +227,32 @@ class Parade {
// if(camera.rotation['x'] > .3) { // if(camera.rotation['x'] > .3) {
// camera.rotation['x'] = .3 // camera.rotation['x'] = .3
// } // }
this.animate(); this.animate(); // update positions of the objects
this.scene.render(); this.scene.render();
} }
animate() { animate() {
let t = window.performance.now() - this.startTime; let n = window.performance.now();
// let maxStepSize = Math.inf; let dt = 0;
if(this.isPlaying) {
// todo: make it come to a slow halt, instead of sudden;
dt = n - this.lastRenderTime;
this.t += dt;
}
this.lastRenderTime = n; // also increment when paused;
// if (this.prevRenderTime === null) {
// } else {
// let tdiff = t - this.prevRenderTime;
// if(tdiff < 1) {
// // sometimes multiple render calls within very short moment??
// // skip some work.
// return;
// }
// maxStepSize = Math.abs(this.maxStepSize * tdiff * this.speedFactor);
// }
// this.prevRenderTime = t;
// console.log(tdiff);
t -= 800/this.speedFactor; // start offset ( milliseconds)
// console.log(maxStepSize); // console.log(maxStepSize);
for(let i of this.images){ for(let i of this.images){
i.updateMeshPosition(t*this.speedFactor/1000); i.updateMeshPosition(this.t, dt, this.config);
t-= this.distanceFactor/this.speedFactor;
} }
} }
createScene() { createScene() {
// This creates a basic Babylon Scene object (non-mesh) // This creates a basic Babylon Scene object
this.scene = new BABYLON.Scene(this.engine); this.scene = new BABYLON.Scene(this.engine);
// return this.scene;
// this.scene.clearColor = new BABYLON.Color3(0.6, 0.6, 0.); // color RGB this.scene.clearColor = new BABYLON.Color3(.22,0.27,0.38); // color RGB
// this.scene.clearColor = new BABYLON.Color4(0.6, 0.6, 0.,0); // transparent - RGBA // this.scene.clearColor = new BABYLON.Color4(0.6, 0.6, 0.,0); // transparent - RGBA
this.scene.fogMode = BABYLON.Scene.FOGMODE_EXP; this.scene.fogMode = BABYLON.Scene.FOGMODE_EXP;
this.scene.fogColor = this.scene.clearColor; this.scene.fogColor = this.scene.clearColor;
@ -134,8 +261,8 @@ class Parade {
// let camera = new BABYLON.UniversalCamera("camera1", new BABYLON.Vector3( 0.0, 0.0, 120.86337575312979), this.scene); // let camera = new BABYLON.UniversalCamera("camera1", new BABYLON.Vector3( 0.0, 0.0, 120.86337575312979), this.scene);
// camera.rotation = new BABYLON.Vector3(0, Math.PI, 0); // camera.rotation = new BABYLON.Vector3(0, Math.PI, 0);
// camera.fov = 1.1; // default: 0.8 // camera.fov = 1.1; // default: 0.8
let camera = new BABYLON.UniversalCamera("camera2", new BABYLON.Vector3( -14.40894348196419, 0.9954991699417852, 17.582966906902247), this.scene); let camera = new BABYLON.UniversalCamera("camera2", new BABYLON.Vector3( -11.79906036421707, -0.7028999316894499, 32.43524104627659 ), this.scene);
camera.rotation = new BABYLON.Vector3(0.023121520223909536, 2.5494248799675163, 0); camera.rotation = new BABYLON.Vector3(-0.16070762395712468, 2.4005702973639886, 0);
camera.fov = 1.5; camera.fov = 1.5;
// camera.rotation = new BABYLON.Vector3(0.1, 0,0); // camera.rotation = new BABYLON.Vector3(0.1, 0,0);
// camera.lowerRadiusLimit = 6; // camera.lowerRadiusLimit = 6;
@ -145,13 +272,13 @@ class Parade {
let radius = 60; let radius = 60;
for(let i in this.images) { for(let i in this.images) {
// console.log(i, this.images, this); // create the Meshes with the images in the scene
this.images[i].makeMesh(this.scene); this.images[i].makeMesh(this.scene);
} }
this.startTime = window.performance.now(); this.lastRenderTime = window.performance.now();
// this.scene.registerBeforeRender(this.animate.bind(this)); this.t = 0; // playback timer
return this.scene; return this.scene;
} }
} }
@ -161,59 +288,85 @@ class ParadeImage {
this.info = info; this.info = info;
this.weight = info['weight']; this.weight = info['weight'];
this.position = position; // index in list this.position = position; // index in list
this.target_position = position;
this.target_position_time = 0;
}
getImageUrl() {
return 'dataset/small/'+ this.info['image'];
}
/**
*
* @param {int} new_position new position in parade
* @param {float} time_complete time at which it should have transitioned to this position
*/
setPosition(new_position, time_complete) {
if(this.target_position == new_position) return;
// console.log('changepos', this.target_position, new_position);
this.target_position = new_position;
this.target_position_time = time_complete;
} }
makeMesh(scene) { makeMesh(scene) {
// let videoSrc = this.images[i]['image']; var planeOptions = {
// random fluctuation in radius // TODO: should size be dynamic?
let i = 1;
let radius = 60
let localRadius = (Math.random() * .2 + 1) * radius;
let x = Math.sin(2*Math.PI * i / 1) * localRadius;
let y = Math.cos(2*Math.PI * i / 1) * localRadius;
var planeOpts = {
height: 5.4762, height: 5.4762,
width: 7.3967, width: 7.3967,
sideOrientation: BABYLON.Mesh.DOUBLESIDE sideOrientation: BABYLON.Mesh.DOUBLESIDE // texture should be visible from front and back
}; };
this.mesh = BABYLON.MeshBuilder.CreatePlane("plane", planeOpts, scene);
var ANote0VideoMat = new BABYLON.StandardMaterial("m", scene); this.mesh = BABYLON.MeshBuilder.CreatePlane("plane", planeOptions, scene);
let imgUrl = 'dataset/small/'+ this.info['image']; var mat = new BABYLON.StandardMaterial("m", scene);
// console.log(imgUrl);
ANote0VideoMat.diffuseTexture = new BABYLON.Texture(imgUrl, scene); mat.diffuseTexture = new BABYLON.Texture(this.getImageUrl(), scene);
ANote0VideoMat.roughness = 1; mat.roughness = 1;
ANote0VideoMat.emissiveColor = new BABYLON.Color3.White(); mat.emissiveColor = new BABYLON.Color3.White();
this.mesh.material = ANote0VideoMat; this.mesh.material = mat;
this.mesh.rotation.y = Math.PI; // rotate 180 deg. to avoid looking at backside from the start.
// this.updateMeshPosition(1, Math.inf); //TODO: remove when animating
} }
updateMeshPosition(t){ updateMeshPosition(t, dt, config){
// first see how we are with our moving positions:
if(dt < 1) {
// don't change positions, because we're paused
}
else if(Math.abs(this.position - this.target_position) > 0.00001) {
if(t > this.target_position_time) {
console.error('were we too slow?');
this.position = this.target_position;
}
let tdiff = this.target_position_time - t;
if(tdiff != 0) {
if(tdiff < dt) {
// never place it too far...
this.position = this.target_position;
} else {
this.position += (this.target_position - this.position) / (tdiff/dt);
// console.log(this.position, this.target_position_time, t);
}
}
} else{
// it's practically the same, but set it to avoid any rounding uglyness.
this.position = this.target_position;
}
let p = t; //0; // if 0, disable movement for a bit.
p *= config.speedFactor/1000
p -= config.startOffset/config.speedFactor; // start offset ( milliseconds)
p -= config.distanceFactor * this.position / 1000;
// factors determine scale of animation // factors determine scale of animation
let target_x = Math.sin(2*t)*100; let target_x = Math.sin(2*p)*config.shapeWidth;
let target_z = Math.sin(t)*1300; let target_z = Math.sin(p)*config.shapeHeight;
// let diff_x = target_x - this.mesh.position.x;
// let diff_z = target_z - this.mesh.position.z;
// let stepSize = Math.sqrt( Math.pow(diff_x, 2) + Math.pow(diff_z, 2) );
// if(stepSize > maxStepSize) {
// // console.log('throttle', maxStepSize, stepSize);
// // throttle the step, so switching items looks like movement
// diff_x *= maxStepSize/stepSize;
// diff_z *= maxStepSize/stepSize;
// }
// // console.log(diff_x, diff_z, stepSize);
// this.mesh.position.x += diff_x;
// this.mesh.position.z += diff_z;
this.mesh.position.x = target_x; this.mesh.position.x = target_x;
this.mesh.position.z = target_z; this.mesh.position.z = target_z;
// this.mesh.position = new BABYLON.Vector3(x,0,y);
} }