From 0b6e4661cba199bb478c9fca52124bf91eda7b9f Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Mon, 19 Apr 2021 22:47:06 +0200 Subject: [PATCH] Sankey v0.1 --- www/graph.css | 12 +- www/graph.js | 399 +++++++++++++++++++++++++++++++++++++++++++++---- www/index.html | 4 +- 3 files changed, 384 insertions(+), 31 deletions(-) diff --git a/www/graph.css b/www/graph.css index 44b1849..2210ad3 100644 --- a/www/graph.css +++ b/www/graph.css @@ -19,7 +19,7 @@ body { margin: 0; - overflow: hidden; + /* overflow: hidden; */ /* background: linear-gradient(to top, #040308, #AD4A28, #DD723C, #fc7001, #dcb697, #9ba5ae, #3e5879, #020b1a); */ background:linear-gradient(to top, #414141, #99a6b8); font-family: sans-serif; @@ -212,4 +212,14 @@ a:hover{ stroke-width: 6px; stroke: white; fill:none; +} + +#alluvial{ + /* position:fixed; */ + top:0; + left:0; +} + +#alluvial .flow_label text{ + font-size: 30; } \ No newline at end of file diff --git a/www/graph.js b/www/graph.js index c56ab1c..5a46c55 100644 --- a/www/graph.js +++ b/www/graph.js @@ -35,6 +35,18 @@ const CONFIG = { "geo_property_map": { // used to work around a bug in SMW "City Coordinates": "City", "Country Coordinates": "City Country", + }, + + "alluvial_cats": ["Deployments"], + "alluvial_props": { + "Country": ["Country"], + "City": ["City"], + // ["Deployment type"], // TODO: select this + // ["Institution type"], // TODO: select this (local gov, etc.) + "Dataset": ["Datasets used"], + "Company": ["Managed by", "Provided by", "Developped by (institutions)"], + "Tech": ["Technologies Used", "Software Deployed"], + "Funding": ["Funded by"], } }; @@ -172,14 +184,14 @@ class NodeMap { box1.left > box2.right || box1.bottom < box2.top || box1.top > box2.bottom) - if (overlap) { + if (overlap) { // TODO: try to flip labels horizontally to see if that helps el.classList.add('overlapping'); overlapping = true; break; } } - if(!overlapping) + if (!overlapping) el.classList.remove('overlapping'); } @@ -422,29 +434,29 @@ class NodeMap { enter => { let group = enter.append("g").attr("class", "link"); group.append("path") - .attr("marker-end", "url(#arrowHead)") - .attr('id', (d, i) => 'linkid_' + i) - .on("mouseover", function (ev,link) { - d3.select(this).classed('hover',true); - const nodes = document.getElementsByClassName('node'); - for(let n of nodes){ - const d = d3.select(n).datum(); - if(d == link.target || d == link.source){ - n.classList.add('linkHover'); + .attr("marker-end", "url(#arrowHead)") + .attr('id', (d, i) => 'linkid_' + i) + .on("mouseover", function (ev, link) { + d3.select(this).classed('hover', true); + const nodes = document.getElementsByClassName('node'); + for (let n of nodes) { + const d = d3.select(n).datum(); + if (d == link.target || d == link.source) { + n.classList.add('linkHover'); + } } - } - // console.log(l); - }).on("mouseout", function (ev, link) { - d3.select(this).classed('hover',false); - const nodes = document.getElementsByClassName('linkHover'); - for(let n of nodes){ - n.classList.remove('linkHover'); - } - // l.classed('hover',false); - // l.target.classed('hover',false); - // l.source.classed('hover',false); - // console.log(l,'l'); - }); + // console.log(l); + }).on("mouseout", function (ev, link) { + d3.select(this).classed('hover', false); + const nodes = document.getElementsByClassName('linkHover'); + for (let n of nodes) { + n.classList.remove('linkHover'); + } + // l.classed('hover',false); + // l.target.classed('hover',false); + // l.source.classed('hover',false); + // console.log(l,'l'); + }); group.filter((l) => l.name != "City").append("text").attr("class", "labelText").text(function (l) { return l.name; }); @@ -580,7 +592,148 @@ class NodeMap { } } -var mapGraph = new NodeMap('#map') +class AlluvialMap { + constructor(parent) { + this.root = d3.select(parent); + this.resizeEvent = window.addEventListener('resize', this.resize.bind(this)); + } + + setData(results) { + this.graph = JsonToAlluvial(results); + } + + resize() { + this.width = window.innerWidth; + this.height = window.innerHeight; + this.vbWidth = 2000; + this.vbHeight = 2000; + this.svg + .attr("viewBox", [0, 0, this.vbWidth, this.vbHeight]) + .attr("width", this.width) + .attr("height", this.height); + + } + + reset() { + console.warning("Not yet implemented"); + } + + update() { + // this.alluvial = this.parseGraph(this.store.graph) + } + + + render() { + this.svg = this.root.append('svg') + this.resize(); + + this.sankey = d3.sankey() + .nodeId(d => d.id) + // .nodeAlign('justify') + .nodeWidth(40) // height + .nodePadding(10) + .extent([[1, 5], [2000 - 1, 2000 - 5]]); + console.log(this.graph); + let s = this.sankey({ + nodes: this.graph.nodes.map(d => Object.assign({}, d)), + links: this.graph.links.map(d => Object.assign({}, d)) + }); + + this.nodes = s.nodes; + this.links = s.links; + + const scale = d3.scaleOrdinal(d3.schemeCategory10); + const color = (c) => c == "Unknown" ? "#333": scale(c); + + + this.svg.append("g") + .attr("stroke", "#000") + .selectAll("rect") + .data(this.nodes) + .join("rect") + .attr("x", d => d.y0) + .attr("y", d => d.x0) + .attr("height", d => d.x1 - d.x0) + .attr("width", d => d.y1 - d.y0) + // .attr("fill", d => color(d.box === undefined ? d.name : d.box)) + .attr("fill", d => color(d.name)) + // .attr("fill", 'blue') + .append("title") + .text(d => `${d.box}: ${d.name}\n${d.value}`); + + const link = this.svg.append("g") + .attr("fill", "none") + .attr("stroke-opacity", 0.5) + .selectAll("g") + .data(this.links) + .join("g") + .style("mix-blend-mode", "multiply"); + + + + const edgeColor = 'path'; // either: path, none, input, output + if (edgeColor === "path") { + const gradient = link.append("linearGradient") + .attr("id", (d,i) => { + const id = `link-${i}`; // thanks https://talk.observablehq.com/t/how-do-i-work-with-the-d3-sankey-example/1696/3 + d.uid = `url(#${id})`; + return id; + }) + .attr("gradientUnits", "userSpaceOnUse") + .attr("y1", d => d.source.x1) + .attr("y2", d => d.target.x0) + .attr("x1",0) + .attr("x2", 0); + // .attr("y1", "0%") + // .attr("y2", "100%") + // .attr("x1", "0%") + // .attr("x2", "0%"); + + gradient.append("stop") + .attr("offset", "0%") + .attr("stop-color", d => color(d.source.name)); + + gradient.append("stop") + .attr("offset", "100%") + .attr("stop-color", d => color(d.target.name)); + } + + link.append("path") + // .attr("d", d3.sankeyLinkHorizontal()) + .attr("d", d3.linkVertical() + .source(function (d) { return [d.y0, d.source.x1]; }) + .target(function (d) { return [d.y1, d.target.x0]; }) + ) + + .attr("stroke", d => edgeColor === "none" ? "#aaa" + : edgeColor === "path" ? d.uid + : edgeColor === "input" ? color(d.source) + : color(d.target)) + // .attr("stroke", 'red') + .attr("stroke-width", d => Math.max(1, d.width)); + + link.append("title") + .text(d => `${d.source.name} → ${d.target.name}\n${d.value}`); + + this.svg.append("g") + .attr("font-family", "sans-serif") + // .attr("font-size", 10) + .attr("class", 'flow_label') + .selectAll("text") + .data(this.nodes) + .join("text") + .attr("x", d => (d.y1 + d.y0) / 2) + .attr("y", d => d.x0 < this.width / 2 ? d.x1 + 6 : d.x0 - 6) + .attr("dy", "0.35em") + .attr("text-anchor", d => d.x0 < this.width / 2 ? "start" : "end") + .text(d => d.name); + + return this.svg.node(); + } + +} + + JsonToGraph = function (data) { let nodes = []; @@ -656,11 +809,192 @@ JsonToGraph = function (data) { return { nodes, links } } +JsonToAlluvial = function (data) { + let boxes = {}; + let links = []; + + for (const box in CONFIG.alluvial_props) { + boxes[box] = [] + } + + const relevant_categories = CONFIG.alluvial_cats.map(c => "Category:" + c); + + // gather boxes ('node' in alluvial) + for (const node_id in data) { + if (Object.hasOwnProperty.call(data, node_id)) { + let node = data[node_id]; + + // we only want deployments + if (!relevant_categories.includes(node.printouts["Category"][0].fulltext)) { + continue; + } + + for (const box in CONFIG.alluvial_props) { + let has_any = false; + for (const prop of CONFIG.alluvial_props[box]) { + // console.log(box, prop); + if (!node.printouts.hasOwnProperty(prop)) { + continue; + } + for (const target_node of node.printouts[prop]) { + boxes[box].push(target_node.fulltext) + has_any = true; + } + } + if (!has_any) { + boxes[box].push("Unknown"); + } + } + } + } + + const nodes = []; + // reduce the items in the boxes by count. And convert to 'nodes' of the Sankey + for (const box in CONFIG.alluvial_props) { + boxes[box] = boxes[box].reduce(function (acc, curr) { + if (typeof acc[curr] == 'undefined') { + acc[curr] = 1; + } else { + acc[curr] += 1; + } + + return acc; + }, {}); + boxSorted = Object.keys(boxes[box]).sort(function (a, b) { return boxes[box][a] - boxes[box][b] }).reverse() + useBox = boxSorted.splice(0, 10); + restBox = boxSorted; + // gather stats: + boxes[box] = { + 'counts': boxes[box], + 'use': useBox, + 'rest': restBox, + } + + for (const name of useBox) { + nodes.push({ + name, + "id": box + "::" + name, + box, + "about": "" // TODO, what can we say here? + }) + }; + + if (restBox.length) { + nodes.push({ + "name": "Other", + "id": box + "::" + "Other", + box, + "about": restBox, //some extra stats for use + }) + } + } + + let linkMap = {}; + // another round: now, we collect the links + for (const node_id in data) { + if (Object.hasOwnProperty.call(data, node_id)) { + let node = data[node_id]; + + // we only want deployments + if (!relevant_categories.includes(node.printouts["Category"][0].fulltext)) { + continue; + } + + let prev_box = null; + for (const box in CONFIG.alluvial_props) { + let cur_box = []; + let has_any = false; + for (const prop of CONFIG.alluvial_props[box]) { + // console.log(box, prop); + if (!node.printouts.hasOwnProperty(prop)) { + continue; + } + for (const target_node of node.printouts[prop]) { + if (boxes[box].use.includes(target_node.fulltext)) { + cur_box.push(box + "::" + target_node.fulltext) + } else { + cur_box.push(box + "::" + "Other") + } + has_any = true; + } + } + if (!has_any) { + cur_box.push(box + "::" + "Unknown") + } + if (prev_box !== null) { + // TODO: links + for (let source of prev_box) { + for (let target of cur_box) { + if (typeof linkMap[source] == 'undefined') { + linkMap[source] = {}; + } + if (typeof linkMap[source][target] == 'undefined') { + linkMap[source][target] = 0; + } + linkMap[source][target] += 1 / (prev_box.length * cur_box.length) // TODO: is this right? + // links.push({ + // source, + // target, + // value: 1 / (prev_box.length * cur_box.length) // TODO: is this right? + // }) + } + } + } + prev_box = cur_box; + } + } + } + + for (const source in linkMap) { + for (const target in linkMap[source]) { + links.push({ + source, + target, + value: linkMap[source][target] + }) + } + } + + + console.log(boxes); + + // let fixes = 0; + // nodes.forEach((node) => { + // // work around SMW bug in Ask. 2/2 + // for (const idx of Object.keys(CONFIG.geo_property_map)) { + // if (!node.printouts[idx].length) { + // const srcProp = CONFIG.geo_property_map[idx]; + // // only retrievable if we know the name of the location (which for some reason _is_ often there) + // if (!node.printouts[srcProp].length) { + // continue; + // } + // const loc = node.printouts[srcProp][0].fulltext; + // if (!smwBugFixLocationMaps.hasOwnProperty(loc)) { + // continue; + // } + // node.printouts[idx] = smwBugFixLocationMaps[loc] + // fixes++; + // } + // } + // }) + + // console.debug(`Fixed location for ${fixes} nodes`); + + // console.log(links.length); + + // nodeMap = Object.fromEntries(nodes.map(d => [d['id'], d])); + // links = links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(l => { + // l.source = nodeMap[l.source]; + // l.target = nodeMap[l.target]; + // return l; + // }); + + + + return { nodes, links, boxes } +} -var typeFilterList = [ - // 'Deployments' -] class Store { constructor(graph, parent) { this.nodes = graph.nodes; @@ -768,6 +1102,9 @@ class Store { } +var mapGraph = new NodeMap('#map') +var alluvialGraph = new AlluvialMap('#alluvial') + // REQUEST ATLAS & GRAPH const req_data = new Request(CONFIG.dataUrl, { method: 'GET' }); const req_world = new Request('https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json', { method: 'GET' }); @@ -778,11 +1115,15 @@ Promise.all([fetch(req_data), fetch(req_world)]) .then(([data, world]) => { var graph = JsonToGraph(data); var store = new Store(graph, '#filters'); + mapGraph.setWorld(world); mapGraph.setStore(store); - store.render() + // console.log(); + alluvialGraph.setData(data.results); + store.render() mapGraph.render() + alluvialGraph.render() }).catch(error => { console.error(error); diff --git a/www/index.html b/www/index.html index 99ba437..5fc6b0e 100644 --- a/www/index.html +++ b/www/index.html @@ -9,6 +9,7 @@
+
@@ -20,7 +21,8 @@ - + +