r
/
greens_report
forked from security_vision/semantic_graph
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Sankey v0.1

This commit is contained in:
Ruben van de Ven 2021-04-19 22:47:06 +02:00
parent 1277131f0d
commit 0b6e4661cb
3 changed files with 384 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
www/graph.css
View File

@ -19,7 +19,7 @@
body {
margin: 0;
overflow: hidden;
/* overflow: hidden; */
/* background: linear-gradient(to top, #040308, #AD4A28, #DD723C, #fc7001, #dcb697, #9ba5ae, #3e5879, #020b1a); */
background:linear-gradient(to top, #414141, #99a6b8);
font-family: sans-serif;
@ -212,4 +212,14 @@ a:hover{
stroke-width: 6px;
stroke: white;
fill:none;
}
#alluvial{
/* position:fixed; */
top:0;
left:0;
}
#alluvial .flow_label text{
font-size: 30;
}

399
www/graph.js
View File

@ -35,6 +35,18 @@ const CONFIG = {
"geo_property_map": { // used to work around a bug in SMW
"City Coordinates": "City",
"Country Coordinates": "City Country",
},
"alluvial_cats": ["Deployments"],
"alluvial_props": {
"Country": ["Country"],
"City": ["City"],
// ["Deployment type"], // TODO: select this
// ["Institution type"], // TODO: select this (local gov, etc.)
"Dataset": ["Datasets used"],
"Company": ["Managed by", "Provided by", "Developped by (institutions)"],
"Tech": ["Technologies Used", "Software Deployed"],
"Funding": ["Funded by"],
}
};
@ -172,14 +184,14 @@ class NodeMap {
box1.left > box2.right ||
box1.bottom < box2.top ||
box1.top > box2.bottom)
if (overlap) {
if (overlap) {
// TODO: try to flip labels horizontally to see if that helps
el.classList.add('overlapping');
overlapping = true;
break;
}
}
if(!overlapping)
if (!overlapping)
el.classList.remove('overlapping');
}
@ -422,29 +434,29 @@ class NodeMap {
enter => {
let group = enter.append("g").attr("class", "link");
group.append("path")
.attr("marker-end", "url(#arrowHead)")
.attr('id', (d, i) => 'linkid_' + i)
.on("mouseover", function (ev,link) {
d3.select(this).classed('hover',true);
const nodes = document.getElementsByClassName('node');
for(let n of nodes){
const d = d3.select(n).datum();
if(d == link.target || d == link.source){
n.classList.add('linkHover');
.attr("marker-end", "url(#arrowHead)")
.attr('id', (d, i) => 'linkid_' + i)
.on("mouseover", function (ev, link) {
d3.select(this).classed('hover', true);
const nodes = document.getElementsByClassName('node');
for (let n of nodes) {
const d = d3.select(n).datum();
if (d == link.target || d == link.source) {
n.classList.add('linkHover');
}
}
}
// console.log(l);
}).on("mouseout", function (ev, link) {
d3.select(this).classed('hover',false);
const nodes = document.getElementsByClassName('linkHover');
for(let n of nodes){
n.classList.remove('linkHover');
}
// l.classed('hover',false);
// l.target.classed('hover',false);
// l.source.classed('hover',false);
// console.log(l,'l');
});
// console.log(l);
}).on("mouseout", function (ev, link) {
d3.select(this).classed('hover', false);
const nodes = document.getElementsByClassName('linkHover');
for (let n of nodes) {
n.classList.remove('linkHover');
}
// l.classed('hover',false);
// l.target.classed('hover',false);
// l.source.classed('hover',false);
// console.log(l,'l');
});
group.filter((l) => l.name != "City").append("text").attr("class", "labelText").text(function (l) {
return l.name;
});
@ -580,7 +592,148 @@ class NodeMap {
}
}
var mapGraph = new NodeMap('#map')
class AlluvialMap {
constructor(parent) {
this.root = d3.select(parent);
this.resizeEvent = window.addEventListener('resize', this.resize.bind(this));
}
setData(results) {
this.graph = JsonToAlluvial(results);
}
resize() {
this.width = window.innerWidth;
this.height = window.innerHeight;
this.vbWidth = 2000;
this.vbHeight = 2000;
this.svg
.attr("viewBox", [0, 0, this.vbWidth, this.vbHeight])
.attr("width", this.width)
.attr("height", this.height);
}
reset() {
console.warning("Not yet implemented");
}
update() {
// this.alluvial = this.parseGraph(this.store.graph)
}
render() {
this.svg = this.root.append('svg')
this.resize();
this.sankey = d3.sankey()
.nodeId(d => d.id)
// .nodeAlign('justify')
.nodeWidth(40) // height
.nodePadding(10)
.extent([[1, 5], [2000 - 1, 2000 - 5]]);
console.log(this.graph);
let s = this.sankey({
nodes: this.graph.nodes.map(d => Object.assign({}, d)),
links: this.graph.links.map(d => Object.assign({}, d))
});
this.nodes = s.nodes;
this.links = s.links;
const scale = d3.scaleOrdinal(d3.schemeCategory10);
const color = (c) => c == "Unknown" ? "#333": scale(c);
this.svg.append("g")
.attr("stroke", "#000")
.selectAll("rect")
.data(this.nodes)
.join("rect")
.attr("x", d => d.y0)
.attr("y", d => d.x0)
.attr("height", d => d.x1 - d.x0)
.attr("width", d => d.y1 - d.y0)
// .attr("fill", d => color(d.box === undefined ? d.name : d.box))
.attr("fill", d => color(d.name))
// .attr("fill", 'blue')
.append("title")
.text(d => `${d.box}: ${d.name}\n${d.value}`);
const link = this.svg.append("g")
.attr("fill", "none")
.attr("stroke-opacity", 0.5)
.selectAll("g")
.data(this.links)
.join("g")
.style("mix-blend-mode", "multiply");
const edgeColor = 'path'; // either: path, none, input, output
if (edgeColor === "path") {
const gradient = link.append("linearGradient")
.attr("id", (d,i) => {
const id = `link-${i}`; // thanks https://talk.observablehq.com/t/how-do-i-work-with-the-d3-sankey-example/1696/3
d.uid = `url(#${id})`;
return id;
})
.attr("gradientUnits", "userSpaceOnUse")
.attr("y1", d => d.source.x1)
.attr("y2", d => d.target.x0)
.attr("x1",0)
.attr("x2", 0);
// .attr("y1", "0%")
// .attr("y2", "100%")
// .attr("x1", "0%")
// .attr("x2", "0%");
gradient.append("stop")
.attr("offset", "0%")
.attr("stop-color", d => color(d.source.name));
gradient.append("stop")
.attr("offset", "100%")
.attr("stop-color", d => color(d.target.name));
}
link.append("path")
// .attr("d", d3.sankeyLinkHorizontal())
.attr("d", d3.linkVertical()
.source(function (d) { return [d.y0, d.source.x1]; })
.target(function (d) { return [d.y1, d.target.x0]; })
)
.attr("stroke", d => edgeColor === "none" ? "#aaa"
: edgeColor === "path" ? d.uid
: edgeColor === "input" ? color(d.source)
: color(d.target))
// .attr("stroke", 'red')
.attr("stroke-width", d => Math.max(1, d.width));
link.append("title")
.text(d => `${d.source.name} → ${d.target.name}\n${d.value}`);
this.svg.append("g")
.attr("font-family", "sans-serif")
// .attr("font-size", 10)
.attr("class", 'flow_label')
.selectAll("text")
.data(this.nodes)
.join("text")
.attr("x", d => (d.y1 + d.y0) / 2)
.attr("y", d => d.x0 < this.width / 2 ? d.x1 + 6 : d.x0 - 6)
.attr("dy", "0.35em")
.attr("text-anchor", d => d.x0 < this.width / 2 ? "start" : "end")
.text(d => d.name);
return this.svg.node();
}
}
JsonToGraph = function (data) {
let nodes = [];
@ -656,11 +809,192 @@ JsonToGraph = function (data) {
return { nodes, links }
}
JsonToAlluvial = function (data) {
let boxes = {};
let links = [];
for (const box in CONFIG.alluvial_props) {
boxes[box] = []
}
const relevant_categories = CONFIG.alluvial_cats.map(c => "Category:" + c);
// gather boxes ('node' in alluvial)
for (const node_id in data) {
if (Object.hasOwnProperty.call(data, node_id)) {
let node = data[node_id];
// we only want deployments
if (!relevant_categories.includes(node.printouts["Category"][0].fulltext)) {
continue;
}
for (const box in CONFIG.alluvial_props) {
let has_any = false;
for (const prop of CONFIG.alluvial_props[box]) {
// console.log(box, prop);
if (!node.printouts.hasOwnProperty(prop)) {
continue;
}
for (const target_node of node.printouts[prop]) {
boxes[box].push(target_node.fulltext)
has_any = true;
}
}
if (!has_any) {
boxes[box].push("Unknown");
}
}
}
}
const nodes = [];
// reduce the items in the boxes by count. And convert to 'nodes' of the Sankey
for (const box in CONFIG.alluvial_props) {
boxes[box] = boxes[box].reduce(function (acc, curr) {
if (typeof acc[curr] == 'undefined') {
acc[curr] = 1;
} else {
acc[curr] += 1;
}
return acc;
}, {});
boxSorted = Object.keys(boxes[box]).sort(function (a, b) { return boxes[box][a] - boxes[box][b] }).reverse()
useBox = boxSorted.splice(0, 10);
restBox = boxSorted;
// gather stats:
boxes[box] = {
'counts': boxes[box],
'use': useBox,
'rest': restBox,
}
for (const name of useBox) {
nodes.push({
name,
"id": box + "::" + name,
box,
"about": "" // TODO, what can we say here?
})
};
if (restBox.length) {
nodes.push({
"name": "Other",
"id": box + "::" + "Other",
box,
"about": restBox, //some extra stats for use
})
}
}
let linkMap = {};
// another round: now, we collect the links
for (const node_id in data) {
if (Object.hasOwnProperty.call(data, node_id)) {
let node = data[node_id];
// we only want deployments
if (!relevant_categories.includes(node.printouts["Category"][0].fulltext)) {
continue;
}
let prev_box = null;
for (const box in CONFIG.alluvial_props) {
let cur_box = [];
let has_any = false;
for (const prop of CONFIG.alluvial_props[box]) {
// console.log(box, prop);
if (!node.printouts.hasOwnProperty(prop)) {
continue;
}
for (const target_node of node.printouts[prop]) {
if (boxes[box].use.includes(target_node.fulltext)) {
cur_box.push(box + "::" + target_node.fulltext)
} else {
cur_box.push(box + "::" + "Other")
}
has_any = true;
}
}
if (!has_any) {
cur_box.push(box + "::" + "Unknown")
}
if (prev_box !== null) {
// TODO: links
for (let source of prev_box) {
for (let target of cur_box) {
if (typeof linkMap[source] == 'undefined') {
linkMap[source] = {};
}
if (typeof linkMap[source][target] == 'undefined') {
linkMap[source][target] = 0;
}
linkMap[source][target] += 1 / (prev_box.length * cur_box.length) // TODO: is this right?
// links.push({
// source,
// target,
// value: 1 / (prev_box.length * cur_box.length) // TODO: is this right?
// })
}
}
}
prev_box = cur_box;
}
}
}
for (const source in linkMap) {
for (const target in linkMap[source]) {
links.push({
source,
target,
value: linkMap[source][target]
})
}
}
console.log(boxes);
// let fixes = 0;
// nodes.forEach((node) => {
// // work around SMW bug in Ask. 2/2
// for (const idx of Object.keys(CONFIG.geo_property_map)) {
// if (!node.printouts[idx].length) {
// const srcProp = CONFIG.geo_property_map[idx];
// // only retrievable if we know the name of the location (which for some reason _is_ often there)
// if (!node.printouts[srcProp].length) {
// continue;
// }
// const loc = node.printouts[srcProp][0].fulltext;
// if (!smwBugFixLocationMaps.hasOwnProperty(loc)) {
// continue;
// }
// node.printouts[idx] = smwBugFixLocationMaps[loc]
// fixes++;
// }
// }
// })
// console.debug(`Fixed location for ${fixes} nodes`);
// console.log(links.length);
// nodeMap = Object.fromEntries(nodes.map(d => [d['id'], d]));
// links = links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(l => {
// l.source = nodeMap[l.source];
// l.target = nodeMap[l.target];
// return l;
// });
return { nodes, links, boxes }
}
var typeFilterList = [
// 'Deployments'
]
class Store {
constructor(graph, parent) {
this.nodes = graph.nodes;
@ -768,6 +1102,9 @@ class Store {
}
var mapGraph = new NodeMap('#map')
var alluvialGraph = new AlluvialMap('#alluvial')
// REQUEST ATLAS & GRAPH
const req_data = new Request(CONFIG.dataUrl, { method: 'GET' });
const req_world = new Request('https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json', { method: 'GET' });
@ -778,11 +1115,15 @@ Promise.all([fetch(req_data), fetch(req_world)])
.then(([data, world]) => {
var graph = JsonToGraph(data);
var store = new Store(graph, '#filters');
mapGraph.setWorld(world);
mapGraph.setStore(store);
store.render()
// console.log();
alluvialGraph.setData(data.results);
store.render()
mapGraph.render()
alluvialGraph.render()
}).catch(error => {
console.error(error);

4
www/index.html
View File

@ -9,6 +9,7 @@
<body>
<div id='map'></div>
<div id='alluvial'></div>
<div id="filters">
@ -20,7 +21,8 @@
<script src="https://d3js.org/topojson.v3.min.js"></script>
<script src="//unpkg.com/d3-geo-zoom"></script>
<script src="//unpkg.com/d3fc@14.0.1"></script>
<script src="https://unpkg.com/d3-sankey@0"></script>
<!-- <script src="//unpkg.com/d3fc@14.0.1"></script> -->
<script src="graph.js"></script>
</body>