r/greens_report
1
0
Fork 0
forked from security_vision/semantic_graph
Code Issues Pull requests Projects Releases Wiki Activity

Little ordering of graph

Browse source
This commit is contained in:
Ruben van de Ven 2021-03-31 16:24:46 +02:00
parent e07b88e9b5
commit 1573eae7e4
5 changed files with 20040 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
wiki_relations.py
View file

@ -16,7 +16,7 @@ default_categories = [
'Deployments',
'Dataset',
'City',
# 'Country',
# 'Country',# for deployments without city we should configure Geolocation
]
parser = argparse.ArgumentParser(description='Turn wiki into nodes & links, usable by d3-force.')
@ -142,6 +142,12 @@ def getPropertiesForPage(page, session, collection):
for sub_obj in data['query']['sobj']:
subSubjectId = sub_obj['subject']
if '#0##_QUERY' in subSubjectId:
logger.info(f"Skip query subobj {subSubjectId}")
continue
if '#0##_ERR' in subSubjectId:
logger.info(f"Skip error subobj {subSubjectId}")
continue
for rel in sub_obj['data']:
addToCollection(subSubjectId, rel, collection, subjectId)

19720
www/d3.v6.js vendored Normal file
View file

File diff suppressed because it is too large Load diff

151
www/graph.css Normal file
View file

@ -0,0 +1,151 @@
:root{
--color1: #f94144;
--color2: #f3722c;
--color3: #f8961e;
/* --color4: #f9844a; */
--color5: #f9c74f;
--color6: #90be6d;
--color7: #43aa8b;
--color8: #4d908e;
--color9: #577590;
--color10: #277da1;
--hover-color: var(--color1);
--selected-color: var(--color1);
--selected-color: var(--color1);
}
body {
margin: 0;
overflow: hidden;
background: linear-gradient(to top, #040308, #AD4A28, #DD723C, #fc7001, #dcb697, #9ba5ae, #3e5879, #020b1a);
font-family: sans-serif;
min-height: 100vh;
}
svg {
cursor: grab;
font-family: sans-serif;
}
svg.dragging {
cursor: grabbing;
}
svg .links line{
stroke: darkgray;
stroke-width: 1;
}
.links text{
display:none;
font-size:5pt;
text-anchor: middle;
fill: whitesmoke;
}
.node text{
text-anchor: middle;
}
.node circle{
fill: white;
}
.node:hover{
cursor: pointer;
}
.node:hover circle{
stroke: var(--hover-color);
stroke-width: 5px;
}
.node.selected circle{
stroke: var(--selected-color);
stroke-width: 5px;
}
.node.City circle{
display:none;
}
.node.City{
fill:white;
stroke: black;
stroke-width: .5px;
font-size: 20px;
}
.node.Person circle {
fill: lightgreen
}
.node.Technology circle {
fill: lightcoral;
}
.node.Deployment circle {
fill: lightblue;
}
.node.Institution circle {
fill: lightgoldenrodyellow
}
.node.Dataset circle {
fill: plum
}
/* .node.Person circle {
fill: var(--color2)
}
.node.Technology circle {
fill: var(--color3);
}
.node.Deployment circle {
fill: var(--color5);
}
.node.Institution circle {
fill: var(--color6)
}
.node.Dataset circle {
fill: var(--color7)
} */
#nodeInfo{
position: fixed;
display:block;
right:20px;
bottom:20px;
background:white;
padding: 10px;
border: solid 1px #ccc;
}
#nodeInfo.hidden{
display:none;
}
#nodeInfo h2{
margin: 0;
padding: 0;
}
#nodeInfo iframe{
width: 50vw;
height: calc(100vh - 40px - 20px - 30px);
}
#closeInfo{
cursor: pointer;
position: absolute;
right: 10px;
top: 10px;
}
#closeInfo:hover{
color: var(--hover-color);
}
a, a:link{
text-decoration: none;
}
a:hover{
text-decoration: underline;
}

382
www/graph.js
View file

@ -1,13 +1,17 @@
const CONFIG = {
'nodeSize': 16,
'nodeSize': 8,
'baseUrl': 'https://www.securityvision.io/wiki/index.php/',
'dataUrl': '../semantic_data.json',
'preSimulate': true, // run simulation before starting, so we don't start with lines jumping around
'preSimulate': false, // run simulation before starting, so we don't start with lines jumping around
'labels': {
'rotate': true,
},
'countries': ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia ', 'Portugal', 'Finland ', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
'countries': ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia ', 'Portugal', 'Finland ', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland'],
'eu': {
'lonMin': -10,
'lonMax': 35
}
};
@ -20,6 +24,9 @@ function getSizeForNode(node) {
if (node.parent) {
return 2;
}
// if(getCategories(node).indexOf('City') !== -1) {
// return 2;
// }
return CONFIG.nodeSize;
}
@ -91,10 +98,16 @@ function getTitle(obj) {
}
return obj['@id'].split('#', 1)[0].replace(/_/g, " ")
}
function getCategories(obj) {
if (!obj._INST) {
return [];
}
return obj['_INST'].map(classId => classId.split('#', 1)[0]);
}
function getClasses(obj) {
if (!obj._INST)
return 'node';
const classes = obj['_INST'].map(classId => classId.split('#', 1)[0]);
const classes = getCategories(obj);
return 'node ' + classes.join(' ');
}
function getUrl(obj) {
@ -137,53 +150,110 @@ Promise.all([fetch(req_data), fetch(req_world)])
function buildGraph(data, world) {
// land = topojson.feature(world, world.objects.land)
const borders = topojson.mesh(world, world.objects.countries, (a, b) => a !== b)
console.log(borders);
const countries = topojson.feature(world, world.objects.countries).features;
// console.log(topojson.feature(world, world.objects.countries).features);
const nodes = data.nodes.filter(n => n._INST || n.parent).map(d => Object.create(d));
console.log('all nodes', nodes);
const nodeMap = Object.fromEntries(nodes.map(d => [d['@id'], d]));
const links = data.links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(d => Object.create(d));
// the .source and .target attributes are still ID's. Only after initialisation of the force are they replaced with their representative objects
const nodesBorderingEu = links.filter(l => l.name == 'City' && (nodeMap[l.target].lon < CONFIG.eu.lonMin || nodeMap[l.target].lon > CONFIG.eu.lonMax) && ).map(l => nodeMap[l.source]);
const nodesEastOfEu = links.filter(l => l.name == 'City' && (nodeMap[l.target].lon < CONFIG.eu.lonMin)).map(l => nodeMap[l.source]);
// console.log(links,nodesBorderingEu, links.filter(l => l.name == 'City').map(l => [l.name, l.target]));
console.log(nodesBorderingEu)
const projection = d3.geoHill()
// .center([0, 0])
.translate([width / 2, height * 1.5])
.rotate([-12, 0, 0])
// .rotate([-12, -52, 0])
// .clipAngle(170)
.scale(height * 1.5);
const proj = d3.geoPath().projection(projection);
const graticule = d3.geoGraticule10();
const euCenter = projection([11, 47]);
// console.log(links);
let linkCounts = [];
const linksWithForce = links;//.filter(l => ( nodesBorderingEu.indexOf(nodeMap[l.source]) === -1 && nodesBorderingEu.indexOf(nodeMap[l.target]) === -1 ))
console.log(linksWithForce);
const simulation = d3.forceSimulation(nodes)
.force("link", d3.forceLink(links)
.force("link", d3.forceLink(linksWithForce)
.id(d => d['@id'])
.iterations(2) // increase to make more rigid
.distance((l) => {
// if (getCategories(l.source).indexOf('City') || getCategories(l.target).indexOf('City')) {
// // if(l.source.lat || l.target.lat) {
// return 2;
// }
return 10;
})
.strength((l) => {
if (linkCounts.length < 1) {
// replicate from d3-force/src/link.js so we have access to this in our own strength function
linkCounts = new Array(nodes.length)
for (let i = 0; i < linksWithForce.length; ++i) {
let link = linksWithForce[i];
linkCounts[link.source.index] = (linkCounts[link.source.index] || 0) + 1;
linkCounts[link.target.index] = (linkCounts[link.target.index] || 0) + 1;
}
}
// if(getTitle(l.source) == "NEC"){
// console.log(l, l.target, nodesBorderingEu.indexOf(l.target) !== -1);
// }
// return 0;
if(nodesBorderingEu.indexOf(l.source) !== -1 || nodesBorderingEu.indexOf(l.target) !== -1) {
// console.log('outside', l.target)
return 0.0001;
}
// original:
return 1 / Math.min(linkCounts[l.source.index], linkCounts[l.target.index]);
})
)
.force("charge", d3.forceManyBody()
.strength(-40)
)
// .force("charge", d3.forceManyBody()
// .strength(-10)
// )
// .force("center", d3.forceCenter(width / 2, height / 2))
.force("collision", d3.forceCollide(function (d) {
return getSizeForNode(d) * 1.5; // avoid overlapping nodes
}));
}))
.force("outsideEu", d3.forceRadial(500, euCenter[0], euCenter[1])
.strength(function (node, idx) {
// return 1;
if (nodesBorderingEu.indexOf(node) !== -1) {
// console.log(node, nodesBorderingEu.indexOf(node) !== -1);
return 1;
}
return 0;
})
);
const svg = d3.select("svg")
.attr("viewBox", [0, 0, width, height]);
const container = svg.append("g").attr("id", "container");
const projection = d3.geoOrthographic()
.center([0, 0])
.translate([width / 2, height / 2])
.rotate([-12, -52, 0])
.clipAngle(90)
.scale(height * 1.5);
const proj = d3.geoPath().projection(projection);
const graticule = d3.geoGraticule10();
// container.append("circle").attr("cx", euCenter[0]).attr("cy", euCenter[1]).attr("r", 500).attr("fill","red")
const g_countries = container.append("g").attr("id", "countries");
const g_borders = container.append("g").attr("id", "borders");
container.append("g")
.append('path')
.attr("class", "graticule")
.attr('d', proj(graticule))
.attr("fill", "none")
.attr("stroke-width", "!px")
.attr("stroke", (n) => {
return "lightgray";
});
.append('path')
.attr("class", "graticule")
.attr('d', proj(graticule))
.attr("fill", "none")
.attr("stroke-width", "!px")
.attr("stroke", (n) => {
return "lightgray";
});
;
@ -219,16 +289,46 @@ function buildGraph(data, world) {
.attr('class', 'links')
.selectAll(".link")
.data(links)
.join("g")
;
console.log(links);
link.enter().join("g")
.attr("class", "link")
;
link.exit().remove();
const linkLine = link
.append("line").
attr("marker-end", "url(#arrowHead)");
const linkText = link.append("text").text(function (l) {
const linkText = link.filter((l) => l.name != "City").append("text").text(function (l) {
return l.name;
});
nodes.forEach(function (d) {
d.x = euCenter[0];
d.y = euCenter[1];
if(nodesBorderingEu.indexOf(d) !== -1) {
if(nodesEastOfEu.indexOf(d) !== -1) {
d.x = 466.5836692678423;
d.y = 466.3493609705728;
} else {
d.x = 1406.608195836305;
d.y = 807.9332721328062;
}
}
if (d.lon && d.lat) {
// console.log("fix node", d);
var p = projection([d.lon, d.lat]);
d.x = p[0];
d.y = p[1];
d.fx = p[0];
d.fy = p[1];
}
})
const node = container.append("g")
.attr('class', 'nodes')
.selectAll(".node")
@ -240,14 +340,6 @@ function buildGraph(data, world) {
;
nodes.forEach(function (d) {
if (d.lon && d.lat) {
// console.log("fix node", d);
var p = projection([d.lon, d.lat]);
d.fx = p[0];
d.fy = p[1];
}
})
node
.append('circle')
@ -390,25 +482,26 @@ function buildGraph(data, world) {
container.selectAll(".graticule").attr("d", proj(graticule));
container.selectAll(".borders").attr("d", proj(borders));
simulation.alpha = 0;
simulation.restart();
nodes.forEach(function (d) {
if (d.lon && d.lat) {
var p = projection([d.lon, d.lat]);
d.fx = p[0];
d.fy = p[1];
}
})
// simulation.alpha = 0;
// simulation.restart();
// nodes.forEach(function (d) {
// if (d.lon && d.lat) {
// var p = projection([d.lon, d.lat]);
// d.fx = p[0];
// d.fy = p[1];
// }
// })
}
// beautiful but causes issues with force
// d3.geoZoom()
// .northUp(true)
// .projection(projection)
// .scaleExtent([.3, 6])
// .onMove(refresh)(svg.node());
d3.geoZoom()
.northUp(true)
.projection(projection)
.scaleExtent([.3, 6])
.onMove(refresh)(svg.node());
function resize() {
width = window.innerWidth;
@ -419,7 +512,7 @@ function buildGraph(data, world) {
.attr("height", height);
projection.translate([width / 2, height / 2]);
projection.scale(height * 1.5);
refresh();
// refresh();
}
window.addEventListener('resize', resize);
@ -443,18 +536,25 @@ color = _ => {
const drag = simulation => {
let ignoreDrag = false;
function dragstarted(event) {
if (event.subject.fx) ignoreDrag = true;
if (!event.active) simulation.alphaTarget(0.3).restart();
event.subject.fx = event.subject.x;
event.subject.fy = event.subject.y;
}
function dragged(event) {
if (ignoreDrag) return;
event.subject.fx = event.x;
event.subject.fy = event.y;
}
function dragended(event) {
if (ignoreDrag) {
ignoreDrag = false;
return;
}
if (!event.active) simulation.alphaTarget(0);
event.subject.fx = null;
event.subject.fy = null;
@ -486,177 +586,3 @@ document.getElementById('closeInfo').addEventListener('click', (evt) => {
document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected'));
document.getElementById('nodeInfo').classList.add('hidden');
})
// var graph = {
// nodes: [
// { id: "New York", lat: 40.706109, lon: -74.01194 },
// { id: "London", lat: 51.508070, lon: -0.126432 },
// { id: "Montevideo", lat: -34.901776, lon: -56.163983 },
// { id: "London-NewYork1" },
// { id: "London-NewYork2" },
// { id: "London-NewYork3" },
// { id: "Montevideo-London" }
// ],
// links: [
// { source: "New York", target: "London-NewYork1" },
// { source: "New York", target: "London-NewYork2" },
// { source: "New York", target: "London-NewYork3" },
// { source: "London-NewYork1", target: "London" },
// { source: "London-NewYork2", target: "London" },
// { source: "London-NewYork3", target: "London" },
// { source: "London", target: "Montevideo-London" },
// { source: "Montevideo-London", target: "Montevideo" }
// ]
// }
// const force = d3.forceSimulation()
// .force("link", d3.forceLink()
// .id(function (d) {
// return d.id;
// })
// .distance(10))
// .force("charge", d3.forceManyBody().strength(-200));
// const svg = d3.select("body")
// .append("svg")
// .attr("width", width)
// .attr("height", height);
// const container = svg.append("g").attr("id", "container");
d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(function (world) {
return;
const links = container.append('g')
.attr("id", "links")
.selectAll("line")
.data(graph.links)
.enter()
.append("line")
.attr("stroke-width", 2)
.attr("stroke", "black");
const nodes = container.append('g')
.attr("id", "nodes")
.selectAll("circle")
.data(graph.nodes)
.enter()
.append("circle")
.attr('r', 5)
.call(d3.drag()
.on("start", dragInicia)
.on("drag", dragging)
.on("end", dragTermina));
force.nodes(graph.nodes);
force.force("link").links(graph.links);
graph.nodes.forEach(function (d) {
if (d.lon && d.lat) {
var p = projection([d.lon, d.lat]);
d.fx = p[0];
d.fy = p[1];
}
})
//simulación y actualizacion de la posicion de los nodos en cada "tick"
force.on("tick", function () {
links
.attr('x1', function (d) {
return d.source.x;
})
.attr('y1', function (d) {
return d.source.y;
})
.attr('x2', function (d) {
return d.target.x;
})
.attr('y2', function (d) {
return d.target.y;
})
;
nodes
.attr('cx', function (d) {
return d.x;
})
.attr('cy', function (d) {
return d.y;
})
;
})
// function dragInicia(d) {
// if (!d.lon || !d.lat) {
// if (!d3.event.active) force.alphaTarget(0.3).restart();
// d.fx = d.x;
// d.fy = d.y;
// }
// }
// function dragging(d) {
// if (!d.lon || !d.lat) {
// d.fx = d3.event.x;
// d.fy = d3.event.y;
// }
// }
// function dragTermina(d) {
// if (!d.lon || !d.lat) {
// if (!d3.event.active) force.alphaTarget(0);
// d.fx = null;
// d.fy = null;
// }
// }
function refresh() {
container.selectAll(".countries").attr("d", proj);
container.selectAll(".graticule").attr("d", proj(graticule));
container.selectAll(".borders").attr("d", proj(borders));
force.alpha = 0;
force.restart();
graph.nodes.forEach(function (d) {
if (d.lon && d.lat) {
var p = projection([d.lon, d.lat]);
d.fx = p[0];
d.fy = p[1];
}
})
}
d3.geoZoom()
.northUp(true)
.projection(projection)
.scaleExtent([.3, 6])
.onMove(refresh)(svg.node());
function resize() {
width = window.innerWidth;
height = window.innerHeight;
d3.selectAll('svg')
.attr("width", width)
.attr("height", height);
projection.translate([width / 2, height / 2]);
projection.scale(height * 1.5);
refresh();
}
window.addEventListener('resize', resize);
});

155
www/index.html
View file

@ -1,152 +1,7 @@
<html>
<head>
<style>
:root{
--color1: #f94144;
--color2: #f3722c;
--color3: #f8961e;
/* --color4: #f9844a; */
--color5: #f9c74f;
--color6: #90be6d;
--color7: #43aa8b;
--color8: #4d908e;
--color9: #577590;
--color10: #277da1;
--hover-color: var(--color1);
--selected-color: var(--color1);
--selected-color: var(--color1);
}
body {
margin: 0;
overflow: hidden;
background: #fff;
font-family: sans-serif;
}
svg {
cursor: grab;
font-family: sans-serif;
}
svg.dragging {
cursor: grabbing;
}
svg .links line{
stroke: lightgray;
stroke-width: 1;
}
.links text{
/* display:none; */
font-size:5pt;
text-anchor: middle;
fill: whitesmoke;
}
.node text{
text-anchor: middle;
}
.node circle{
fill: white;
}
.node:hover{
cursor: pointer;
}
.node:hover circle{
stroke: var(--hover-color);
stroke-width: 5px;
}
.node.selected circle{
stroke: var(--selected-color);
stroke-width: 5px;
}
.node.City {
display:none;
}
.node.Person circle {
fill: lightgreen
}
.node.Technology circle {
fill: lightcoral;
}
.node.Deployment circle {
fill: lightblue;
}
.node.Institution circle {
fill: lightgoldenrodyellow
}
.node.Dataset circle {
fill: plum
}
/* .node.Person circle {
fill: var(--color2)
}
.node.Technology circle {
fill: var(--color3);
}
.node.Deployment circle {
fill: var(--color5);
}
.node.Institution circle {
fill: var(--color6)
}
.node.Dataset circle {
fill: var(--color7)
} */
#nodeInfo{
position: fixed;
display:block;
right:20px;
bottom:20px;
background:white;
padding: 10px;
border: solid 1px #ccc;
}
#nodeInfo.hidden{
display:none;
}
#nodeInfo h2{
margin: 0;
padding: 0;
}
#nodeInfo iframe{
width: 50vw;
height: calc(100vh - 40px - 20px - 30px);
}
#closeInfo{
cursor: pointer;
position: absolute;
right: 10px;
top: 10px;
}
#closeInfo:hover{
color: var(--hover-color);
}
a, a:link{
text-decoration: none;
}
a:hover{
text-decoration: underline;
}
</style>
<link rel="stylesheet" href="graph.css">
</head>
<body>
@ -160,8 +15,14 @@
</svg>
<div id="nodeInfo" class='hidden'>
<h2 class='nodeTitle'><a class='nodeHref' target="_blank"></a></h2>
<div id='closeInfo'>&times;</div>
<iframe class='nodeContents'></iframe>
</div>
<script src="https://d3js.org/d3.v6.min.js"></script>
<script src="https://d3js.org/d3.v6.js"></script>
<script src="https://d3js.org/d3-geo-projection.v3.min.js"></script>
<script src="https://d3js.org/topojson.v3.min.js"></script>
<script src="//unpkg.com/d3-geo-zoom"></script>
<script src="graph.js"></script>