Little ordering of graph

wiki_relations.py

@@ -16,7 +16,7 @@ default_categories = [
     'Deployments',
     'Dataset',
     'City',
-    # 'Country',
+    # 'Country',# for deployments without city we should configure Geolocation
     ]
 
 parser = argparse.ArgumentParser(description='Turn wiki into nodes & links, usable by d3-force.')
@@ -142,6 +142,12 @@ def getPropertiesForPage(page, session, collection):
 
     for sub_obj in data['query']['sobj']:
         subSubjectId = sub_obj['subject']
+        if '#0##_QUERY' in subSubjectId:
+            logger.info(f"Skip query subobj {subSubjectId}")
+            continue
+        if '#0##_ERR' in subSubjectId:
+            logger.info(f"Skip error subobj {subSubjectId}")
+            continue
         for rel in sub_obj['data']:
             addToCollection(subSubjectId, rel, collection, subjectId)
 

www/graph.css

@@ -0,0 +1,151 @@
+
+:root{
+    --color1: #f94144;
+    --color2: #f3722c;
+    --color3: #f8961e;
+    /* --color4: #f9844a; */
+    --color5: #f9c74f;
+    --color6: #90be6d;
+    --color7: #43aa8b;
+    --color8: #4d908e;
+    --color9: #577590;
+    --color10: #277da1;
+
+    --hover-color: var(--color1);
+    --selected-color: var(--color1);
+    --selected-color: var(--color1);
+}
+
+body {
+    margin: 0;
+    overflow: hidden;
+    background: linear-gradient(to top, #040308, #AD4A28, #DD723C, #fc7001, #dcb697, #9ba5ae, #3e5879, #020b1a);
+    font-family: sans-serif;
+    min-height: 100vh;
+}
+
+svg {
+    cursor: grab;
+    font-family: sans-serif;
+}
+
+svg.dragging {
+    cursor: grabbing;
+}
+
+svg .links line{
+    stroke: darkgray;
+    stroke-width: 1;
+}
+
+.links text{
+    display:none;
+    font-size:5pt;
+    text-anchor: middle;
+    fill: whitesmoke;
+}
+
+.node text{
+    text-anchor: middle;
+}
+
+.node circle{
+    fill: white;
+}
+
+.node:hover{
+    cursor: pointer;
+}
+
+.node:hover circle{
+    stroke: var(--hover-color);
+    stroke-width: 5px;
+}
+.node.selected circle{
+    stroke: var(--selected-color);
+    stroke-width: 5px;
+}
+
+.node.City circle{
+    display:none;
+}
+.node.City{
+    fill:white;
+    stroke: black;
+    stroke-width: .5px;
+    font-size: 20px;
+}
+.node.Person circle {
+    fill: lightgreen
+}
+.node.Technology circle {
+    fill: lightcoral;
+}
+.node.Deployment circle {
+    fill: lightblue;
+}
+.node.Institution circle {
+    fill: lightgoldenrodyellow
+}
+.node.Dataset circle {
+    fill: plum
+}
+
+/*  .node.Person circle {
+    fill: var(--color2)
+}
+.node.Technology circle {
+    fill: var(--color3);
+}
+.node.Deployment circle {
+    fill: var(--color5);
+}
+.node.Institution circle {
+    fill: var(--color6)
+}
+.node.Dataset circle {
+    fill: var(--color7)
+} */
+
+
+#nodeInfo{
+    position: fixed;
+    display:block;
+    right:20px;
+    bottom:20px;
+    background:white;
+    padding: 10px;
+    border: solid 1px #ccc;
+}
+
+#nodeInfo.hidden{
+    display:none;
+}
+
+#nodeInfo h2{
+    margin: 0;
+    padding: 0;
+}
+
+#nodeInfo iframe{
+    width: 50vw;
+    height: calc(100vh - 40px - 20px - 30px);
+}
+
+#closeInfo{
+    cursor: pointer;
+    position: absolute;
+    right: 10px;
+    top: 10px;
+}
+
+#closeInfo:hover{
+    color: var(--hover-color);
+}
+
+a, a:link{
+    text-decoration: none;
+}
+a:hover{
+    text-decoration: underline;
+}

www/graph.js

@@ -1,13 +1,17 @@
 
 const CONFIG = {
-    'nodeSize': 16,
+    'nodeSize': 8,
     'baseUrl': 'https://www.securityvision.io/wiki/index.php/',
     'dataUrl': '../semantic_data.json',
-    'preSimulate': true, // run simulation before starting, so we don't start with lines jumping around
+    'preSimulate': false, // run simulation before starting, so we don't start with lines jumping around
     'labels': {
         'rotate': true,
     },
-    'countries': ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia	', 'Portugal', 'Finland	', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
+    'countries': ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia	', 'Portugal', 'Finland	', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland'],
+    'eu': {
+        'lonMin': -10,
+        'lonMax': 35
+    }
 };
 
 
@@ -20,6 +24,9 @@ function getSizeForNode(node) {
     if (node.parent) {
         return 2;
     }
+    // if(getCategories(node).indexOf('City') !== -1) {
+    //     return 2;
+    // }
     return CONFIG.nodeSize;
 }
 
@@ -91,10 +98,16 @@ function getTitle(obj) {
     }
     return obj['@id'].split('#', 1)[0].replace(/_/g, " ")
 }
+function getCategories(obj) {
+    if (!obj._INST) {
+        return [];
+    }
+    return obj['_INST'].map(classId => classId.split('#', 1)[0]);
+}
 function getClasses(obj) {
     if (!obj._INST)
         return 'node';
-    const classes = obj['_INST'].map(classId => classId.split('#', 1)[0]);
+    const classes = getCategories(obj);
     return 'node ' + classes.join(' ');
 }
 function getUrl(obj) {
@@ -137,41 +150,98 @@ Promise.all([fetch(req_data), fetch(req_world)])
 function buildGraph(data, world) {
     // land = topojson.feature(world, world.objects.land)
     const borders = topojson.mesh(world, world.objects.countries, (a, b) => a !== b)
-    console.log(borders);
     const countries = topojson.feature(world, world.objects.countries).features;
-    // console.log(topojson.feature(world, world.objects.countries).features);
 
     const nodes = data.nodes.filter(n => n._INST || n.parent).map(d => Object.create(d));
+    console.log('all nodes', nodes);
     const nodeMap = Object.fromEntries(nodes.map(d => [d['@id'], d]));
     const links = data.links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(d => Object.create(d));
 
+    // the .source and .target attributes are still ID's. Only after initialisation of the force are they replaced with their representative objects
+    const nodesBorderingEu = links.filter(l => l.name == 'City' && (nodeMap[l.target].lon < CONFIG.eu.lonMin || nodeMap[l.target].lon > CONFIG.eu.lonMax) &&  ).map(l => nodeMap[l.source]);
+    const nodesEastOfEu = links.filter(l => l.name == 'City' && (nodeMap[l.target].lon < CONFIG.eu.lonMin)).map(l => nodeMap[l.source]);
+    // console.log(links,nodesBorderingEu, links.filter(l => l.name == 'City').map(l => [l.name, l.target]));
+    console.log(nodesBorderingEu)
+
+    const projection = d3.geoHill()
+        // .center([0, 0])
+        .translate([width / 2, height * 1.5])
+        .rotate([-12, 0, 0])
+        // .rotate([-12, -52, 0])
+        // .clipAngle(170)
+        .scale(height * 1.5);
+
+    const proj = d3.geoPath().projection(projection);
+    const graticule = d3.geoGraticule10();
+
+
+    const euCenter = projection([11, 47]);
+
+    
+
+    // console.log(links);
+
+    let linkCounts = [];
+    const linksWithForce = links;//.filter(l => ( nodesBorderingEu.indexOf(nodeMap[l.source]) === -1 &&  nodesBorderingEu.indexOf(nodeMap[l.target]) === -1 ))
+    console.log(linksWithForce);
+
     const simulation = d3.forceSimulation(nodes)
-        .force("link", d3.forceLink(links)
+        .force("link", d3.forceLink(linksWithForce)
             .id(d => d['@id'])
             .iterations(2) // increase to make more rigid
+            .distance((l) => {
+                // if (getCategories(l.source).indexOf('City') || getCategories(l.target).indexOf('City')) {
+                //     // if(l.source.lat || l.target.lat) {
+                //     return 2;
+                // }
+                return 10;
+            })
+            .strength((l) => {
+                if (linkCounts.length < 1) {
+                    // replicate from d3-force/src/link.js so we have access to this in our own strength function
+                    linkCounts = new Array(nodes.length)
+                    for (let i = 0; i < linksWithForce.length; ++i) {
+                        let link = linksWithForce[i];
+                        linkCounts[link.source.index] = (linkCounts[link.source.index] || 0) + 1;
+                        linkCounts[link.target.index] = (linkCounts[link.target.index] || 0) + 1;
+                    }
+                }
+                // if(getTitle(l.source) == "NEC"){
+                //     console.log(l, l.target, nodesBorderingEu.indexOf(l.target) !== -1);
+                // }
+                // return 0;
+
+                if(nodesBorderingEu.indexOf(l.source) !== -1 || nodesBorderingEu.indexOf(l.target) !== -1) {
+                    // console.log('outside', l.target)
+                    return 0.0001;
+                }
+                // original:
+                return 1 / Math.min(linkCounts[l.source.index], linkCounts[l.target.index]);
+            })
         )
-        .force("charge", d3.forceManyBody()
+        // .force("charge", d3.forceManyBody()
-            .strength(-40)
+        //     .strength(-10)
-        )
+        // )
         // .force("center", d3.forceCenter(width / 2, height / 2))
         .force("collision", d3.forceCollide(function (d) {
             return getSizeForNode(d) * 1.5; // avoid overlapping nodes
-        }));
+        }))
+        .force("outsideEu", d3.forceRadial(500, euCenter[0], euCenter[1])
+            .strength(function (node, idx) {
+                // return 1;
+                if (nodesBorderingEu.indexOf(node) !== -1) {
+                    // console.log(node, nodesBorderingEu.indexOf(node) !== -1);
+                    return 1;
+                }
+                return 0;
+            })
+        );
 
     const svg = d3.select("svg")
         .attr("viewBox", [0, 0, width, height]);
     const container = svg.append("g").attr("id", "container");
 
-
+    // container.append("circle").attr("cx", euCenter[0]).attr("cy", euCenter[1]).attr("r", 500).attr("fill","red")
-    const projection = d3.geoOrthographic()
-    .center([0, 0])
-    .translate([width / 2, height / 2])
-    .rotate([-12, -52, 0])
-    .clipAngle(90)
-    .scale(height * 1.5);
-
-    const proj = d3.geoPath().projection(projection);
-    const graticule = d3.geoGraticule10();
 
     const g_countries = container.append("g").attr("id", "countries");
     const g_borders = container.append("g").attr("id", "borders");
@@ -219,16 +289,46 @@ function buildGraph(data, world) {
         .attr('class', 'links')
         .selectAll(".link")
         .data(links)
-        .join("g")
+        ;
+
+        console.log(links);
+        
+        link.enter().join("g")
         .attr("class", "link")
+        ;
+        link.exit().remove();
 
     const linkLine = link
         .append("line").
         attr("marker-end", "url(#arrowHead)");
-    const linkText = link.append("text").text(function (l) {
+    const linkText = link.filter((l) => l.name != "City").append("text").text(function (l) {
         return l.name;
     });
 
+    nodes.forEach(function (d) {
+        d.x = euCenter[0];
+        d.y = euCenter[1];
+
+        if(nodesBorderingEu.indexOf(d) !== -1) {
+            if(nodesEastOfEu.indexOf(d) !== -1) {
+                d.x = 466.5836692678423;
+                d.y = 466.3493609705728;
+            } else {
+                d.x = 1406.608195836305;
+                d.y = 807.9332721328062;
+            }
+            
+        }
+
+        if (d.lon && d.lat) {
+            // console.log("fix node", d);
+            var p = projection([d.lon, d.lat]);
+            d.x = p[0];
+            d.y = p[1];
+            d.fx = p[0];
+            d.fy = p[1];
+        }
+    })
     const node = container.append("g")
         .attr('class', 'nodes')
         .selectAll(".node")
@@ -240,14 +340,6 @@ function buildGraph(data, world) {
         ;
 
 
-    nodes.forEach(function (d) {
-        if (d.lon && d.lat) {
-            // console.log("fix node", d);
-            var p = projection([d.lon, d.lat]);
-            d.fx = p[0];
-            d.fy = p[1];
-        }
-    })
 
     node
         .append('circle')
@@ -390,25 +482,26 @@ function buildGraph(data, world) {
         container.selectAll(".graticule").attr("d", proj(graticule));
         container.selectAll(".borders").attr("d", proj(borders));
 
-        simulation.alpha = 0;
+        // simulation.alpha = 0;
-        simulation.restart();
+        // simulation.restart();
-        nodes.forEach(function (d) {
+        // nodes.forEach(function (d) {
-            if (d.lon && d.lat) {
+        //     if (d.lon && d.lat) {
-                var p = projection([d.lon, d.lat]);
+        //         var p = projection([d.lon, d.lat]);
-                d.fx = p[0];
+        //         d.fx = p[0];
-                d.fy = p[1];
+        //         d.fy = p[1];
-            }
+        //     }
-        })
+        // })
 
     }
 
 
+    // beautiful but causes issues with force
+    // d3.geoZoom()
+    //     .northUp(true)
+    //     .projection(projection)
+    //     .scaleExtent([.3, 6])
+    //     .onMove(refresh)(svg.node());
 
-    d3.geoZoom()
-        .northUp(true)
-        .projection(projection)
-        .scaleExtent([.3, 6])
-        .onMove(refresh)(svg.node());
 
     function resize() {
         width = window.innerWidth;
@@ -419,7 +512,7 @@ function buildGraph(data, world) {
             .attr("height", height);
         projection.translate([width / 2, height / 2]);
         projection.scale(height * 1.5);
-        refresh();
+        // refresh();
     }
     window.addEventListener('resize', resize);
 
@@ -443,18 +536,25 @@ color = _ => {
 
 const drag = simulation => {
 
+    let ignoreDrag = false;
     function dragstarted(event) {
+        if (event.subject.fx) ignoreDrag = true;
         if (!event.active) simulation.alphaTarget(0.3).restart();
         event.subject.fx = event.subject.x;
         event.subject.fy = event.subject.y;
     }
 
     function dragged(event) {
+        if (ignoreDrag) return;
         event.subject.fx = event.x;
         event.subject.fy = event.y;
     }
 
     function dragended(event) {
+        if (ignoreDrag) {
+            ignoreDrag = false;
+            return;
+        }
         if (!event.active) simulation.alphaTarget(0);
         event.subject.fx = null;
         event.subject.fy = null;
@@ -486,177 +586,3 @@ document.getElementById('closeInfo').addEventListener('click', (evt) => {
     document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected'));
     document.getElementById('nodeInfo').classList.add('hidden');
 })
-
-
-// var graph = {
-//     nodes: [
-//         { id: "New York", lat: 40.706109, lon: -74.01194 },
-//         { id: "London", lat: 51.508070, lon: -0.126432 },
-//         { id: "Montevideo", lat: -34.901776, lon: -56.163983 },
-//         { id: "London-NewYork1" },
-//         { id: "London-NewYork2" },
-//         { id: "London-NewYork3" },
-//         { id: "Montevideo-London" }
-//     ],
-//     links: [
-//         { source: "New York", target: "London-NewYork1" },
-//         { source: "New York", target: "London-NewYork2" },
-//         { source: "New York", target: "London-NewYork3" },
-//         { source: "London-NewYork1", target: "London" },
-//         { source: "London-NewYork2", target: "London" },
-//         { source: "London-NewYork3", target: "London" },
-//         { source: "London", target: "Montevideo-London" },
-//         { source: "Montevideo-London", target: "Montevideo" }
-//     ]
-// }
-
-
-// const force = d3.forceSimulation()
-//     .force("link", d3.forceLink()
-//         .id(function (d) {
-//             return d.id;
-//         })
-//         .distance(10))
-//     .force("charge", d3.forceManyBody().strength(-200));
-
-
-// const svg = d3.select("body")
-//     .append("svg")
-//     .attr("width", width)
-//     .attr("height", height);
-
-// const container = svg.append("g").attr("id", "container");
-
-
-d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(function (world) {
-
-return;
-    
-    const links = container.append('g')
-        .attr("id", "links")
-        .selectAll("line")
-        .data(graph.links)
-        .enter()
-        .append("line")
-        .attr("stroke-width", 2)
-        .attr("stroke", "black");
-
-
-    const nodes = container.append('g')
-        .attr("id", "nodes")
-        .selectAll("circle")
-        .data(graph.nodes)
-        .enter()
-        .append("circle")
-        .attr('r', 5)
-        .call(d3.drag()
-            .on("start", dragInicia)
-            .on("drag", dragging)
-            .on("end", dragTermina));
-
-
-
-    force.nodes(graph.nodes);
-    force.force("link").links(graph.links);
-
-    graph.nodes.forEach(function (d) {
-        if (d.lon && d.lat) {
-            var p = projection([d.lon, d.lat]);
-            d.fx = p[0];
-            d.fy = p[1];
-        }
-    })
-
-    //simulación y actualizacion de la posicion de los nodos en cada "tick"
-    force.on("tick", function () {
-        links
-            .attr('x1', function (d) {
-                return d.source.x;
-            })
-            .attr('y1', function (d) {
-                return d.source.y;
-            })
-            .attr('x2', function (d) {
-                return d.target.x;
-            })
-            .attr('y2', function (d) {
-                return d.target.y;
-            })
-            ;
-
-        nodes
-            .attr('cx', function (d) {
-                return d.x;
-            })
-            .attr('cy', function (d) {
-                return d.y;
-            })
-            ;
-    })
-
-
-    // function dragInicia(d) {
-    //     if (!d.lon || !d.lat) {
-    //         if (!d3.event.active) force.alphaTarget(0.3).restart();
-    //         d.fx = d.x;
-    //         d.fy = d.y;
-    //     }
-    // }
-
-    // function dragging(d) {
-    //     if (!d.lon || !d.lat) {
-    //         d.fx = d3.event.x;
-    //         d.fy = d3.event.y;
-    //     }
-    // }
-
-    // function dragTermina(d) {
-    //     if (!d.lon || !d.lat) {
-    //         if (!d3.event.active) force.alphaTarget(0);
-    //         d.fx = null;
-    //         d.fy = null;
-    //     }
-    // }
-
-
-
-    function refresh() {
-        container.selectAll(".countries").attr("d", proj);
-        container.selectAll(".graticule").attr("d", proj(graticule));
-        container.selectAll(".borders").attr("d", proj(borders));
-
-        force.alpha = 0;
-        force.restart();
-        graph.nodes.forEach(function (d) {
-            if (d.lon && d.lat) {
-                var p = projection([d.lon, d.lat]);
-                d.fx = p[0];
-                d.fy = p[1];
-            }
-        })
-
-    }
-
-
-
-    d3.geoZoom()
-        .northUp(true)
-        .projection(projection)
-        .scaleExtent([.3, 6])
-        .onMove(refresh)(svg.node());
-
-    function resize() {
-        width = window.innerWidth;
-        height = window.innerHeight;
-
-        d3.selectAll('svg')
-            .attr("width", width)
-            .attr("height", height);
-        projection.translate([width / 2, height / 2]);
-        projection.scale(height * 1.5);
-        refresh();
-    }
-    window.addEventListener('resize', resize);
-});
-
-

www/index.html

@@ -1,152 +1,7 @@
 <html>
 
 <head>
-    <style>
+    <link rel="stylesheet" href="graph.css">
-        
-        :root{
-            --color1: #f94144;
-            --color2: #f3722c;
-            --color3: #f8961e;
-            /* --color4: #f9844a; */
-            --color5: #f9c74f;
-            --color6: #90be6d;
-            --color7: #43aa8b;
-            --color8: #4d908e;
-            --color9: #577590;
-            --color10: #277da1;
-
-            --hover-color: var(--color1);
-            --selected-color: var(--color1);
-            --selected-color: var(--color1);
-        }
-
-        body {
-            margin: 0;
-            overflow: hidden;
-            background: #fff;
-            font-family: sans-serif;
-        }
-
-        svg {
-            cursor: grab;
-            font-family: sans-serif;
-        }
-
-        svg.dragging {
-            cursor: grabbing;
-        }
-
-        svg .links line{
-            stroke: lightgray;
-            stroke-width: 1;
-        }
-
-        .links text{
-            /* display:none; */
-            font-size:5pt;
-            text-anchor: middle;
-            fill: whitesmoke;
-        }
-
-        .node text{
-            text-anchor: middle;
-        }
-
-        .node circle{
-            fill: white;
-        }
-
-        .node:hover{
-            cursor: pointer;
-        }
-
-        .node:hover circle{
-            stroke: var(--hover-color);
-            stroke-width: 5px;
-        }
-        .node.selected circle{
-            stroke: var(--selected-color);
-            stroke-width: 5px;
-        }
-
-        .node.City {
-            display:none;
-        }
-        .node.Person circle {
-            fill: lightgreen
-        }
-        .node.Technology circle {
-            fill: lightcoral;
-        }
-        .node.Deployment circle {
-            fill: lightblue;
-        }
-        .node.Institution circle {
-            fill: lightgoldenrodyellow
-        }
-        .node.Dataset circle {
-            fill: plum
-        }
-
-        /*  .node.Person circle {
-            fill: var(--color2)
-        }
-        .node.Technology circle {
-            fill: var(--color3);
-        }
-        .node.Deployment circle {
-            fill: var(--color5);
-        }
-        .node.Institution circle {
-            fill: var(--color6)
-        }
-        .node.Dataset circle {
-            fill: var(--color7)
-        } */
-
-
-        #nodeInfo{
-            position: fixed;
-            display:block;
-            right:20px;
-            bottom:20px;
-            background:white;
-            padding: 10px;
-            border: solid 1px #ccc;
-        }
-
-        #nodeInfo.hidden{
-            display:none;
-        }
-
-        #nodeInfo h2{
-            margin: 0;
-            padding: 0;
-        }
-
-        #nodeInfo iframe{
-            width: 50vw;
-            height: calc(100vh - 40px - 20px - 30px);
-        }
-
-        #closeInfo{
-            cursor: pointer;
-            position: absolute;
-            right: 10px;
-            top: 10px;
-        }
-
-        #closeInfo:hover{
-            color: var(--hover-color);
-        }
-
-        a, a:link{
-            text-decoration: none;
-        }
-        a:hover{
-            text-decoration: underline;
-        }
-    </style>
 </head>
 
 <body>
@@ -160,8 +15,14 @@
 
 </svg>
 
+<div id="nodeInfo" class='hidden'>
+    <h2 class='nodeTitle'><a class='nodeHref' target="_blank"></a></h2>
+    <div id='closeInfo'>&times;</div>
+    <iframe class='nodeContents'></iframe>
+</div>
 
-<script src="https://d3js.org/d3.v6.min.js"></script>
+<script src="https://d3js.org/d3.v6.js"></script>
+<script src="https://d3js.org/d3-geo-projection.v3.min.js"></script>
 <script src="https://d3js.org/topojson.v3.min.js"></script>
 <script src="//unpkg.com/d3-geo-zoom"></script>
 <script src="graph.js"></script>