Option to generate nodes and edges CSV files

wiki_relations.py

@@ -4,6 +4,7 @@ import requests
 import argparse
 import datetime
 import tqdm
+import csv
 
 
 
@@ -12,11 +13,12 @@ logger = logging.getLogger('wiki')
 default_categories = [
     # 'Person',
     'Institution',
-    'Technology',
+    'Products',
     'Deployments',
     'Dataset',
     'City',
-    # 'Country',# for deployments without city we should configure Geolocation
+    #'Country',# for deployments without city we should configure Geolocation
+    'Technology Type',
     ]
 
 parser = argparse.ArgumentParser(description='Turn wiki into nodes & links, usable by d3-force.')
@@ -28,6 +30,8 @@ parser.add_argument('--output',  default="semantic_data.json",
                     help='Output JSON file')
 parser.add_argument('--credentials',  default="no_credentials.json",
                     help="JSON file containing the Bot's credentials")
+parser.add_argument('--generate-csv',  action='store_true',
+                    help="generate edge.csv & nodes.csv")
 
 args = parser.parse_args()
 
@@ -257,3 +261,17 @@ if __name__ == "__main__":
     with open(args.output, 'w') as fp:
         json.dump(collection, fp)
 
+    if args.generate_csv:    
+        with open('nodes.csv', 'w') as csvfile:
+            all_keys = set().union(*(d.keys() for d in collection['nodes']))
+            # all_keys = ['@id']
+            dict_writer = csv.DictWriter(csvfile, fieldnames=all_keys, extrasaction='ignore', restval='')
+            dict_writer.writeheader()
+            dict_writer.writerows(collection['nodes'])
+            
+        with open('edges.csv', 'w') as csvfile:
+            all_keys = set().union(*(d.keys() for d in collection['links']))
+            dict_writer = csv.DictWriter(csvfile, fieldnames=all_keys, extrasaction='ignore', restval='')
+            dict_writer.writeheader()
+            dict_writer.writerows(collection['links'])
+            

www/graph.css

@@ -391,6 +391,10 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
     border-radius: 5px;
     box-shadow: 2px 2px 5px rgba(0, 0, 0, .5);
 }
+#tooltip.deploymentTooltip{
+    background-color: black;
+    color: white;
+}
 #tooltip.link{
 
 }
@@ -415,6 +419,9 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
     color: black;
     text-align: center;;
 }
+#tooltip.deploymentTooltip .category{
+    color: white;
+}
 #tooltip .category::before{
     content:'· '
 }
@@ -426,6 +433,12 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
     color: gray;
     text-align: center;;
 }
+#tooltip .node_sources::before{
+    content: 'source ';
+}
+#tooltip .node_sources{
+    text-align: center;
+}
 
 #closeInfo {
     cursor: pointer;
@@ -564,19 +577,55 @@ p.subtitle {
 
 #closeSelection{
     pointer-events: none;
-    opacity: 0;
+    position: fixed;
-    position: absolute;
+    bottom: -100px;
-    top: 0;
+    left: 0;
-    right: 0;
     background-color: white;
-    font-size: 200%;
+    font-size: 50px;
-    width: 35px;
+    width: 80px;
+    height: 80px;
     text-align:center;
     z-index: 10;
+    line-height: 80px;
+    transition: bottom .2s;
 }
 
 .selectedNode #closeSelection{
-    pointer-events: all;;
+    pointer-events: all;
-    opacity: 1;
     cursor: pointer;
+    bottom: 0;
+}
+
+#sources{
+    position: fixed;
+    bottom: 0;
+    left: 80px; 
+    height: 47px;
+    padding: 20px 20px 13px 20px;
+    background: black;
+    color: white;
+    transition: bottom .2s;
+    line-height: 40px;
+}
+
+#sources:not(.visible){
+    bottom:-100px;
+}
+
+#sources h3{
+    margin: 5px 0;
+    text-align: left;
+    display: inline-block;
+    font-size: 100%;
+}
+#sources h3::after{
+    content:':';
+}
+
+#sources .node_sources{
+    display: inline;
+}
+
+#sources a{
+    color: lightblue;
 }

www/graph.js

@@ -366,6 +366,7 @@ class NodeMap {
         this.root = d3.select(parent);
         this.resizeEvent = window.addEventListener('resize', this.resize.bind(this));
         this.tooltipEl = document.getElementById('tooltip');
+        this.sourcesEl = document.getElementById('sources');
         this.selectedNode = null;
 
         document.getElementById('closeSelection').addEventListener('click', (ev) => this.deselectNode());
@@ -760,10 +761,17 @@ class NodeMap {
             }
             el = parentEl.querySelector('path');
         }
+
+        const categories = getCategories(node);
+        if (categories.includes('Deployments')){
+            this.tooltipEl.classList.add('deploymentTooltip');
+        } else {
+            this.tooltipEl.classList.remove('deploymentTooltip');
+        }
         
         // TODO: make links optional (otherwise collect links here)
         this.tooltipEl.innerHTML = `
-        <span class='category'>${getCategories(node)[0]}</span>
+        <span class='category'>${categories[0]}</span>
         <h3>${node.fulltext}</h3>
         `;
         if (links.length) {
@@ -772,6 +780,7 @@ class NodeMap {
             <span class='clickForMore'>Click to examine ${links.length} ${rels}</span>
             `;
         }
+
         const rect = el.getBoundingClientRect()
         const rectTT = this.tooltipEl.getBoundingClientRect();
         this.tooltipEl.style.top = (rect.top - rectTT.height) + 'px';
@@ -783,6 +792,7 @@ class NodeMap {
 
     showRelationTooltip(link, evt) {
         const {label, swap} = getLinkLabelConfig(link.name);
+        this.tooltipEl.classList.remove('deploymentTooltip');
         if(swap){
             this.tooltipEl.innerHTML = `
             <span class='relation'>
@@ -845,12 +855,15 @@ class NodeMap {
         this.container.classed('selectedNode', true);
 
         document.body.classList.add('selectedNode');
+
+        this.showSources(node);
         // TODO: show details;
 
         // alert('not yet implemented');
     }
 
     deselectNode() {
+        this.hideSources();
         this.selectedNode = null;
         let nodeEls = document.getElementsByClassName('selected');
         while (nodeEls.length) {
@@ -864,6 +877,37 @@ class NodeMap {
         document.body.classList.remove('selectedNode');
     }
 
+    showSources(node){
+        
+        const categories = getCategories(node);
+        if (!categories.includes('Deployments')){
+            return;
+        }
+        if(!node.printouts['Source'].length) {
+            return;
+        }
+        
+        
+        setTimeout(() => { // give potential visible sources time to hide
+            let sources = []; 
+            for(let source of node.printouts['Source']){
+                const url = document.createElement('a');
+                url.href = source;
+                const hostname = url.hostname.startsWith('www.') ? url.hostname.substring(4) : url.hostname;
+                sources .push(`<a href="${source}" target="_blank">${hostname}</a>`);
+            }
+        
+            
+            const title = node.printouts['Source'].length > 1 ? "Sources" : "Source"
+            this.sourcesEl.innerHTML = `<h3>${title}</h3> ` + sources.join(', ');
+            this.sourcesEl.classList.add('visible');
+        }, 500); 
+    }
+
+    hideSources(){
+        this.sourcesEl.classList.remove('visible');
+    }
+
     update() {
         // console.log(this.graph)
 
@@ -941,64 +985,6 @@ class NodeMap {
 
 
 
-        // const labelPadding = 1;
-
-        // // // the component used to render each label
-        // var textLabel = fc.layoutTextLabel()
-        //     .padding(labelPadding)
-        //     //.value(function(d) { return map_data.properties.iso; });
-        //     //.value(function(d) { return d.properties.iso; });
-        //     .value( (d) => getTitle(d));
-
-        // // a strategy that combines simulated annealing with removal
-        // // of overlapping labels
-        // // */fc.layoutGreedy
-        // const strategy = fc.layoutRemoveOverlaps(fc.layoutGreedy());
-
-        // // create the layout that positions the labels
-        // this.layoutLabels = fc.layoutLabel(strategy)
-        //     .size((d, i, g) => {
-        //         // measure the label and add the required padding
-        //         const textSize = g[i].getElementsByTagName('text')[0].getBBox();
-        //         console.log(textSize);
-        //         // return [30, 20];
-        //         return [textSize.width + labelPadding * 2, textSize.height + labelPadding * 2];
-        //     })
-        //     .position(d => [d.x, d.y])
-        //     .component(textLabel);
-
-        // // render!
-        // // this.node.datum(this.graph.nodes,).call(labels)
-        // this.labels = this.container.append('g').attr('class','labels');
-        // this.labels.datum(this.graph.nodes)
-        // // // this.node
-        //      .call(this.layoutLabels);
-
-
-        // // use simulate annealing to find minimum overlapping text label positions
-        // //https://github.com/d3fc/d3fc-label-layout/blob/master/README.md
-        // var strategy = fc.layoutGreedy();
-        // //var strategy = fc.layoutAnnealing();
-
-        // // create the layout that positions the labels
-        // var labels = fc.layoutLabel(strategy)
-        //     .size(function (_, i, g) {
-        //         // measure the label and add the required padding
-        //         var textSize = d3.select(g[i])
-        //             .select('text')
-        //             .node()
-        //             .getBBox();
-        //         return [textSize.width + labelPadding * 2, textSize.height + labelPadding * 2];
-        //     })
-        //     .position((d) => this.projection([d.lon, d.lat]); })
-        //     .component(textLabel);
-
-        // // render!
-        // this.container.datum(countries)
-        //     .call(labels);
-
-
-
         this.link = this.link
             .data(this.graph.links)
             .join(
@@ -1020,8 +1006,6 @@ class NodeMap {
                             }
 
                             this.showRelationTooltip(link, ev);
-                            
-                            // console.log(l);
                         }).on("mouseout", (ev, link) => {
                             this.hideTooltip();
                             d3.select(ev.target).classed('hover', false);
@@ -1029,10 +1013,6 @@ class NodeMap {
                             while (nodes.length) {
                                 nodes[0].classList.remove('linkHover');
                             }
-                            // l.classed('hover',false);
-                            // l.target.classed('hover',false);
-                            // l.source.classed('hover',false);
-                            // console.log(l,'l');
                         }).on("click", (ev, link) => {
                             ev.stopPropagation();
                             this.selectNode(link.source);
@@ -1105,28 +1085,25 @@ class NodeMap {
                 var tgtSize = _mapGraph.getSizeForNode(l.target);
 
                 // Compute the line endpoint such that the arrow
-                // is touching the edge of the node rectangle perfectly.
+                // it not in the center, but rather slightly out of it
-                l.sourceX = sourceX + Math.sin(angle) * srcSize;
+                // use a small ofset for the angle to compensate roughly for the curve 
-                l.targetX = targetX - Math.sin(angle) * tgtSize;
+                l.sourceX = sourceX + Math.sin(angle+.5) * srcSize;
-                l.sourceY = l.source.y + Math.cos(angle) * srcSize;
+                l.targetX = targetX - Math.sin(angle-.5) * tgtSize;
-                l.targetY = l.target.y - Math.cos(angle) * tgtSize;
+                l.sourceY = l.source.y + Math.cos(angle+.5) * srcSize;
+                l.targetY = l.target.y - Math.cos(angle-.5) * tgtSize;
 
                 // const coor_source = _mapGraph.projection.invert([l.source.x, l.source.y]);
                 // const coor_target = _mapGraph.projection.invert([l.target.x, l.target.y]);
                 // const middleCoor = [coor_source[0] * .5 + coor_target[0] * .5, coor_source[1] * .5 + coor_target[1] * .5];
                 // const middlePoint = _mapGraph.projection(middleCoor);
 
+                // find radius of arc based on distance between points
                 const dr = Math.sqrt(dx * dx + dy * dy);
 
                 // "M" + d.source.x + "," + d.source.y + "A" + dr + "," + dr + " 0 0,1 " + d.target.x + "," + d.target.y
                 let rel = d3.select(this);
-                rel.select("path") //${middlePoint[0]},${middlePoint[1]}
+                rel.select("path")
-                    // .attr('d', `M ${l.sourceX},${l.sourceY} L   ${l.targetX},${l.targetY}`)
                     .attr('d', `M ${l.sourceX},${l.sourceY} A ${dr},${dr} 0 0,1   ${l.targetX},${l.targetY}`)
-                // .attr("x1", l.sourceX)
-                // .attr("y1", l.sourceY)
-                // .attr("x2", l.targetX)
-                // .attr("y2", l.targetY)
 
                 rel.select('text')
                     .attr("transform", function (d) {

www/index.html

@@ -9,6 +9,7 @@
 <body>
 
     <div id='tooltip'></div>
+    <div id='sources'></div>
     <div id='closeSelection'>&times;</div>
     <div id='map'></div>
 
@@ -16,7 +17,6 @@
         <h1>Remote Biometric Identification</h1>
         <p class='subtitle'>A survey of the European Union</p>
 
-
         <aside id="filters">
             <h3 onclick="this.parentNode.classList.toggle('hide');">Filter</h3>
             <div id="filter-items"></div>