security_vision/semantic_graph
1
0
Fork 1
Code Issues Pull requests Projects Releases 1 Wiki Activity

Compare commits

base: security_vision:main
Branches Tags
security_vision:main
security_vision:filtering
security_vision:v1
..
compare: security_vision:v1
Branches Tags
security_vision:main
security_vision:filtering
security_vision:v1

No commits in common. "main" and "v1" have entirely different histories.
main ... v1

4 changed files with 87 additions and 131 deletions
Show stats Expand all files Collapse all files

22
wiki_relations.py
View file

@ -4,7 +4,6 @@ import requests
import argparse
import datetime
import tqdm
import csv
@ -13,12 +12,11 @@ logger = logging.getLogger('wiki')
default_categories = [
# 'Person',
'Institution',
'Products',
'Technology',
'Deployments',
'Dataset',
'City',
#'Country',# for deployments without city we should configure Geolocation
'Technology Type',
# 'Country',# for deployments without city we should configure Geolocation
]
parser = argparse.ArgumentParser(description='Turn wiki into nodes & links, usable by d3-force.')
@ -30,8 +28,6 @@ parser.add_argument('--output', default="semantic_data.json",
help='Output JSON file')
parser.add_argument('--credentials', default="no_credentials.json",
help="JSON file containing the Bot's credentials")
parser.add_argument('--generate-csv', action='store_true',
help="generate edge.csv & nodes.csv")
args = parser.parse_args()
@ -261,17 +257,3 @@ if __name__ == "__main__":
with open(args.output, 'w') as fp:
json.dump(collection, fp)
if args.generate_csv:
with open('nodes.csv', 'w') as csvfile:
all_keys = set().union(*(d.keys() for d in collection['nodes']))
# all_keys = ['@id']
dict_writer = csv.DictWriter(csvfile, fieldnames=all_keys, extrasaction='ignore', restval='')
dict_writer.writeheader()
dict_writer.writerows(collection['nodes'])
with open('edges.csv', 'w') as csvfile:
all_keys = set().union(*(d.keys() for d in collection['links']))
dict_writer = csv.DictWriter(csvfile, fieldnames=all_keys, extrasaction='ignore', restval='')
dict_writer.writeheader()
dict_writer.writerows(collection['links'])

65
www/graph.css
View file

@ -391,10 +391,6 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
border-radius: 5px;
box-shadow: 2px 2px 5px rgba(0, 0, 0, .5);
}
#tooltip.deploymentTooltip{
background-color: black;
color: white;
}
#tooltip.link{
}
@ -419,9 +415,6 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
color: black;
text-align: center;;
}
#tooltip.deploymentTooltip .category{
color: white;
}
#tooltip .category::before{
content:'· '
}
@ -433,12 +426,6 @@ svg.zoomed.zoomed2 .node text.nodeTitle {
color: gray;
text-align: center;;
}
#tooltip .node_sources::before{
content: 'source ';
}
#tooltip .node_sources{
text-align: center;
}
#closeInfo {
cursor: pointer;
@ -577,55 +564,19 @@ p.subtitle {
#closeSelection{
pointer-events: none;
position: fixed;
bottom: -100px;
left: 0;
opacity: 0;
position: absolute;
top: 0;
right: 0;
background-color: white;
font-size: 50px;
width: 80px;
height: 80px;
font-size: 200%;
width: 35px;
text-align:center;
z-index: 10;
line-height: 80px;
transition: bottom .2s;
}
.selectedNode #closeSelection{
pointer-events: all;
pointer-events: all;;
opacity: 1;
cursor: pointer;
bottom: 0;
}
#sources{
position: fixed;
bottom: 0;
left: 80px;
height: 47px;
padding: 20px 20px 13px 20px;
background: black;
color: white;
transition: bottom .2s;
line-height: 40px;
}
#sources:not(.visible){
bottom:-100px;
}
#sources h3{
margin: 5px 0;
text-align: left;
display: inline-block;
font-size: 100%;
}
#sources h3::after{
content:':';
}
#sources .node_sources{
display: inline;
}
#sources a{
color: lightblue;
}

129
www/graph.js
View file

@ -366,7 +366,6 @@ class NodeMap {
this.root = d3.select(parent);
this.resizeEvent = window.addEventListener('resize', this.resize.bind(this));
this.tooltipEl = document.getElementById('tooltip');
this.sourcesEl = document.getElementById('sources');
this.selectedNode = null;
document.getElementById('closeSelection').addEventListener('click', (ev) => this.deselectNode());
@ -761,17 +760,10 @@ class NodeMap {
}
el = parentEl.querySelector('path');
}
const categories = getCategories(node);
if (categories.includes('Deployments')){
this.tooltipEl.classList.add('deploymentTooltip');
} else {
this.tooltipEl.classList.remove('deploymentTooltip');
}
// TODO: make links optional (otherwise collect links here)
this.tooltipEl.innerHTML = `
<span class='category'>${categories[0]}</span>
<span class='category'>${getCategories(node)[0]}</span>
<h3>${node.fulltext}</h3>
`;
if (links.length) {
@ -780,7 +772,6 @@ class NodeMap {
<span class='clickForMore'>Click to examine ${links.length} ${rels}</span>
`;
}
const rect = el.getBoundingClientRect()
const rectTT = this.tooltipEl.getBoundingClientRect();
this.tooltipEl.style.top = (rect.top - rectTT.height) + 'px';
@ -792,7 +783,6 @@ class NodeMap {
showRelationTooltip(link, evt) {
const {label, swap} = getLinkLabelConfig(link.name);
this.tooltipEl.classList.remove('deploymentTooltip');
if(swap){
this.tooltipEl.innerHTML = `
<span class='relation'>
@ -855,15 +845,12 @@ class NodeMap {
this.container.classed('selectedNode', true);
document.body.classList.add('selectedNode');
this.showSources(node);
// TODO: show details;
// alert('not yet implemented');
}
deselectNode() {
this.hideSources();
this.selectedNode = null;
let nodeEls = document.getElementsByClassName('selected');
while (nodeEls.length) {
@ -877,37 +864,6 @@ class NodeMap {
document.body.classList.remove('selectedNode');
}
showSources(node){
const categories = getCategories(node);
if (!categories.includes('Deployments')){
return;
}
if(!node.printouts['Source'].length) {
return;
}
setTimeout(() => { // give potential visible sources time to hide
let sources = [];
for(let source of node.printouts['Source']){
const url = document.createElement('a');
url.href = source;
const hostname = url.hostname.startsWith('www.') ? url.hostname.substring(4) : url.hostname;
sources .push(`<a href="${source}" target="_blank">${hostname}</a>`);
}
const title = node.printouts['Source'].length > 1 ? "Sources" : "Source"
this.sourcesEl.innerHTML = `<h3>${title}</h3> ` + sources.join(', ');
this.sourcesEl.classList.add('visible');
}, 500);
}
hideSources(){
this.sourcesEl.classList.remove('visible');
}
update() {
// console.log(this.graph)
@ -985,6 +941,64 @@ class NodeMap {
// const labelPadding = 1;
// // // the component used to render each label
// var textLabel = fc.layoutTextLabel()
// .padding(labelPadding)
// //.value(function(d) { return map_data.properties.iso; });
// //.value(function(d) { return d.properties.iso; });
// .value( (d) => getTitle(d));
// // a strategy that combines simulated annealing with removal
// // of overlapping labels
// // */fc.layoutGreedy
// const strategy = fc.layoutRemoveOverlaps(fc.layoutGreedy());
// // create the layout that positions the labels
// this.layoutLabels = fc.layoutLabel(strategy)
// .size((d, i, g) => {
// // measure the label and add the required padding
// const textSize = g[i].getElementsByTagName('text')[0].getBBox();
// console.log(textSize);
// // return [30, 20];
// return [textSize.width + labelPadding * 2, textSize.height + labelPadding * 2];
// })
// .position(d => [d.x, d.y])
// .component(textLabel);
// // render!
// // this.node.datum(this.graph.nodes,).call(labels)
// this.labels = this.container.append('g').attr('class','labels');
// this.labels.datum(this.graph.nodes)
// // // this.node
// .call(this.layoutLabels);
// // use simulate annealing to find minimum overlapping text label positions
// //https://github.com/d3fc/d3fc-label-layout/blob/master/README.md
// var strategy = fc.layoutGreedy();
// //var strategy = fc.layoutAnnealing();
// // create the layout that positions the labels
// var labels = fc.layoutLabel(strategy)
// .size(function (_, i, g) {
// // measure the label and add the required padding
// var textSize = d3.select(g[i])
// .select('text')
// .node()
// .getBBox();
// return [textSize.width + labelPadding * 2, textSize.height + labelPadding * 2];
// })
// .position((d) => this.projection([d.lon, d.lat]); })
// .component(textLabel);
// // render!
// this.container.datum(countries)
// .call(labels);
this.link = this.link
.data(this.graph.links)
.join(
@ -1006,6 +1020,8 @@ class NodeMap {
}
this.showRelationTooltip(link, ev);
// console.log(l);
}).on("mouseout", (ev, link) => {
this.hideTooltip();
d3.select(ev.target).classed('hover', false);
@ -1013,6 +1029,10 @@ class NodeMap {
while (nodes.length) {
nodes[0].classList.remove('linkHover');
}
// l.classed('hover',false);
// l.target.classed('hover',false);
// l.source.classed('hover',false);
// console.log(l,'l');
}).on("click", (ev, link) => {
ev.stopPropagation();
this.selectNode(link.source);
@ -1085,25 +1105,28 @@ class NodeMap {
var tgtSize = _mapGraph.getSizeForNode(l.target);
// Compute the line endpoint such that the arrow
// it not in the center, but rather slightly out of it
// use a small ofset for the angle to compensate roughly for the curve
l.sourceX = sourceX + Math.sin(angle+.5) * srcSize;
l.targetX = targetX - Math.sin(angle-.5) * tgtSize;
l.sourceY = l.source.y + Math.cos(angle+.5) * srcSize;
l.targetY = l.target.y - Math.cos(angle-.5) * tgtSize;
// is touching the edge of the node rectangle perfectly.
l.sourceX = sourceX + Math.sin(angle) * srcSize;
l.targetX = targetX - Math.sin(angle) * tgtSize;
l.sourceY = l.source.y + Math.cos(angle) * srcSize;
l.targetY = l.target.y - Math.cos(angle) * tgtSize;
// const coor_source = _mapGraph.projection.invert([l.source.x, l.source.y]);
// const coor_target = _mapGraph.projection.invert([l.target.x, l.target.y]);
// const middleCoor = [coor_source[0] * .5 + coor_target[0] * .5, coor_source[1] * .5 + coor_target[1] * .5];
// const middlePoint = _mapGraph.projection(middleCoor);
// find radius of arc based on distance between points
const dr = Math.sqrt(dx * dx + dy * dy);
// "M" + d.source.x + "," + d.source.y + "A" + dr + "," + dr + " 0 0,1 " + d.target.x + "," + d.target.y
let rel = d3.select(this);
rel.select("path")
rel.select("path") //${middlePoint[0]},${middlePoint[1]}
// .attr('d', `M ${l.sourceX},${l.sourceY} L ${l.targetX},${l.targetY}`)
.attr('d', `M ${l.sourceX},${l.sourceY} A ${dr},${dr} 0 0,1 ${l.targetX},${l.targetY}`)
// .attr("x1", l.sourceX)
// .attr("y1", l.sourceY)
// .attr("x2", l.targetX)
// .attr("y2", l.targetY)
rel.select('text')
.attr("transform", function (d) {

2
www/index.html
View file

@ -9,7 +9,6 @@
<body>
<div id='tooltip'></div>
<div id='sources'></div>
<div id='closeSelection'>&times;</div>
<div id='map'></div>
@ -17,6 +16,7 @@
<h1>Remote Biometric Identification</h1>
<p class='subtitle'>A survey of the European Union</p>
<aside id="filters">
<h3 onclick="this.parentNode.classList.toggle('hide');">Filter</h3>
<div id="filter-items"></div>