diff --git a/wiki_relations.py b/wiki_relations.py index 8557676..5a8f82a 100644 --- a/wiki_relations.py +++ b/wiki_relations.py @@ -10,12 +10,12 @@ import tqdm logger = logging.getLogger('wiki') default_categories = [ - 'Person', + # 'Person', 'Institution', 'Technology', 'Deployments', 'Dataset', - # 'City', + 'City', # 'Country', ] @@ -195,13 +195,16 @@ def addDataitemToCollection(subjectId, prop, data, collection): collection['nodes'][subjectId][prop].append(json.dumps(data)) else: # TODO: map as properties on link! - if data['item'] not in collection['nodes']: - collection['nodes'][data['item']] = getObjForSubject(data['item']) - collection['links'].append({ - 'source': subjectId, - 'target': data['item'], - 'name': prop - }) + if '#0##_QUERY' in data['item']: + logger.warning(f"Skip query for {subjectId}: {data}") + else: + if data['item'] not in collection['nodes']: + collection['nodes'][data['item']] = getObjForSubject(data['item']) + collection['links'].append({ + 'source': subjectId, + 'target': data['item'], + 'name': prop + }) elif data['type'] == 7: # Geolocation lat, lon = data['item'].split(',') diff --git a/www/graph.js b/www/graph.js index 69ddea7..7673612 100644 --- a/www/graph.js +++ b/www/graph.js @@ -1,82 +1,194 @@ + +const CONFIG = { + 'nodeSize': 16, + 'baseUrl': 'https://www.securityvision.io/wiki/index.php/', + 'dataUrl': '../semantic_data.json', + 'preSimulate': true, // run simulation before starting, so we don't start with lines jumping around + 'labels': { + 'rotate': true, + }, + 'countries': ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia ', 'Portugal', 'Finland ', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland'] +}; + + +function getSizeForNode(node) { + // if (node.hasOwnProperty('https://schema.org/thumbnailUrl')) return nodeSize; + // if (weights[node['@id']]) return nodeSize * weights[node['@id']]; + // if (node['@id'] == firstNodeId) return nodeSize * 1.2; + // // everynode has at least one link. these should equal 1 + // return nodeSize * (.7 + Math.min(20, linkMap[node['@id']].length) / 40); + if (node.parent) { + return 2; + } + return CONFIG.nodeSize; +} + +function splitText(text) { + var characters = [" ", "-", "_", '\xAD']; + var charSplitPos = {}; + var mid = Math.floor(text.length / 2); + var splitPos = false; + var splitPosChar = false; + // split sentences + var _iteratorNormalCompletion6 = true; + var _didIteratorError6 = false; + var _iteratorError6 = undefined; + + try { + for (var _iterator6 = characters[Symbol.iterator](), _step6; !(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done); _iteratorNormalCompletion6 = true) { + var char = _step6.value; + + if (text.indexOf(char) < 0) { + continue; + } + var tmid = text.substr(0, mid).lastIndexOf(char); + if (tmid === -1) { + tmid = text.indexOf(char); + } + tmid += 1; // we want to cut _after_ the character + // console.log("Char", char, tmid); + if (splitPos === false || Math.abs(tmid - mid) < Math.abs(splitPos - mid)) { + // console.log("least!"); + splitPos = tmid; + splitPosChar = char; + } + } + // console.log("pos",splitPos) + + } catch (err) { + _didIteratorError6 = true; + _iteratorError6 = err; + } finally { + try { + if (!_iteratorNormalCompletion6 && _iterator6.return) { + _iterator6.return(); + } + } finally { + if (_didIteratorError6) { + throw _iteratorError6; + } + } + } + + if (splitPos === false) { + return false; + } + + var text1 = text.substr(0, splitPos).trim(); + var text2 = text.substr(splitPos).trim(); + + if (splitPosChar == '\xAD') { + text1 += "-"; + } + + // find most equal split + return [text1, text2]; +}; + +function getTitle(obj) { + if (obj.parent) { + return "sub of " + obj.parent.split('#', 1)[0].replace(/_/g, " ") + } + return obj['@id'].split('#', 1)[0].replace(/_/g, " ") +} +function getClasses(obj) { + if (!obj._INST) + return 'node'; + const classes = obj['_INST'].map(classId => classId.split('#', 1)[0]); + return 'node ' + classes.join(' '); +} +function getUrl(obj) { + return CONFIG.baseUrl + obj['@id'].split('#', 1)[0]; +} + + // see also: http://bl.ocks.org/dwtkns/4973620 -var width = window.innerWidth; -var height = window.innerHeight; - -const eu_countries = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia', 'Netherlands', 'Denmark', 'Poland', 'Estonia ', 'Portugal', 'Finland ', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia', 'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']; - -const config = { - 'max_y_rotation' : 55, -} - -var graph = { - nodes: [ - { id: "New York", lat: 40.706109, lon: -74.01194 }, - { id: "London", lat: 51.508070, lon: -0.126432 }, - { id: "Montevideo", lat: -34.901776, lon: -56.163983 }, - { id: "London-NewYork1" }, - { id: "London-NewYork2" }, - { id: "London-NewYork3" }, - { id: "Montevideo-London" } - ], - links: [ - { source: "New York", target: "London-NewYork1" }, - { source: "New York", target: "London-NewYork2" }, - { source: "New York", target: "London-NewYork3" }, - { source: "London-NewYork1", target: "London" }, - { source: "London-NewYork2", target: "London" }, - { source: "London-NewYork3", target: "London" }, - { source: "London", target: "Montevideo-London" }, - { source: "Montevideo-London", target: "Montevideo" } - ] -} +let width = window.innerWidth; +let height = window.innerHeight; -const force = d3.forceSimulation() - .force("link", d3.forceLink() - .id(function (d) { - return d.id; - }) - .distance(10)) - .force("charge", d3.forceManyBody().strength(-200)); - +const req_data = new Request(CONFIG.dataUrl, { method: 'GET' }); +const req_world = new Request('https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json', { method: 'GET' }); -const svg = d3.select("body") - .append("svg") - .attr("width", width) - .attr("height", height); +Promise.all([fetch(req_data), fetch(req_world)]) + .then(([res_data, res_world]) => { + return Promise.all([res_data.json(), res_world.json()]); + }) + .then(([data, world]) => { + buildGraph(data, world); + }).catch(error => { + console.error(error); + }); +; +// fetch(request) +// .then(response => { +// if (response.status === 200) { +// return response.json(); +// } else { +// throw new Error('Something went wrong on api server!'); +// } +// }) +// .then(data => { +// buildGraph(data); +// }).catch(error => { +// console.error(error); +// }); -const container = svg.append("g").attr("id", "container"); - -const projection = d3.geoOrthographic() - .center([0, 0]) - .translate([width / 2, height / 2]) - .rotate([ -12, -52, 0]) - .clipAngle(90) - .scale(height*1.5); - -const proj = d3.geoPath().projection(projection); -const graticule = d3.geoGraticule10(); - -const g_countries = container.append("g").attr("id", "countries"); -const g_borders = container.append("g").attr("id", "borders"); -container.append("g") - .append('path') - .attr("class", "graticule") - .attr('d', proj(graticule)) - .attr("fill", "none") - .attr("stroke-width", "!px") - .attr("stroke", (n) => { - return "lightgray"; - }); - ; - - -d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(function (world) { +function buildGraph(data, world) { // land = topojson.feature(world, world.objects.land) const borders = topojson.mesh(world, world.objects.countries, (a, b) => a !== b) console.log(borders); const countries = topojson.feature(world, world.objects.countries).features; // console.log(topojson.feature(world, world.objects.countries).features); + + const nodes = data.nodes.filter(n => n._INST || n.parent).map(d => Object.create(d)); + const nodeMap = Object.fromEntries(nodes.map(d => [d['@id'], d])); + const links = data.links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(d => Object.create(d)); + + const simulation = d3.forceSimulation(nodes) + .force("link", d3.forceLink(links) + .id(d => d['@id']) + .iterations(2) // increase to make more rigid + ) + .force("charge", d3.forceManyBody() + .strength(-40) + ) + // .force("center", d3.forceCenter(width / 2, height / 2)) + .force("collision", d3.forceCollide(function (d) { + return getSizeForNode(d) * 1.5; // avoid overlapping nodes + })); + + const svg = d3.select("svg") + .attr("viewBox", [0, 0, width, height]); + const container = svg.append("g").attr("id", "container"); + + + const projection = d3.geoOrthographic() + .center([0, 0]) + .translate([width / 2, height / 2]) + .rotate([-12, -52, 0]) + .clipAngle(90) + .scale(height * 1.5); + + const proj = d3.geoPath().projection(projection); + const graticule = d3.geoGraticule10(); + + const g_countries = container.append("g").attr("id", "countries"); + const g_borders = container.append("g").attr("id", "borders"); + container.append("g") + .append('path') + .attr("class", "graticule") + .attr('d', proj(graticule)) + .attr("fill", "none") + .attr("stroke-width", "!px") + .attr("stroke", (n) => { + return "lightgray"; + }); + ; + + + + g_countries.selectAll("path") .data(countries) .enter() @@ -84,12 +196,12 @@ d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(f .attr("class", "countries") .attr("d", proj) .attr("fill", (n) => { - if(eu_countries.indexOf(n.properties.name) !== -1) { + if (CONFIG.countries.indexOf(n.properties.name) !== -1) { return ''; } return "lightgray"; }); - + g_borders//.selectAll("path") // .data(borders) // .enter() @@ -102,6 +214,324 @@ d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(f return "white"; }); + + const link = container.append("g") + .attr('class', 'links') + .selectAll(".link") + .data(links) + .join("g") + .attr("class", "link") + + const linkLine = link + .append("line"). + attr("marker-end", "url(#arrowHead)"); + const linkText = link.append("text").text(function (l) { + return l.name; + }); + + const node = container.append("g") + .attr('class', 'nodes') + .selectAll(".node") + .data(nodes) + .join("g") + .attr('class', getClasses) + .call(drag(simulation)) + .on("click", (evt, n) => selectNode(evt, n, node)) + ; + + + nodes.forEach(function (d) { + if (d.lon && d.lat) { + // console.log("fix node", d); + var p = projection([d.lon, d.lat]); + d.fx = p[0]; + d.fy = p[1]; + } + }) + + node + .append('circle') + .attr("r", getSizeForNode) + // .call(drag(simulation)); + + var nodeTitle = node.append('text').attr("class", "nodeTitle").attr("y", "5"); + nodeTitle + .each(function (node, nodes) { + var textLength = void 0; + const self = d3.select(this); + const titleText = getTitle(node); + var titleTexts = false; + if (titleText.length > 20) { + titleTexts = splitText(titleText); + } + if (titleTexts !== false) { + const tspan1 = self.append("tspan").text(titleTexts[0]).attr("y", "-10").attr("x", "0"); + const tspan = self.append("tspan").text(titleTexts[1]).attr("y", "10").attr("x", "0"); + const textLength1 = tspan.node().getComputedTextLength(); + const textLength2 = tspan.node().getComputedTextLength(); + textLength = Math.max(textLength1, textLength2); + } else { + self.text(titleText); + textLength = self.node().getComputedTextLength(); + } + // scale according to text length: + if (textLength > getSizeForNode(node) * 2) { + self.attr('transform', 'scale(' + getSizeForNode(node) * 2 / textLength / 1.05 + ')'); + } + }); + + // node.append("title") + // .text(d => d['@id']); + + svg.call(d3.zoom().scaleExtent([0.3, 6]).on("start", function () { + svg.node().classList.add("dragging"); + }).on("end", function () { + svg.node().classList.remove("dragging"); + }).on("zoom", function ({ transform }) { + container.attr("transform", transform); + })); + + + simulation.on("tick", () => { + + + data.nodes.forEach(function (d, idx) { + d.leftX = d.rightX = d.x; + // fix first node on center + // if(idx === 0) { + // d.fx = width/2; + // d.fy = height/2; + // return; + // } + }); + link + .attr("x1", d => d.source.x) + .attr("y1", d => d.source.y) + .attr("x2", d => d.target.x) + .attr("y2", d => d.target.y); + + linkLine.each(function (d) { + var sourceX, targetX, midX, dx, dy, angle; + + // This mess makes the arrows exactly perfect. + // thanks to http://bl.ocks.org/curran/9b73eb564c1c8a3d8f3ab207de364bf4 + if (d.source.x < d.target.x) { + sourceX = d.source.x; + targetX = d.target.x; + } else if (d.target.x < d.source.x) { + targetX = d.target.x; + sourceX = d.source.x; + } else if (d.target.isCircle) { + targetX = sourceX = d.target.x; + } else if (d.source.isCircle) { + targetX = sourceX = d.source.x; + } else { + midX = (d.source.x + d.target.x) / 2; + if (midX > d.target.x) { + midX = d.target.x; + } else if (midX > d.source.x) { + midX = d.source.x; + } else if (midX < d.target.x) { + midX = d.target.x; + } else if (midX < d.source.x) { + midX = d.source.x; + } + targetX = sourceX = midX; + } + + dx = targetX - sourceX; + dy = d.target.y - d.source.y; + angle = Math.atan2(dx, dy); + + /* DISABLED + srcSize = (typeof nodePositions[d.source.index] != 'undefined') ? selectedNodeSize : nodeSize; + tgtSize = (typeof nodePositions[d.target.index] != 'undefined') ? selectedNodeSize : nodeSize; + */ + var srcSize = getSizeForNode(d.source); + var tgtSize = getSizeForNode(d.target); + + // Compute the line endpoint such that the arrow + // is touching the edge of the node rectangle perfectly. + d.sourceX = sourceX + Math.sin(angle) * srcSize; + d.targetX = targetX - Math.sin(angle) * tgtSize; + d.sourceY = d.source.y + Math.cos(angle) * srcSize; + d.targetY = d.target.y - Math.cos(angle) * tgtSize; + }).attr("x1", function (d) { + return d.sourceX; + }).attr("y1", function (d) { + return d.sourceY; + }).attr("x2", function (d) { + return d.targetX; + }).attr("y2", function (d) { + return d.targetY; + }); + linkText.attr("transform", function (d) { + const dx = (d.target.x - d.source.x) / 2; + const dy = (d.target.y - d.source.y) / 2; + const x = d.source.x + dx; + const y = d.source.y + dy; + const deg = Math.atan(dy / dx) * 180 / Math.PI; + // if dx/dy == 0/0 -> deg == NaN + if (isNaN(deg)) { + return ""; + } + // return ""; + return "translate(" + x + " " + y + ") rotate(" + (CONFIG.labels.rotate ? deg : 0) + ")"; + }); + + node + .attr("transform", d => `translate(${d.x}, ${d.y})`); + }); + + + + function refresh() { + container.selectAll(".countries").attr("d", proj); + container.selectAll(".graticule").attr("d", proj(graticule)); + container.selectAll(".borders").attr("d", proj(borders)); + + simulation.alpha = 0; + simulation.restart(); + nodes.forEach(function (d) { + if (d.lon && d.lat) { + var p = projection([d.lon, d.lat]); + d.fx = p[0]; + d.fy = p[1]; + } + }) + + } + + + + d3.geoZoom() + .northUp(true) + .projection(projection) + .scaleExtent([.3, 6]) + .onMove(refresh)(svg.node()); + + function resize() { + width = window.innerWidth; + height = window.innerHeight; + + d3.selectAll('svg') + .attr("width", width) + .attr("height", height); + projection.translate([width / 2, height / 2]); + projection.scale(height * 1.5); + refresh(); + } + window.addEventListener('resize', resize); + + + + // simulate the first bit without drawing, so we don't have the 'jumping' graph in the beginning + if (CONFIG.preSimulate) { + for (var i = 0, n = Math.ceil(Math.log(simulation.alphaMin()) / Math.log(1 - simulation.alphaDecay())); i < n; ++i) { + simulation.tick(); + } + } + + + return svg.node(); +} + +color = _ => { + const scale = d3.scaleOrdinal(d3.schemeCategory10); + return d => scale(d.group); +}; + +const drag = simulation => { + + function dragstarted(event) { + if (!event.active) simulation.alphaTarget(0.3).restart(); + event.subject.fx = event.subject.x; + event.subject.fy = event.subject.y; + } + + function dragged(event) { + event.subject.fx = event.x; + event.subject.fy = event.y; + } + + function dragended(event) { + if (!event.active) simulation.alphaTarget(0); + event.subject.fx = null; + event.subject.fy = null; + } + + return d3.drag() + .on("start", dragstarted) + .on("drag", dragged) + .on("end", dragended); +}; + +function selectNode(evt, node, d3Node) { + console.log(evt, node, d3Node); + document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected')); + d3Node._groups[0][node.index].classList.add('selected'); + + infoEl = document.getElementById('nodeInfo'); + infoEl.classList.remove('hidden'); + + const url = getUrl(node); + const hrefEl = infoEl.querySelector('.nodeHref'); + hrefEl.textContent = getTitle(node); + hrefEl.setAttribute('href', url); + infoEl.querySelector('.nodeContents').src = url; + +} + +document.getElementById('closeInfo').addEventListener('click', (evt) => { + document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected')); + document.getElementById('nodeInfo').classList.add('hidden'); +}) + + +// var graph = { +// nodes: [ +// { id: "New York", lat: 40.706109, lon: -74.01194 }, +// { id: "London", lat: 51.508070, lon: -0.126432 }, +// { id: "Montevideo", lat: -34.901776, lon: -56.163983 }, +// { id: "London-NewYork1" }, +// { id: "London-NewYork2" }, +// { id: "London-NewYork3" }, +// { id: "Montevideo-London" } +// ], +// links: [ +// { source: "New York", target: "London-NewYork1" }, +// { source: "New York", target: "London-NewYork2" }, +// { source: "New York", target: "London-NewYork3" }, +// { source: "London-NewYork1", target: "London" }, +// { source: "London-NewYork2", target: "London" }, +// { source: "London-NewYork3", target: "London" }, +// { source: "London", target: "Montevideo-London" }, +// { source: "Montevideo-London", target: "Montevideo" } +// ] +// } + + +// const force = d3.forceSimulation() +// .force("link", d3.forceLink() +// .id(function (d) { +// return d.id; +// }) +// .distance(10)) +// .force("charge", d3.forceManyBody().strength(-200)); + + +// const svg = d3.select("body") +// .append("svg") +// .attr("width", width) +// .attr("height", height); + +// const container = svg.append("g").attr("id", "container"); + + +d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(function (world) { + +return; + const links = container.append('g') .attr("id", "links") .selectAll("line") @@ -165,120 +595,65 @@ d3.json("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json").then(f }) - function dragInicia(d) { - if (!d.lon || !d.lat) { - if (!d3.event.active) force.alphaTarget(0.3).restart(); - d.fx = d.x; - d.fy = d.y; - } - } + // function dragInicia(d) { + // if (!d.lon || !d.lat) { + // if (!d3.event.active) force.alphaTarget(0.3).restart(); + // d.fx = d.x; + // d.fy = d.y; + // } + // } - function dragging(d) { - if (!d.lon || !d.lat) { - d.fx = d3.event.x; - d.fy = d3.event.y; - } - } + // function dragging(d) { + // if (!d.lon || !d.lat) { + // d.fx = d3.event.x; + // d.fy = d3.event.y; + // } + // } - function dragTermina(d) { - if (!d.lon || !d.lat) { - if (!d3.event.active) force.alphaTarget(0); - d.fx = null; - d.fy = null; - } - } + // function dragTermina(d) { + // if (!d.lon || !d.lat) { + // if (!d3.event.active) force.alphaTarget(0); + // d.fx = null; + // d.fy = null; + // } + // } function refresh() { - container.selectAll(".countries").attr("d", proj); - container.selectAll(".graticule").attr("d", proj(graticule)); - container.selectAll(".borders").attr("d", proj(borders)); + container.selectAll(".countries").attr("d", proj); + container.selectAll(".graticule").attr("d", proj(graticule)); + container.selectAll(".borders").attr("d", proj(borders)); - // console.log(graph.nodes); - force.alpha = 0; - force.restart(); - graph.nodes.forEach(function (d) { - if (d.lon && d.lat) { - var p = projection([d.lon, d.lat]); - d.fx = p[0]; - d.fy = p[1]; - } - }) + force.alpha = 0; + force.restart(); + graph.nodes.forEach(function (d) { + if (d.lon && d.lat) { + var p = projection([d.lon, d.lat]); + d.fx = p[0]; + d.fy = p[1]; + } + }) } - - // svg.call(d3.zoom().extent([[0,0],[1,1]]).scaleExtent([0.3, 6]).on("start", function () { - // // svg.node().classList.add("dragging"); - // }).on("end", function () { - // // svg.node().classList.remove("dragging"); - // }).on("zoom", function ({ transform }) { - // // container.attr("transform", transform); - // o = [transform.x/6, -transform.y/6]; - // o[1] = o[1] > config.max_y_rotation ? config.max_y_rotation : - // o[1] < -config.max_y_rotation ? -config.max_y_rotation : - // o[1]; - // projection .rotate(o).scale(window.innerHeight*1.5*transform.k) - - // refresh(); - // // container.attr("transform", "scale(" + transform.k + ")"); - // })); - // svg.on("wheel.zoom", (e) => { - - // }, {passive: false}) - d3.geoZoom() - .northUp(true) - .projection(projection) - .scaleExtent([.3, 6]) - .onMove(refresh)(svg.node()); - // (); + .northUp(true) + .projection(projection) + .scaleExtent([.3, 6]) + .onMove(refresh)(svg.node()); - - - // d3.select('svg').call(d3.drag() - // .on("start", mousedown) - // .on("drag", mousemove) - // .on("end", mouseup)); - // var m0, o0; - // function mousedown(e) { - // m0 = [d3.event.x, d3.event.y]; - // o0 = projection.rotate(); - // console.log(e, d3.event, m0,o0); - // // d3.event.preventDefault(); - // } - // function mousemove() { - // if (m0) { - // var m1 = [d3.event.x, d3.event.y] - // , o1 = [o0[0] + (m1[0] - m0[0]) / 6, o0[1] + (m0[1] - m1[1]) / 6]; - // o1[1] = o1[1] > config.max_y_rotation ? config.max_y_rotation : - // o1[1] < -config.max_y_rotation ? -config.max_y_rotation : - // o1[1]; - // projection.rotate(o1); - // // sky.rotate(o1); - // refresh(); - // } - // } - // function mouseup() { - // if (m0) { - // mousemove(); - // m0 = null; - // } - // } - - - function resize(){ + function resize() { width = window.innerWidth; height = window.innerHeight; d3.selectAll('svg') - .attr("width", width) - .attr("height", height); + .attr("width", width) + .attr("height", height); projection.translate([width / 2, height / 2]); - projection.scale(height*1.5); + projection.scale(height * 1.5); refresh(); } window.addEventListener('resize', resize); diff --git a/www/index.html b/www/index.html index feb9543..f62819e 100644 --- a/www/index.html +++ b/www/index.html @@ -2,308 +2,169 @@ - - - - - + + + + \ No newline at end of file