Basic graph version

This commit is contained in:
Ruben van de Ven 2021-03-10 21:15:12 +01:00
commit 157bdef2ee
6 changed files with 782 additions and 0 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
semantic_data.json
d3.v6.min.js
.vscode/

18
README.md Normal file
View File

@ -0,0 +1,18 @@
# SMW Graph
This repository contains a script to pull semantic data out of Semantic Mediawiki and into a json. In turn, this json file can be loaded by graph.js to build a d3 graph.
Built for [Security Vision](https://securityvision.io).
## Installation
```bash
pip install -r requirements.txt
wget https://d3js.org/d3.v6.min.js
```
## Update:
```
python wiki_relations.py
```

352
graph.js Normal file
View File

@ -0,0 +1,352 @@
const CONFIG = {
'nodeSize': 16,
'baseUrl': 'https://www.securityvision.io/wiki/index.php/',
'dataUrl': 'semantic_data.json',
'labels': {
'rotate': true,
},
};
function getSizeForNode(node) {
// if (node.hasOwnProperty('https://schema.org/thumbnailUrl')) return nodeSize;
// if (weights[node['@id']]) return nodeSize * weights[node['@id']];
// if (node['@id'] == firstNodeId) return nodeSize * 1.2;
// // everynode has at least one link. these should equal 1
// return nodeSize * (.7 + Math.min(20, linkMap[node['@id']].length) / 40);
if (node.parent) {
return 2;
}
return CONFIG.nodeSize;
}
function splitText(text) {
var characters = [" ", "-", "_", '\xAD'];
var charSplitPos = {};
var mid = Math.floor(text.length / 2);
var splitPos = false;
var splitPosChar = false;
// split sentences
var _iteratorNormalCompletion6 = true;
var _didIteratorError6 = false;
var _iteratorError6 = undefined;
try {
for (var _iterator6 = characters[Symbol.iterator](), _step6; !(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done); _iteratorNormalCompletion6 = true) {
var char = _step6.value;
if (text.indexOf(char) < 0) {
continue;
}
var tmid = text.substr(0, mid).lastIndexOf(char);
if (tmid === -1) {
tmid = text.indexOf(char);
}
tmid += 1; // we want to cut _after_ the character
// console.log("Char", char, tmid);
if (splitPos === false || Math.abs(tmid - mid) < Math.abs(splitPos - mid)) {
// console.log("least!");
splitPos = tmid;
splitPosChar = char;
}
}
// console.log("pos",splitPos)
} catch (err) {
_didIteratorError6 = true;
_iteratorError6 = err;
} finally {
try {
if (!_iteratorNormalCompletion6 && _iterator6.return) {
_iterator6.return();
}
} finally {
if (_didIteratorError6) {
throw _iteratorError6;
}
}
}
if (splitPos === false) {
return false;
}
var text1 = text.substr(0, splitPos).trim();
var text2 = text.substr(splitPos).trim();
if (splitPosChar == '\xAD') {
text1 += "-";
}
// find most equal split
return [text1, text2];
};
function getTitle(obj) {
if(obj.parent) {
return "sub of " + obj.parent.split('#', 1)[0].replace('_', ' ');
}
return obj['@id'].split('#', 1)[0].replace('_', ' ');
}
function getClasses(obj) {
if (!obj._INST)
return 'node';
const classes = obj['_INST'].map(classId => classId.split('#', 1)[0]);
return 'node ' + classes.join(' ');
}
function getUrl(obj) {
return CONFIG.baseUrl + obj['@id'].split('#', 1)[0];
}
let width = window.innerWidth;
let height = window.innerHeight;
const request = new Request(CONFIG.dataUrl, { method: 'GET' });
fetch(request)
.then(response => {
if (response.status === 200) {
return response.json();
} else {
throw new Error('Something went wrong on api server!');
}
})
.then(data => {
buildGraph(data);
}).catch(error => {
console.error(error);
});
function buildGraph(data) {
console.log(data);
const nodes = data.nodes.filter(n => n._INST || n.parent).map(d => Object.create(d));
const nodeMap = Object.fromEntries(nodes.map(d => [d['@id'], d]));
const links = data.links.filter(l => nodeMap[l.source] && nodeMap[l.target]).map(d => Object.create(d));
const simulation = d3.forceSimulation(nodes)
.force("link", d3.forceLink(links)
.id(d => d['@id'])
.iterations(2) // increase to make more rigid
)
.force("charge", d3.forceManyBody()
.strength(-50)
)
.force("center", d3.forceCenter(width / 2, height / 2))
.force("collision", d3.forceCollide(function (d) {
return getSizeForNode(d) * 1.5; // avoid overlapping nodes
}));
const svg = d3.select("svg")
.attr("viewBox", [0, 0, width, height]);
const container = svg.append("g").attr("id", "container");
const link = container.append("g")
.attr('class', 'links')
.selectAll(".link")
.data(links)
.join("g")
.attr("class", "link")
const linkLine = link
.append("line").
attr("marker-end", "url(#arrowHead)");
const linkText = link.append("text").text(function (l) {
return l.name;
});
const node = container.append("g")
.attr('class', 'nodes')
.selectAll(".node")
.data(nodes)
.join("g")
.attr('class', getClasses)
.call(drag(simulation))
.on("click", (evt, n) => selectNode(evt, n, node))
;
node
.append('circle')
.attr("r", getSizeForNode)
// .call(drag(simulation));
var nodeTitle = node.append('text').attr("class", "nodeTitle").attr("y", "5");
nodeTitle
.each(function (node, nodes) {
var textLength = void 0;
const self = d3.select(this);
const titleText = getTitle(node);
var titleTexts = false;
if (titleText.length > 20) {
titleTexts = splitText(titleText);
}
if (titleTexts !== false) {
const tspan1 = self.append("tspan").text(titleTexts[0]).attr("y", "-10").attr("x", "0");
const tspan = self.append("tspan").text(titleTexts[1]).attr("y", "10").attr("x", "0");
const textLength1 = tspan.node().getComputedTextLength();
const textLength2 = tspan.node().getComputedTextLength();
textLength = Math.max(textLength1, textLength2);
} else {
self.text(titleText);
textLength = self.node().getComputedTextLength();
}
// scale according to text length:
if (textLength > getSizeForNode(node) * 2) {
self.attr('transform', 'scale(' + getSizeForNode(node) * 2 / textLength / 1.05 + ')');
}
});
// node.append("title")
// .text(d => d['@id']);
svg.call(d3.zoom().scaleExtent([0.3, 6]).on("start", function () {
svg.node().classList.add("dragging");
}).on("end", function () {
svg.node().classList.remove("dragging");
}).on("zoom", function ({ transform }) {
container.attr("transform", transform);
}));
simulation.on("tick", () => {
data.nodes.forEach(function (d, idx) {
d.leftX = d.rightX = d.x;
// fix first node on center
// if(idx === 0) {
// d.fx = width/2;
// d.fy = height/2;
// return;
// }
});
link
.attr("x1", d => d.source.x)
.attr("y1", d => d.source.y)
.attr("x2", d => d.target.x)
.attr("y2", d => d.target.y);
linkLine.each(function (d) {
var sourceX, targetX, midX, dx, dy, angle;
// This mess makes the arrows exactly perfect.
// thanks to http://bl.ocks.org/curran/9b73eb564c1c8a3d8f3ab207de364bf4
if (d.source.x < d.target.x) {
sourceX = d.source.x;
targetX = d.target.x;
} else if (d.target.x < d.source.x) {
targetX = d.target.x;
sourceX = d.source.x;
} else if (d.target.isCircle) {
targetX = sourceX = d.target.x;
} else if (d.source.isCircle) {
targetX = sourceX = d.source.x;
} else {
midX = (d.source.x + d.target.x) / 2;
if (midX > d.target.x) {
midX = d.target.x;
} else if (midX > d.source.x) {
midX = d.source.x;
} else if (midX < d.target.x) {
midX = d.target.x;
} else if (midX < d.source.x) {
midX = d.source.x;
}
targetX = sourceX = midX;
}
dx = targetX - sourceX;
dy = d.target.y - d.source.y;
angle = Math.atan2(dx, dy);
/* DISABLED
srcSize = (typeof nodePositions[d.source.index] != 'undefined') ? selectedNodeSize : nodeSize;
tgtSize = (typeof nodePositions[d.target.index] != 'undefined') ? selectedNodeSize : nodeSize;
*/
var srcSize = getSizeForNode(d.source);
var tgtSize = getSizeForNode(d.target);
// Compute the line endpoint such that the arrow
// is touching the edge of the node rectangle perfectly.
d.sourceX = sourceX + Math.sin(angle) * srcSize;
d.targetX = targetX - Math.sin(angle) * tgtSize;
d.sourceY = d.source.y + Math.cos(angle) * srcSize;
d.targetY = d.target.y - Math.cos(angle) * tgtSize;
}).attr("x1", function (d) {
return d.sourceX;
}).attr("y1", function (d) {
return d.sourceY;
}).attr("x2", function (d) {
return d.targetX;
}).attr("y2", function (d) {
return d.targetY;
});
linkText.attr("transform", function (d) {
const dx = (d.target.x - d.source.x) / 2;
const dy = (d.target.y - d.source.y) / 2;
const x = d.source.x + dx;
const y = d.source.y + dy;
const deg = Math.atan(dy / dx) * 180 / Math.PI;
// if dx/dy == 0/0 -> deg == NaN
if (isNaN(deg)) {
return "";
}
// return "";
return "translate(" + x + " " + y + ") rotate(" + (CONFIG.labels.rotate ? deg : 0) + ")";
});
node
.attr("transform", d => `translate(${d.x}, ${d.y})`);
});
return svg.node();
}
color = _ => {
const scale = d3.scaleOrdinal(d3.schemeCategory10);
return d => scale(d.group);
};
const drag = simulation => {
function dragstarted(event) {
if (!event.active) simulation.alphaTarget(0.3).restart();
event.subject.fx = event.subject.x;
event.subject.fy = event.subject.y;
}
function dragged(event) {
event.subject.fx = event.x;
event.subject.fy = event.y;
}
function dragended(event) {
if (!event.active) simulation.alphaTarget(0);
event.subject.fx = null;
event.subject.fy = null;
}
return d3.drag()
.on("start", dragstarted)
.on("drag", dragged)
.on("end", dragended);
};
function selectNode(evt, node, d3Node){
console.log(evt, node, d3Node);
document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected'));
d3Node._groups[0][node.index].classList.add('selected');
infoEl = document.getElementById('nodeInfo');
infoEl.classList.remove('hidden');
infoEl.querySelector('.nodeTitle').textContent = getTitle(node);
infoEl.querySelector('.nodeContents').src = getUrl(node);
}
document.getElementById('closeInfo').addEventListener('click', (evt) => {
document.querySelectorAll('svg .node').forEach(n => n.classList.remove('selected'));
document.getElementById('nodeInfo').classList.add('hidden');
})

168
index.html Normal file
View File

@ -0,0 +1,168 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Security Vision Semantic Graph</title>
<style>
:root{
--color1: #f94144;
--color2: #f3722c;
--color3: #f8961e;
/* --color4: #f9844a; */
--color5: #f9c74f;
--color6: #90be6d;
--color7: #43aa8b;
--color8: #4d908e;
--color9: #577590;
--color10: #277da1;
--hover-color: var(--color1);
--selected-color: var(--color1);
--selected-color: var(--color1);
}
body {
margin: 0;
overflow: hidden;
background: #333;
font-family: sans-serif;
}
svg {
cursor: grab;
font-family: sans-serif;
}
svg.dragging {
cursor: grabbing;
}
svg .links line{
stroke: lightgray;
stroke-width: 1;
}
.links text{
/* display:none; */
font-size:5pt;
text-anchor: middle;
fill: whitesmoke;
}
.node text{
text-anchor: middle;
}
.node circle{
fill: white;
}
.node:hover{
cursor: pointer;
}
.node:hover circle{
stroke: var(--hover-color);
stroke-width: 5px;
}
.node.selected circle{
stroke: var(--selected-color);
stroke-width: 5px;
}
.node.Person circle {
fill: lightgreen
}
.node.Technology circle {
fill: lightcoral;
}
.node.Deployment circle {
fill: lightblue;
}
.node.Institution circle {
fill: lightgoldenrodyellow
}
.node.Dataset circle {
fill: plum
}
/* .node.Person circle {
fill: var(--color2)
}
.node.Technology circle {
fill: var(--color3);
}
.node.Deployment circle {
fill: var(--color5);
}
.node.Institution circle {
fill: var(--color6)
}
.node.Dataset circle {
fill: var(--color7)
} */
#nodeInfo{
position: fixed;
display:block;
right:20px;
bottom:20px;
background:white;
padding: 10px;
border: solid 1px #ccc;
}
#nodeInfo.hidden{
display:none;
}
#nodeInfo h2{
margin: 0;
padding: 0;
}
#nodeInfo iframe{
width: 50vw;
height: calc(100vh - 40px - 20px - 30px);
}
#closeInfo{
cursor: pointer;
position: absolute;
right: 10px;
top: 10px;
}
#closeInfo:hover{
color: var(--hover-color);
}
</style>
</head>
<body>
<svg id='graph'>
<defs>
<marker markerHeight="4" markerWidth="4" refY="0" refX="6" viewBox="0 -3 8 6" preserveAspectRatio="none" orient="auto" id="arrowHead" fill="lightgray"><path d="M0,-3L8,0L0,3"></path></marker>
<marker markerHeight="4" markerWidth="4" refY="0" refX="6" viewBox="0 -3 8 6" preserveAspectRatio="none" orient="auto" id="arrowHeadSelected"><path d="M0,-3L8,0L0,3" fill="white"></path></marker>
</defs>
</svg>
<div id="nodeInfo" class='hidden'>
<h2 class='nodeTitle'></h2>
<div id='closeInfo'>&times;</div>
<iframe class='nodeContents'></iframe>
</div>
</body>
<!-- <script src="https://d3js.org/d3.v6.min.js"></script> -->
<script src="d3.v6.min.js"></script>
<script src="graph.js"></script>
</html>

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
tqdm
requests

238
wiki_relations.py Normal file
View File

@ -0,0 +1,238 @@
import urllib.request, json
import logging
import requests
import argparse
import datetime
import tqdm
logger = logging.getLogger('wiki')
default_categories = [
'Person',
'Institution',
'Technology',
'Deployment',
'Dataset',
]
username = "Ruben2@SemanticGraphFetcher"
password = "bdqjse4jodn34rbj73l0agrtb306v693"
parser = argparse.ArgumentParser(description='Turn wiki into nodes & links, usable by d3-force.')
parser.add_argument('--categories', metavar='categories', default=default_categories, nargs='+',
help='Categories')
parser.add_argument('--url', default="https://www.securityvision.io/wiki/api.php",
help='Wiki API URL')
parser.add_argument('--output', default="semantic_data.json",
help='Output JSON file')
args = parser.parse_args()
def get_session():
S = requests.Session()
URL = args.url
# Retrieve login token first
PARAMS_0 = {
'action':"query",
'meta':"tokens",
'type':"login",
'format':"json"
}
R = S.get(url=URL, params=PARAMS_0)
DATA = R.json()
logger.debug(DATA)
LOGIN_TOKEN = DATA['query']['tokens']['logintoken']
logger.debug(LOGIN_TOKEN)
# Send a post request to login. Using the main account for login is not
# supported. Obtain credentials via Special:BotPasswords
# (https://www.mediawiki.org/wiki/Special:BotPasswords) for lgname & lgpassword
PARAMS_1 = {
'action':"login",
'lgname':username,
'lgpassword': password,
'lgtoken':LOGIN_TOKEN,
'format':"json"
}
R = S.post(URL, data=PARAMS_1)
DATA = R.json()
logger.debug(DATA)
if DATA['login']['result'] != 'Success':
raise Exception("Failed logging in")
return S
def getPagesForCategory(category, session):
logging.info(f"Get pages in category: {category}")
pages = []
baseurl = f"{args.url}?action=query&list=categorymembers&cmtitle=Category:{category}&format=json"
params = {
'action': 'query',
'list': 'categorymembers',
'cmtitle': f'Category:{category}',
'format': 'json'
}
while True:
logger.debug(args.url, params)
response = session.post(args.url, data=params)
data = response.json()
try:
logger.debug(f"Fetched {len(data['query']['categorymembers'])} of category {category}")
pages.extend(data['query']['categorymembers'])
except Exception as e:
logger.error(data)
raise e
if 'continue' not in data:
break
params['cmcontinue'] = data['continue']['cmcontinue']
return pages
def getPropertiesForPages(pages, session, collection):
for page in tqdm.tqdm(pages):
links = getPropertiesForPage(page, session, collection)
def getPropertiesForPage(page, session, collection):
links = []
params = {
'action': 'smwbrowse',
'browse': 'subject',
'format': 'json',
'params': json.dumps({
'subject': page['title'],
'ns': page['ns'],
"iw": ""
})
}
response = session.post(args.url, data=params)
data = response.json()
# subject:
# data:
# sobj: subobjects
# print(data['query']['data'])
# Types:
# - 2: Text/String
# - 6: Date
# - 9: Page
subjectId = data['query']['subject']
for rel in data['query']['data']:
addToCollection(subjectId, rel, collection)
if 'sobj' not in data['query']:
return
for sub_obj in data['query']['sobj']:
subSubjectId = sub_obj['subject']
for rel in sub_obj['data']:
addToCollection(subSubjectId, rel, collection, subjectId)
def addToCollection(subjectId, rel, collection, isSubObjOf = None):
if rel['property'] in ["_SKEY", "_MDAT", "_ASKDE", "_ASKSI"]:
logger.debug(f"Skipping {rel['property']} for {subjectId}")
return
if subjectId not in collection['nodes']:
collection['nodes'][subjectId] = getObjForSubject(subjectId)
if isSubObjOf:
collection['nodes'][subjectId]['parent'] = isSubObjOf
for data in rel['dataitem']:
addDataitemToCollection(subjectId, rel['property'], data, collection)
def addDataitemToCollection(subjectId, prop, data, collection):
# 2: Number (float or int) - keep string
# 2: string - keep string
# 5: url - keep string
# 6: date(time) : various resolutions 1/2021/3/1/21/54/54/0 or 1/2020
if data['type'] == 1 or data['type'] == 2 or data['type'] == 5 or data['type'] == 6:
if prop not in collection['nodes'][subjectId]:
collection['nodes'][subjectId][prop] = []
value = data['item']
if data['type'] == 6:
parts = value.split("/")
if parts[0] == "2":
logger.warning(f"Date string seems to be Julian Calendar, not supported but ignored for '{subjectId}'? {parts}")
elif parts[0] != "1":
logger.error(f"Date seems invallid for '{subjectId}'? {parts}")
del parts[0]
value = "/".join(parts)
# parts = [int(p) for p in parts]
# value = datetime.datetime(*parts).isoformat()
collection['nodes'][subjectId][prop].append(value)
# page (thus: a link/relationship)
elif data['type'] == 9:
if prop == '_INST':
# Category shouldn't be mapped as link for us
if prop not in collection['nodes'][subjectId]:
collection['nodes'][subjectId][prop] = []
collection['nodes'][subjectId][prop].append(data['item'])
elif prop in ['_ERRC', '_ERRP', '_ERRC']:
logger.warning(f"Error on page {subjectId}: {data}")
if prop not in collection['nodes'][subjectId]:
collection['nodes'][subjectId][prop] = []
collection['nodes'][subjectId][prop].append(json.dumps(data))
else:
if data['item'] not in collection['nodes']:
collection['nodes'][data['item']] = getObjForSubject(data['item'])
collection['links'].append({
'source': subjectId,
'target': data['item'],
'name': prop
})
else:
logger.error(f"Unknown type: {data['type']}: {prop} : {data}")
def getObjForSubject(sub):
obj = {
'@id': sub,
}
return obj
if __name__ == "__main__":
logger.setLevel(logging.INFO)
session = get_session()
collection = {'nodes': {}, 'links': []}
for category in args.categories:
logger.info(f"Fetch pages for category '{category}'")
pages = getPagesForCategory(category, session)
logger.info(f"Pages in category '{category}': {len(pages)}")
getPropertiesForPages(pages, session, collection)
custompage = {
'title': 'Resources',
'ns': 0
}
getPropertiesForPages([custompage], session, collection)
# [{'property': 'Based_in', 'dataitem': [{'type': 9, 'item': 'Berlin#0##'}]}, {'property': 'WikidataID', 'dataitem': [{'type': 2, 'item': 'Q57168389'}]}, {'property': '_INST', 'dataitem': [{'type': 9, 'item': 'Person#14##'}]}, {'property': '_MDAT', 'dataitem': [{'type': 6, 'item': '1/2021/3/1/21/13/7/0'}]}, {'property': '_SKEY', 'dataitem': [{'type': 2, 'item': 'Adam Harvey'}]}]
logger.info(f"Nodes: {len(collection['nodes'])} Links: {len(collection['links'])}")
# convert to list
collection['nodes'] = list(collection['nodes'].values())
logger.info(f"Write to {args.output}")
with open(args.output, 'w') as fp:
json.dump(collection, fp)